komplex-hub/KomplexHubPlugin/packcompiler.cpp

#include "packcompiler.h"
#include "common/exceptions.h"
#include <utility>

PackCompiler::PackCompiler(QObject *parent)
    : QObject(parent)
{
    if(m_variableExpressions.isEmpty())
    {
        for(const QString &variable : m_updateVariables)
        {
            m_variableExpressions.append
            (
                QRegularExpression
                (
                    QStringLiteral("\\b(?<!ubuf\\.)%1\\b").arg
                    (
                        variable
                    )
                )
            );
        }
    }
}

PackCompiler::~PackCompiler() = default;

auto PackCompiler::reset() -> void
{
    setError({}, {});
    setProgress(std::numeric_limits<qreal>::infinity());
    setTotalSteps(0);
    setCurrentStep(0);
    setCompilerOutput({});
    setStatus({});
    setState(Idle);
}

auto PackCompiler::process(const QUrl &uri) -> QFuture<QUrl>
{
    return QtConcurrent::run
    (
        [this, uri] () -> QUrl
        {
            if(!uri.isLocalFile())
            {
                setError
                (
                    QStringLiteral("File Error"),
                    QStringLiteral("URI %1 is not a local file").arg
                    (
                        uri.toString()
                    )
                );

                return {};
            }

            reset();
            setStatus(QStringLiteral("Compiling %1").arg(uri.fileName()));
            setState(Compiling);

            if(!validateDirectory(uri))
            {
                throw FileException
                (
                    QStringLiteral("Source directory is invalid")
                );
            }

            QUrl buildUri
            (
                QStringLiteral("file://%1/komplex/build/%2").arg
                (
                    QStandardPaths::writableLocation
                    (
                        QStandardPaths::TempLocation
                    ),
                    uri.fileName()
                )
            );

            try
            {
                extract(uri, buildUri);
                prepareShaders(buildUri);
                compile(buildUri);
            }
            catch (FileException e)
            {
                removeDirectory(buildUri);
                setError(QStringLiteral("File Error"), e.message);
                throw e;
            }
            catch (ShaderCompilerException e)
            {
                removeDirectory(buildUri);
                setError(QStringLiteral("Compiler Error"), e.message);
                throw e;
            }

            return buildUri;
        }
    );
}

auto PackCompiler::prepareShaders(const QUrl &uri) noexcept(false) -> void
{
    if(!uri.isLocalFile())
    {
        throw FileException
        (
            QStringLiteral("URI %1 is not a local file uri").arg
            (
                uri.toString()
            )
        );
    }

    QDir sourceDirectory(uri.toLocalFile());
    QDir shaderDirectory
    (
        sourceDirectory.absoluteFilePath
        (
            QStringLiteral("shaders")
        )
    );

    QStringList entries = shaderDirectory.entryList
    (
        QDir::NoDotAndDotDot | QDir::Files
    );

    setTotalSteps(entries.count() * 2);
    setProgress(0);

    QByteArray commonFragmentData = loadCommonFragmentData(uri);

    if(commonFragmentData.length() > 0)
    {
        setTotalSteps(m_totalSteps -= 1);
        incrementCompileStep();
    }

    QByteArray commonVertexData = loadCommonVertexData(uri);

    if(commonVertexData.length() > 0)
    {
        setTotalSteps(m_totalSteps -= 1);
        incrementCompileStep();
    }

    QByteArray globalData = loadGlobalData(uri);

    if(globalData.length() > 0)
    {
        setTotalSteps(m_totalSteps -= 1);
        incrementCompileStep();
    }

    for(const QString &entry : std::as_const(entries))
    {
        auto commonFragMatch = m_commonFragmentExpression.match(entry);
        auto commonVertMatch = m_commonVertexExpression.match(entry);

        if(commonFragMatch.hasMatch() || commonVertMatch.hasMatch())
        {
            continue;
        }

        auto fragMatch = m_fragmentExpression.match(entry);
        auto vertMatch = m_vertexExpression.match(entry);

        QFile file(shaderDirectory.absoluteFilePath(entry));

        if(!file.open(QFile::ReadWrite))
        {
            throw FileException
            (
                QStringLiteral("Could not open file %1 for preperation").arg
                (
                    shaderDirectory.absoluteFilePath(entry)
                )
            );
        }

        QByteArray fileData = file.readAll();

        if(fileData.length() != file.size())
        {
            throw FileException
            (
                QStringLiteral("Could not read file %1 for preperation").arg
                (
                    shaderDirectory.absoluteFilePath(entry)
                )
            );
        }

        file.close();

        QByteArray commonData;

        if(fragMatch.hasMatch())
        {
            commonData = commonFragmentData;
        }
        else if(vertMatch.hasMatch())
        {
            commonData = commonVertexData;
        }
        else
        {
            throw FileException
            (
                QStringLiteral("File %1 is an unrecognized shader type (frag/vert)").arg
                (
                    shaderDirectory.absoluteFilePath(entry)
                )
            );
        }

        QByteArray combinedData = QByteArray
        (
            globalData + QByteArray("\n") +
            commonData + QByteArray("\n") +
            fileData
        );

        for(const QRegularExpression &expression : std::as_const(m_variableExpressions))
        {
            auto match = expression.match(combinedData);

            if(match.hasMatch())
            {
                QByteArray replacement = QByteArray("ubuf.") + match.captured().toLocal8Bit();
                combinedData.replace(match.captured().toLocal8Bit(), replacement);
            }
        }

        QByteArray preparedData = QByteArray
        (
            m_header.toLocal8Bit() + QByteArray("\n") +
            combinedData + QByteArray("\n") +
            m_footer.toLocal8Bit()
        );

        if(!file.open(QFile::ReadWrite | QFile::Truncate))
        {
            throw FileException
            (
                QStringLiteral("Could not open file %1 for writing preperation").arg
                (
                    shaderDirectory.absoluteFilePath(entry)
                )
            );
        }

        if(file.write(preparedData) != preparedData.length())
        {
            throw FileException
            (
                QStringLiteral("Could not write file %1 preperation").arg
                (
                    shaderDirectory.absoluteFilePath(entry)
                )
            );
        }

        file.close();
        incrementCompileStep();
    }
}

auto PackCompiler::loadCommonFragmentData(const QUrl &uri) -> QByteArray
{
    if(!uri.isLocalFile())
    {
        throw FileException
        (
            QStringLiteral("URI %1 is not a local file uri").arg
            (
                uri.toString()
            )
        );
    }

    QDir sourceDirectory
    (
        QStringLiteral("%1/shaders/").arg
        (
            uri.toLocalFile()
        )
    );

    const QStringList entries = sourceDirectory.entryList
    (
        QDir::NoDotAndDotDot | QDir::Files
    );

    QByteArray data;

    for(const QString &entry : entries)
    {
        if (entry.toLower() != QStringLiteral("common.frag"))
        {
            continue;
        }

        QFile file(sourceDirectory.absoluteFilePath(entry));

        if(!file.open(QFile::ReadWrite))
        {
            throw FileException(QStringLiteral("Could not open common shader data"));
        }

        data = file.readAll();

        if(data.length() != file.size())
        {
            throw FileException(QStringLiteral("Common fragment shader file read error"));
        }

        file.close();
        break;
    }

    return data;
}

auto PackCompiler::loadCommonVertexData(const QUrl &uri) -> QByteArray
{
    if(!uri.isLocalFile())
    {
        throw FileException
        (
            QStringLiteral("URI %1 is not a local file uri").arg
            (
                uri.toString()
            )
        );
    }

    QDir sourceDirectory
    (
        QStringLiteral("%1/shaders/").arg
        (
            uri.toLocalFile()
        )
    );

    const QStringList entries = sourceDirectory.entryList
    (
        QDir::NoDotAndDotDot | QDir::Files
    );

    QByteArray data;

    for(const QString &entry : entries)
    {
        if (entry.toLower() != QStringLiteral("common.vert"))
        {
            continue;
        }

        QFile file(sourceDirectory.absoluteFilePath(entry));

        if(!file.open(QFile::ReadWrite))
        {
            throw FileException(QStringLiteral("Could not open common vertex shader data"));
        }

        data = file.readAll();

        if(data.length() != file.size())
        {
            throw FileException(QStringLiteral("Common vertex shader file read error"));
        }

        file.close();
        break;
    }

    return data;
}

auto PackCompiler::loadGlobalData(const QUrl &uri) -> QByteArray
{
    if(!uri.isLocalFile())
    {
        throw FileException
        (
            QStringLiteral("URI %1 is not a local file uri").arg
            (
                uri.toString()
            )
        );
    }

    QDir sourceDirectory
    (
        QStringLiteral("%1/shaders/").arg
        (
            uri.toLocalFile()
        )
    );

    const QStringList entries = sourceDirectory.entryList
    (
        QDir::NoDotAndDotDot | QDir::Files
    );

    QByteArray data;

    for(const QString &entry : entries)
    {
        if (entry.toLower() != QStringLiteral("global.glsl"))
        {
            continue;
        }

        QFile file(sourceDirectory.absoluteFilePath(entry));

        if(!file.open(QFile::ReadWrite))
        {
            throw FileException
            (
                QStringLiteral("Could not open global shader data")
            );
        }

        data = file.readAll();

        if(data.length() != file.size())
        {
            throw FileException
            (
                QStringLiteral("Global shader file read error")
            );
        }

        file.close();
        break;
    }

    return data;
}

auto PackCompiler::validateDirectory(const QUrl &uri) -> bool
{
    QDir directory(uri.toLocalFile());

    return
    (
        directory.exists(QStringLiteral("shaders")) &&
        directory.exists(QStringLiteral("pack.json"))
    );
}

auto PackCompiler::extract(const QUrl &sourceUri, const QUrl &destinationUri) noexcept(false) -> void
{
    if(!sourceUri.isLocalFile() || !sourceUri.isValid())
    {
        throw FileException(QStringLiteral("Uri needs to be a local file"), 0);
    }

    QUrl outputUri
    (
        QStringLiteral("file://%1_proc").arg
        (
            sourceUri.toLocalFile()
        )
    );

    QStringList arguments =
    {
        QStringLiteral("-xzf"),
        sourceUri.toLocalFile(),
        QStringLiteral("-C"),
        outputUri.toLocalFile()
    };

    QProcess *process = new QProcess(this);

    QObject::connect
    (
        process,
        &QProcess::readyReadStandardOutput,
        this,
        [this, process]()
        {
            QByteArray processData = process->readAllStandardOutput();
            setCompilerOutput(m_compilerOutput + processData);
        }
    );

    QObject::connect
    (
        process,
        &QProcess::readyReadStandardError,
        this,
        [this, process]()
        {
            QByteArray processData = process->readAllStandardError();

            if(!processData.isValidUtf8())
            {
                qWarning() << QStringLiteral("Process output not valid UTF8 data");
                return;
            }

            setCompilerOutput(m_compilerOutput + processData);
        }
    );

    process->start(QStringLiteral("tar"), arguments);

    if(!process->waitForStarted(3000))
    {
        process->deleteLater();
        throw FileException(QStringLiteral("Could not start preprocessor"));
    }

    if(!process->waitForFinished())
    {
        process->deleteLater();
        throw FileException(QStringLiteral("Preprocessor timeout"));
    }

    if(process->exitCode() != 0)
    {
        process->deleteLater();
        throw ShaderCompilerException(m_compilerOutput);
    }

    process->deleteLater();
}

auto PackCompiler::copyFile(const QUrl &sourceUri, const QUrl &destinationUri) noexcept(false) -> void
{
    if(!QFile::exists(sourceUri.toLocalFile()))
    {
        return;
    }

    if(!QFile::copy(sourceUri.toLocalFile(), destinationUri.toLocalFile()))
    {
        throw FileException(QStringLiteral("Could not copy file"));
    }
}

auto PackCompiler::compile(const QUrl &uri) noexcept(false) -> void
{
    if(!uri.isLocalFile() || !uri.isValid())
    {
        throw FileException(QStringLiteral("Uri needs to be a local file"), 0);
    }

    setState(Compiling);

    QDir packDirectory(uri.toLocalFile());
    QDir shaderDirectory
    (
        packDirectory.absoluteFilePath
        (
            QStringLiteral("shaders")
        )
    );

    shaderDirectory.setNameFilters
    (
        {
            QStringLiteral("*.frag"),
            QStringLiteral("*.vert")
        }
    );

    const QStringList shaders = shaderDirectory.entryList
    (
        QDir::Files | QDir::NoDotAndDotDot
    );

    for(const QString &shader : shaders)
    {
        if(shader.startsWith(QStringLiteral("common."), Qt::CaseInsensitive))
        {
            continue;
        }

        QUrl shaderUri
        (
            QStringLiteral("file://%1").arg
            (
                shaderDirectory.absoluteFilePath(shader)
            )
        );

        preprocess(shaderUri);
        appendVersion(shaderUri);
        compileShader(shaderUri);

        incrementCompileStep();
    }
}

auto PackCompiler::preprocess(const QUrl &uri) noexcept(false) -> void
{
    if(!uri.isLocalFile() || !uri.isValid())
    {
        throw FileException(QStringLiteral("Uri needs to be a local file"), 0);
    }

    QUrl outputUri
    (
        QStringLiteral("file://%1_proc").arg
        (
            uri.toLocalFile()
        )
    );

    QStringList arguments =
    {
        QStringLiteral("-P"),
        uri.toLocalFile(),
        outputUri.toLocalFile()
    };

    QProcess *process = new QProcess(this);

    QObject::connect
    (
        process,
        &QProcess::readyReadStandardOutput,
        this,
        [this, process]()
        {
            QByteArray processData = process->readAllStandardOutput();
            setCompilerOutput(m_compilerOutput + processData);
        }
    );

    QObject::connect
    (
        process,
        &QProcess::readyReadStandardError,
        this,
        [this, process]()
        {
            QByteArray processData = process->readAllStandardError();

            if(!processData.isValidUtf8())
            {
                qWarning() << QStringLiteral("Process output not valid UTF8 data");
                return;
            }

            setCompilerOutput(m_compilerOutput + processData);
        }
    );

    process->start(QStringLiteral("cpp"), arguments);

    if(!process->waitForStarted(3000))
    {
        process->deleteLater();
        throw FileException(QStringLiteral("Could not start preprocessor"));
    }

    if(!process->waitForFinished())
    {
        process->deleteLater();
        throw FileException(QStringLiteral("Preprocessor timeout"));
    }

    if(process->exitCode() != 0)
    {
        process->deleteLater();
        throw ShaderCompilerException(m_compilerOutput);
    }

    process->deleteLater();

    QFile::remove(uri.toLocalFile());
    QFile::rename(outputUri.toLocalFile(), uri.toLocalFile());
}

auto PackCompiler::appendVersion(const QUrl &uri) noexcept(false) -> void
{
    if(!uri.isValid() || !uri.isLocalFile())
    {
        throw FileException(QStringLiteral("File is not a valid local file"));
    }

    QFile file(uri.toLocalFile());

    if(!file.open(QFile::ReadWrite))
    {
        throw FileException(QStringLiteral("Could not open file for appending"));
    }

    QByteArray data = file.readAll();

    file.seek(0);

    file.write(m_version);
    file.write(QByteArray("\n\n"));
    file.write(data);

    file.close();
}

auto PackCompiler::compileShader(const QUrl &uri) noexcept(false) -> void
{
    if(!uri.isLocalFile())
    {
        throw FileException
        (
            QStringLiteral("Uri needs to be a local file")
        );
    }

    QString qsb = QStringLiteral("/usr/lib/qt6/bin/qsb");

    if(!QFile::exists(qsb))
    {
        throw FileException(QStringLiteral("QSB missing"));
    }

    QUrl outputUri
    (
        QStringLiteral("file://%1.qsb").arg
        (
            uri.toLocalFile()
        )
    );

    QStringList arguments =
    {
        QStringLiteral("--glsl"),
        QStringLiteral("330es, 330, 440"),
        QStringLiteral("-o"),
        outputUri.toLocalFile(),
        uri.toLocalFile()
    };

    QProcess *process = new QProcess(this);
    QString path = uri.toDisplayString(QUrl::RemoveFilename | QUrl::RemoveScheme).remove(QStringLiteral("//"));
    process->setWorkingDirectory(path);

    QObject::connect
    (
        process,
        &QProcess::readyReadStandardOutput,
        this,
        [this, process]()
        {
            QByteArray processData = process->readAllStandardOutput();
            setCompilerOutput(m_compilerOutput + processData);
        }
    );

    QObject::connect
    (
        process,
        &QProcess::readyReadStandardError,
        this,
        [this, process]()
        {
            QByteArray processData = process->readAllStandardError();

            if(!processData.isValidUtf8())
            {
                qWarning() << QStringLiteral("Process output not valid UTF8 data");
                return;
            }

            setCompilerOutput(m_compilerOutput + processData);
        }
    );

    process->start(qsb, arguments);

    if(!process->waitForStarted(3000))
    {
        process->deleteLater();
        throw FileException(QStringLiteral("Could not start shader compiler"));
    }

    if(!process->waitForFinished())
    {
        process->deleteLater();
        throw FileException(QStringLiteral("Shader compiler timeout"));
    }

    if(process->exitCode() != 0)
    {
        process->deleteLater();
        throw ShaderCompilerException(m_compilerOutput);
    }

    process->deleteLater();

    if(!QFile::remove(uri.toLocalFile()))
    {
        throw FileException(QStringLiteral("Could not clean up"));
    }
}

auto PackCompiler::removeDirectory(const QUrl &uri) -> bool
{
    QStringList arguments =
    {
        QStringLiteral("-rf"),
        uri.toLocalFile()
    };

    QProcess *process = new QProcess(this);

    process->start(QStringLiteral("rm"), arguments);

    if(!process->waitForStarted(3000))
    {
        process->deleteLater();
        return false;
    }

    if(!process->waitForFinished())
    {
        process->deleteLater();
        return false;
    }

    if(process->exitCode() != 0)
    {
        process->deleteLater();
        return false;
    }

    process->deleteLater();
    return true;
}

auto PackCompiler::setProgress(qreal progress) -> void
{
    if(qFuzzyCompare(progress, m_progress))
    {
        return;
    }

    m_progress = progress;
    Q_EMIT progressChanged();
}

void PackCompiler::setCompilerOutput(const QString &compilerOutput)
{
    if (m_compilerOutput == compilerOutput)
    {
        return;
    }

    m_compilerOutput = compilerOutput;
    emit compilerOutputChanged();
}

auto PackCompiler::setError(const QString &title, const QString &message, const QUrl &uri) -> void
{
    if(title == m_errorTitle && message == m_errorMessage)
    {
        return;
    }

    m_errorMessage = std::move(message);
    m_errorTitle = std::move(title);

    if(uri.isValid() && uri.isLocalFile())
    {
        QFile errorFile(uri.toLocalFile());

        if(errorFile.open(QFile::ReadWrite | QFile::Append))
        {
            errorFile.write(message.toLocal8Bit() + QByteArray("\n"));
        }
    }

    Q_EMIT errorTitleChanged();
    Q_EMIT errorMessageChanged();
    Q_EMIT errorOcurred();
}

auto PackCompiler::setState(State state) -> void
{
    if(state == m_state)
    {
        return;
    }

    m_state = state;
    Q_EMIT stateChanged();
}

auto PackCompiler::setStatus(const QString &status) -> void
{
    if (m_status == status)
    {
        return;
    }

    m_status = status;
    Q_EMIT statusChanged();
}

auto PackCompiler::setTotalSteps(qint64 steps) -> void
{
    if(steps == m_totalSteps)
    {
        return;
    }

    m_totalSteps = steps;
    Q_EMIT totalStepsChanged();
}

auto PackCompiler::setCurrentStep(qint64 step) -> void
{
    if(step == m_currentStep)
    {
        return;
    }

    m_currentStep = step;
    Q_EMIT currentStepChanged();
}

auto PackCompiler::incrementCompileStep() -> void
{
    setCurrentStep(m_currentStep + 1);

    setProgress
    (
        static_cast<qreal>(m_currentStep) / m_totalSteps
    );
}