From f3adc14a6e74ebcc51337c499ebb2ae63b023f89 Mon Sep 17 00:00:00 2001 From: Digital Artifex <7929434+DigitalArtifex@users.noreply.github.com> Date: Tue, 25 Aug 2026 04:35:15 -0400 Subject: [PATCH] Reworked PackCompiler PackCompiler now processes everything in memory to reduce file writes. Will also be able to expand error handling later --- KomplexHubPlugin/packcompiler.cpp | 1333 +++++++++-------------------- KomplexHubPlugin/packcompiler.h | 355 +++++--- 2 files changed, 655 insertions(+), 1033 deletions(-) diff --git a/KomplexHubPlugin/packcompiler.cpp b/KomplexHubPlugin/packcompiler.cpp index d0b1ae8..f9cbe24 100644 --- a/KomplexHubPlugin/packcompiler.cpp +++ b/KomplexHubPlugin/packcompiler.cpp @@ -1,9 +1,8 @@ #include "packcompiler.h" #include "common/exceptions.h" -#include PackCompiler::PackCompiler(QObject *parent) - : QObject(parent) + : QObject{parent} { if(m_variableExpressions.isEmpty()) { @@ -23,8 +22,6 @@ PackCompiler::PackCompiler(QObject *parent) } } -PackCompiler::~PackCompiler() = default; - auto PackCompiler::reset() -> void { setError({}, {}); @@ -36,550 +33,21 @@ auto PackCompiler::reset() -> void setState(Idle); } -auto PackCompiler::process(const QUrl &uri) -> QFuture +auto PackCompiler::build(const QUrl &uri) -> QUrl { - return QtConcurrent::run - ( - [this, uri] () -> QUrl - { - validateUri(uri); - reset(); - setStatus(QStringLiteral("Compiling %1").arg(uri.fileName())); - setState(Compiling); + BuildContext *context = new BuildContext; + QUrl extractedUri = extract(uri); - QFileInfo info(uri.toLocalFile()); + validatePack(extractedUri); + context->commonData = loadCommonData(extractedUri); + processShaders(extractedUri, context); - QUrl buildUri; + delete context; - try - { - buildUri = extract(uri); - prepareShaders(buildUri); - compile(buildUri); - } - catch (const std::filesystem::filesystem_error &e) - { - removeDirectory(buildUri); - setError(QStringLiteral("File Error"), e.what()); - throw e; - } - catch (const std::logic_error &e) - { - removeDirectory(buildUri); - setError(QStringLiteral("Compiler Error"), e.what()); - throw e; - } - catch (const std::exception &e) - { - removeDirectory(buildUri); - setError(QStringLiteral("Generic Error"), e.what()); - throw e; - } - catch (...) - { - removeDirectory(buildUri); - setError(QStringLiteral("Unknown Error"), QString()); - throw std::exception(); - } - - return buildUri; - } - ); + return extractedUri; } -auto PackCompiler::prepareShaders(const QUrl &uri) noexcept(false) -> void -{ - if(!uri.isLocalFile()) - { - throw std::filesystem::filesystem_error - ( - QStringLiteral("URI %1 is not a local file uri").arg - ( - uri.toString() - ).toStdString(), - std::error_code - ( - ENOENT, - std::system_category() - ) - ); - } - - QDir sourceDirectory(uri.toLocalFile()); - QDir shaderDirectory - ( - sourceDirectory.absoluteFilePath - ( - QStringLiteral("shaders") - ) - ); - - QStringList entries = shaderDirectory.entryList - ( - QDir::NoDotAndDotDot | QDir::Files - ); - - setTotalSteps(entries.count() * 2); - setProgress(0); - - QByteArray commonFragmentData = loadCommonFragmentData(uri); - - if(commonFragmentData.length() > 0) - { - setTotalSteps(m_totalSteps -= 1); - incrementCompileStep(); - } - - QByteArray commonVertexData = loadCommonVertexData(uri); - - if(commonVertexData.length() > 0) - { - setTotalSteps(m_totalSteps -= 1); - incrementCompileStep(); - } - - QByteArray globalData = loadGlobalData(uri); - - if(globalData.length() > 0) - { - setTotalSteps(m_totalSteps -= 1); - incrementCompileStep(); - } - - for(const QString &entry : std::as_const(entries)) - { - auto commonFragMatch = m_commonFragmentExpression.match(entry); - auto commonVertMatch = m_commonVertexExpression.match(entry); - - if(commonFragMatch.hasMatch() || commonVertMatch.hasMatch()) - { - continue; - } - - auto fragMatch = m_fragmentExpression.match(entry); - auto vertMatch = m_vertexExpression.match(entry); - - QFile file(shaderDirectory.absoluteFilePath(entry)); - - if(!file.open(QFile::ReadWrite)) - { - throw std::filesystem::filesystem_error - ( - QStringLiteral("Could not open file %1 for preperation").arg - ( - shaderDirectory.absoluteFilePath(entry) - ).toStdString(), - std::error_code - ( - errno, - std::system_category() - ) - ); - } - - QByteArray fileData = file.readAll(); - - if(fileData.length() != file.size()) - { - throw std::filesystem::filesystem_error - ( - QStringLiteral("Could not read file %1 for preperation").arg - ( - shaderDirectory.absoluteFilePath(entry) - ).toStdString(), - std::error_code - ( - EIO, - std::system_category() - ) - ); - } - - file.close(); - - QByteArray commonData; - - if(fragMatch.hasMatch()) - { - commonData = commonFragmentData; - } - else if(vertMatch.hasMatch()) - { - commonData = commonVertexData; - } - else - { - throw std::logic_error - ( - QStringLiteral("File %1 is an unrecognized shader type (frag/vert)").arg - ( - shaderDirectory.absoluteFilePath(entry) - ).toStdString() - ); - } - - QByteArray combinedData = QByteArray - ( - globalData + QByteArray("\n") + - commonData + QByteArray("\n") + - fileData - ); - - for(const QRegularExpression &expression : std::as_const(m_variableExpressions)) - { - auto match = expression.match(combinedData); - - if(match.hasMatch()) - { - QByteArray replacement = QByteArray("ubuf.") + match.captured().toLocal8Bit(); - combinedData.replace(match.captured().toLocal8Bit(), replacement); - } - } - - QByteArray preparedData = QByteArray - ( - m_header.toLocal8Bit() + QByteArray("\n") + - combinedData + QByteArray("\n") + - m_footer.toLocal8Bit() - ); - - if(!file.open(QFile::ReadWrite | QFile::Truncate)) - { - throw std::filesystem::filesystem_error - ( - QStringLiteral("Could not open file %1 for writing preperation").arg - ( - shaderDirectory.absoluteFilePath(entry) - ).toStdString(), - std::error_code - ( - errno, - std::system_category() - ) - ); - } - - if(file.write(preparedData) != preparedData.length()) - { - throw std::filesystem::filesystem_error - ( - QStringLiteral("Could not write file %1 preperation").arg - ( - shaderDirectory.absoluteFilePath(entry) - ).toStdString(), - std::error_code - ( - errno, - std::system_category() - ) - ); - } - - file.close(); - incrementCompileStep(); - } -} - -auto PackCompiler::loadCommonFragmentData(const QUrl &uri) -> QByteArray -{ - validateUri(uri); - - QDir sourceDirectory - ( - QStringLiteral("%1/shaders/").arg - ( - uri.toLocalFile() - ) - ); - - const QStringList entries = sourceDirectory.entryList - ( - QDir::NoDotAndDotDot | QDir::Files - ); - - QByteArray data; - - for(const QString &entry : entries) - { - if (entry.toLower() != QStringLiteral("common.frag")) - { - continue; - } - - QFile file(sourceDirectory.absoluteFilePath(entry)); - - if(!file.open(QFile::ReadWrite)) - { - throw std::filesystem::filesystem_error - ( - QStringLiteral("Could not open common shader data").toStdString(), - std::error_code - ( - errno, - std::system_category() - ) - ); - } - - data = file.readAll(); - - if(data.length() != file.size()) - { - throw std::filesystem::filesystem_error - ( - QStringLiteral("Common file read error").toStdString(), - std::error_code - ( - errno, - std::system_category() - ) - ); - } - - file.close(); - break; - } - - return data; -} - -auto PackCompiler::loadCommonVertexData(const QUrl &uri) -> QByteArray -{ - validateUri(uri); - - QDir sourceDirectory - ( - QStringLiteral("%1/shaders/").arg - ( - uri.toLocalFile() - ) - ); - - const QStringList entries = sourceDirectory.entryList - ( - QDir::NoDotAndDotDot | QDir::Files - ); - - QByteArray data; - - for(const QString &entry : entries) - { - if (entry.toLower() != QStringLiteral("common.vert")) - { - continue; - } - - QFile file(sourceDirectory.absoluteFilePath(entry)); - - if(!file.open(QFile::ReadWrite)) - { - throw std::filesystem::filesystem_error - ( - QStringLiteral("Common vertex file open error").toStdString(), - std::error_code - ( - errno, - std::system_category() - ) - ); - } - - data = file.readAll(); - - if(data.length() != file.size()) - { - throw std::filesystem::filesystem_error - ( - QStringLiteral("Common vertex file read error").toStdString(), - std::error_code - ( - EIO, - std::system_category() - ) - ); - } - - file.close(); - break; - } - - return data; -} - -auto PackCompiler::loadGlobalData(const QUrl &uri) -> QByteArray -{ - validateUri(uri); - - QDir sourceDirectory - ( - QStringLiteral("%1/shaders/").arg - ( - uri.toLocalFile() - ) - ); - - const QStringList entries = sourceDirectory.entryList - ( - QDir::NoDotAndDotDot | QDir::Files - ); - - QByteArray data; - - for(const QString &entry : entries) - { - if (entry.toLower() != QStringLiteral("global.glsl")) - { - continue; - } - - QFile file(sourceDirectory.absoluteFilePath(entry)); - - if(!file.open(QFile::ReadWrite)) - { - throw std::filesystem::filesystem_error - ( - QStringLiteral("Common global file open error").toStdString(), - std::error_code - ( - errno, - std::system_category() - ) - ); - } - - data = file.readAll(); - - if(data.length() != file.size()) - { - throw std::underflow_error - ( - QStringLiteral("Common vertex file read error").toStdString() - ); - } - - file.close(); - break; - } - - return data; -} - -auto PackCompiler::validateDirectory(const QUrl &uri) -> bool -{ - QDir directory(uri.toLocalFile()); - - return - ( - directory.exists(QStringLiteral("shaders")) && - directory.exists(QStringLiteral("pack.json")) - ); -} - -auto PackCompiler::createDirectory(const QUrl &uri) -> void -{ - validateUri(uri); - - QStringList arguments = - { - QStringLiteral("-p"), - uri.toLocalFile() - }; - - QProcess process; - - // QObject::connect - // ( - // &process, - // &QProcess::readyReadStandardOutput, - // this, - // [this, &process]() - // { - // QByteArray processData = process.readAllStandardOutput(); - // setCompilerOutput(m_compilerOutput + processData); - // } - // ); - - // QObject::connect - // ( - // &process, - // &QProcess::readyReadStandardError, - // this, - // [this, &process]() - // { - // QByteArray processData = process.readAllStandardError(); - - // if(!processData.isValidUtf8()) - // { - // qWarning() << QStringLiteral("Process output not valid UTF8 data"); - // return; - // } - - // setCompilerOutput(m_compilerOutput + processData); - // } - // ); - - process.start(QStringLiteral("mkdir"), arguments); - - if(!process.waitForStarted(3000)) - { - throw std::filesystem::filesystem_error - ( - QStringLiteral("Could not start process for create directory").toStdString(), - std::error_code - ( - errno, - std::system_category() - ) - ); - } - - if(!process.waitForFinished()) - { - throw std::filesystem::filesystem_error - ( - QStringLiteral("Process timeout for create directory").toStdString(), - std::error_code - ( - errno, - std::system_category() - ) - ); - } - - if(process.exitCode() != 0) - { - throw std::ios_base::failure - ( - QStringLiteral("Process exited abnormally").toStdString(), - std::error_code - ( - process.exitCode(), - std::system_category() - ) - ); - } -} - -auto PackCompiler::validateUri(const QUrl &uri) noexcept(false) -> void -{ - if(!uri.isValid() || !uri.isLocalFile()) - { - throw std::filesystem::filesystem_error - ( - QStringLiteral("URI %1 is not a local file uri").arg - ( - uri.toString() - ).toStdString(), - std::error_code - ( - ENOENT, - std::system_category() - ) - ); - } -} - -auto PackCompiler::extract(const QUrl &sourceUri) noexcept(false) -> QUrl +auto PackCompiler::extract(const QUrl &sourceUri) noexcept(false) -> const QUrl { validateUri(sourceUri); @@ -608,401 +76,277 @@ auto PackCompiler::extract(const QUrl &sourceUri) noexcept(false) -> QUrl createDirectory(outputUri); } - QProcess process; - - // QObject::connect - // ( - // &process, - // &QProcess::readyReadStandardOutput, - // this, - // [this, &process]() - // { - // QByteArray processData = process.readAllStandardOutput(); - // setCompilerOutput(m_compilerOutput + processData); - // } - // ); - - // QObject::connect - // ( - // &process, - // &QProcess::readyReadStandardError, - // this, - // [this, &process]() - // { - // QByteArray processData = process.readAllStandardError(); - - // if(!processData.isValidUtf8()) - // { - // qWarning() << QStringLiteral("Process output not valid UTF8 data"); - // return; - // } - - // setCompilerOutput(m_compilerOutput + processData); - // } - // ); - - process.start(QStringLiteral("tar"), arguments); - - if(!process.waitForStarted(3000)) - { - throw std::filesystem::filesystem_error - ( - QStringLiteral("Could not start process for extract").toStdString(), - std::error_code - ( - errno, - std::system_category() - ) - ); - } - - if(!process.waitForFinished()) - { - throw std::filesystem::filesystem_error - ( - QStringLiteral("Process timeout for extract").toStdString(), - std::error_code - ( - errno, - std::system_category() - ) - ); - } - - if(process.exitCode() != 0) - { - throw std::ios_base::failure - ( - QStringLiteral("Extract process exited abnormally: %1").arg - ( - process.readAllStandardError() - ).toStdString(), - std::error_code - ( - process.exitCode(), - std::system_category() - ) - ); - } + run(QStringLiteral("tar"), arguments); QFile::remove(sourceUri.toLocalFile()); - return outputUri; + return std::move(outputUri); +} + +auto PackCompiler::validatePack(const QUrl &uri) const noexcept(false) -> void +{ + QDir directory(uri.toLocalFile()); + + if (!directory.exists(QStringLiteral("pack.json"))) + { + throw shader::logic_error("Compiler Error", QStringLiteral("Invalid pack file").toStdString(), ENOEXEC); + } + + if (!directory.exists(QStringLiteral("shaders"))) + { + throw shader::logic_error("Compiler Error", QStringLiteral("Nothing to compile").toStdString(), ENOEXEC); + } +} + +auto PackCompiler::loadCommonData(const QUrl &uri) -> QMap +{ + validateUri(uri); + + QMap commonData; + + QDir packDirectory(uri.toLocalFile()); + QDir shaderDirectory(packDirectory.absoluteFilePath(QStringLiteral("shaders"))); + shaderDirectory.setNameFilters(m_shaderNameFilters); + shaderDirectory.setFilter(QDir::Files | QDir::NoDotAndDotDot); + + const QStringList entries = shaderDirectory.entryList(); + + for(const QString &entry : entries) + { + if(entry.startsWith(QStringLiteral("common"), Qt::CaseInsensitive)) + { + QFileInfo entryInfo(shaderDirectory.absoluteFilePath(entry)); + QByteArray data = commonData.take(entryInfo.suffix().toUtf8()); + data += readFile + ( + QStringLiteral("file://%1").arg + ( + entryInfo.absoluteFilePath() + ) + ); + + commonData.insert(entryInfo.suffix().toUpper().toUtf8(), data); + + QFile::remove(entryInfo.absoluteFilePath()); + } + + else if(entry.toLower() == QStringLiteral("global.glsl")) + { + if(commonData.contains(QByteArray("GLOBAL"))) + { + commonData.remove(QByteArray("GLOBAL")); + } + + QFileInfo entryInfo(shaderDirectory.absoluteFilePath(entry)); + QByteArray data = readFile + ( + QStringLiteral("file://%1").arg + ( + entryInfo.absoluteFilePath() + ) + ); + + commonData.insert(QByteArray("GLOBAL"), data); + + QFile::remove(entryInfo.absoluteFilePath()); + } + } + + return std::move(commonData); +} + +auto PackCompiler::processShaders(const QUrl &uri, BuildContext *context) -> void +{ + validateUri(uri); + + QDir packDirectory(uri.toLocalFile()); + QDir shaderDirectory(packDirectory.absoluteFilePath(QStringLiteral("shaders"))); + shaderDirectory.setNameFilters(m_shaderNameFilters); + shaderDirectory.setFilter(QDir::Files | QDir::NoDotAndDotDot); + + const QStringList entries = shaderDirectory.entryList(); + + for(const QString &entry : entries) + { + QFileInfo entryInfo(shaderDirectory.absoluteFilePath(entry)); + QByteArray shaderData; + QUrl entryUri = QStringLiteral("file://%1").arg + ( + entryInfo.absoluteFilePath() + ); + + shaderData += readFile(entryUri); + + QShader::Stage stage = getStageFromSuffix(entryInfo.suffix()); + + appendCommonData(&shaderData, entryInfo.suffix().toUtf8(), context); + replaceUniformVariables(&shaderData); + appendUniformHeader(&shaderData); + appendUniformFooter(&shaderData); + preprocess(&shaderData); + appendVersion(&shaderData); + + QUrl outputUri = QStringLiteral + ( + "file://%1.qsb" + ).arg(entryInfo.absoluteFilePath()); + + QShader shader = compile(shaderData, outputUri.fileName(), stage); + save(outputUri, shader); + + QFile::remove(entryUri.toLocalFile()); + } +} + +auto PackCompiler::preprocess(QByteArray *data) noexcept(false) -> void +{ + QStringList arguments; + arguments << "-E" // Run only the preprocessor + << "-P" // Omit #line markers + << "-x" << "c++" + << "-"; // Read source from stdin + + QByteArray temp = run(QStringLiteral("clang"), arguments, *data); + data->clear(); + data->swap(temp); +} + +auto PackCompiler::appendVersion(QByteArray *data) noexcept(true) -> void +{ + data->push_front(m_version + QByteArray("\n\n")); +} + +auto PackCompiler::compile(const QByteArray &data, const QString &filename, const QShader::Stage stage) noexcept(false) -> QShader +{ + QList targets; + +#ifdef HAS_VULKAN + targets.append({ QShader::SpirvShader, QShaderVersion(vulkanVersion()) }); +#endif + +#ifdef HAS_OPENGL + targets.append({ QShader::GlslShader, QShaderVersion(openGlVersion()) }); +#endif + + QShaderBaker baker; + baker.setGeneratedShaderVariants({ QShader::StandardShader }); + baker.setGeneratedShaders(targets); + baker.setSourceString(data, stage, filename); + + QShader shader = baker.bake(); + + if(!shader.isValid()) + { + QUrl errorUri + ( + QStringLiteral("file://tmp/_err") + ); + + writeFile(errorUri, data); + + throw shader::logic_error + ( + std::string("Compiler Error"), + baker.errorMessage().toStdString() + ); + } + + return shader; +} + +#ifdef HAS_VULKAN +auto PackCompiler::vulkanVersion() const -> qint32 +{ + uint32_t instanceVersion = VK_API_VERSION_1_0; + auto FN_vkEnumerateInstanceVersion = PFN_vkEnumerateInstanceVersion(vkGetInstanceProcAddr(nullptr, "vkEnumerateInstanceVersion")); + if(FN_vkEnumerateInstanceVersion) + { + FN_vkEnumerateInstanceVersion(&instanceVersion); + } + + uint32_t major = VK_VERSION_MAJOR(instanceVersion); + uint32_t minor = VK_VERSION_MINOR(instanceVersion); + + QString versionString = QStringLiteral("%1%2%3").arg(major).arg(minor).arg(0); + return versionString.toInt(); +} +#endif + +auto PackCompiler::save(const QUrl &uri, const QShader &shader) noexcept(false) -> void +{ + writeFile(uri, shader.serialized()); +} + +auto PackCompiler::appendCommonData(QByteArray *data, const QByteArray &suffix, BuildContext *context) noexcept(true) -> void +{ + if(context->commonData.contains(QByteArray("GLOBAL"))) + { + data->push_front + ( + context->commonData.value(QByteArray("GLOBAL")) + QByteArray("\n") + ); + } + + if(context->commonData.contains(suffix.toUpper())) + { + data->push_front + ( + context->commonData.value(suffix.toUpper()) + QByteArray("\n") + ); + } +} + +auto PackCompiler::appendUniformHeader(QByteArray *data) noexcept(true) -> void +{ + data->push_front(m_header + QByteArray("\n\n")); +} + +auto PackCompiler::appendUniformFooter(QByteArray *data) noexcept(true) -> void +{ + data->push_back(QByteArray("\n\n") + m_footer); +} + +auto PackCompiler::replaceUniformVariables(QByteArray *data) noexcept(true) -> void +{ + for(const QRegularExpression &expression : std::as_const(m_variableExpressions)) + { + auto match = expression.match(*data); + + if(match.hasMatch()) + { + QByteArray replacement = QByteArray("ubuf.") + match.captured().toLocal8Bit(); + data->replace(match.captured().toLocal8Bit(), replacement); + } + } } auto PackCompiler::copyFile(const QUrl &sourceUri, const QUrl &destinationUri) noexcept(false) -> void { if(!QFile::exists(sourceUri.toLocalFile())) { - return; + throw file_exception(QStringLiteral("File does not exist").toStdString(), ENOENT); } if(!QFile::copy(sourceUri.toLocalFile(), destinationUri.toLocalFile())) { - throw FileException(QStringLiteral("Could not copy file")); + throw file_exception(QStringLiteral("Could not copy file").toStdString()); } } -auto PackCompiler::compile(const QUrl &uri) noexcept(false) -> void +auto PackCompiler::createDirectory(const QUrl &uri) -> void { validateUri(uri); - setState(Compiling); - - QDir packDirectory(uri.toLocalFile()); - QDir shaderDirectory - ( - packDirectory.absoluteFilePath - ( - QStringLiteral("shaders") - ) - ); - - shaderDirectory.setNameFilters - ( - { - QStringLiteral("*.frag"), - QStringLiteral("*.vert") - } - ); - - const QStringList shaders = shaderDirectory.entryList - ( - QDir::Files | QDir::NoDotAndDotDot - ); - - for(const QString &shader : shaders) - { - if(shader.startsWith(QStringLiteral("common."), Qt::CaseInsensitive)) - { - continue; - } - - QUrl shaderUri - ( - QStringLiteral("file://%1").arg - ( - shaderDirectory.absoluteFilePath(shader) - ) - ); - - preprocess(shaderUri); - appendVersion(shaderUri); - compileShader(shaderUri); - - incrementCompileStep(); - } -} - -auto PackCompiler::preprocess(const QUrl &uri) noexcept(false) -> void -{ - if(!uri.isLocalFile() || !uri.isValid()) - { - throw FileException(QStringLiteral("Uri needs to be a local file"), 0); - } - - QUrl outputUri - ( - QStringLiteral("file://%1_proc").arg - ( - uri.toLocalFile() - ) - ); - QStringList arguments = { - QStringLiteral("-P"), - uri.toLocalFile(), - outputUri.toLocalFile() - }; - - QProcess process; - - // QObject::connect - // ( - // &process, - // &QProcess::readyReadStandardOutput, - // this, - // [this, &process]() - // { - // QByteArray processData = process.readAllStandardOutput(); - // setCompilerOutput(m_compilerOutput + processData); - // } - // ); - - // QObject::connect - // ( - // &process, - // &QProcess::readyReadStandardError, - // this, - // [this, &process]() - // { - // QByteArray processData = process.readAllStandardError(); - - // if(!processData.isValidUtf8()) - // { - // qWarning() << QStringLiteral("Process output not valid UTF8 data"); - // return; - // } - - // setCompilerOutput(m_compilerOutput + processData); - // } - // ); - - process.start(QStringLiteral("cpp"), arguments); - - if(!process.waitForStarted(3000)) - { - throw std::filesystem::filesystem_error - ( - QStringLiteral("Could not start preprocessor").toStdString(), - std::error_code - ( - errno, - std::system_category() - ) - ); - } - - if(!process.waitForFinished()) - { - throw std::filesystem::filesystem_error - ( - QStringLiteral("Process timeout for preprocessor").toStdString(), - std::error_code - ( - errno, - std::system_category() - ) - ); - } - - if(process.exitCode() != 0) - { - throw std::ios_base::failure - ( - QStringLiteral("Preprocessor exited abnormally").toStdString(), - std::error_code - ( - process.exitCode(), - std::system_category() - ) - ); - } - - QFile::remove(uri.toLocalFile()); - QFile::rename(outputUri.toLocalFile(), uri.toLocalFile()); -} - -auto PackCompiler::appendVersion(const QUrl &uri) noexcept(false) -> void -{ - validateUri(uri); - - QFile file(uri.toLocalFile()); - - if(!file.open(QFile::ReadWrite)) - { - throw std::filesystem::filesystem_error - ( - QStringLiteral("Could not open file for appending").toStdString(), - std::error_code - ( - errno, - std::system_category() - ) - ); - } - - QByteArray data = file.readAll(); - - file.seek(0); - - file.write(m_version); - file.write(QByteArray("\n\n")); - file.write(data); - - file.close(); -} - -auto PackCompiler::compileShader(const QUrl &uri) noexcept(false) -> void -{ - validateUri(uri); - - QString qsb = QStringLiteral("/usr/lib/qt6/bin/qsb"); - - if(!QFile::exists(qsb)) - { - throw FileException(QStringLiteral("QSB missing")); - } - - QUrl outputUri - ( - QStringLiteral("file://%1.qsb").arg - ( - uri.toLocalFile() - ) - ); - - QStringList arguments = - { - QStringLiteral("--glsl"), - QStringLiteral("330es, 330, 440"), - QStringLiteral("-o"), - outputUri.toLocalFile(), + QStringLiteral("-p"), uri.toLocalFile() }; - QProcess process; - QString path = uri.toDisplayString(QUrl::RemoveFilename | QUrl::RemoveScheme).remove(QStringLiteral("//")); - process.setWorkingDirectory(path); - - // QObject::connect - // ( - // &process, - // &QProcess::readyReadStandardOutput, - // this, - // [this, &process]() - // { - // QByteArray processData = process.readAllStandardOutput(); - // setCompilerOutput(m_compilerOutput + processData); - // } - // ); - - // QObject::connect - // ( - // &process, - // &QProcess::readyReadStandardError, - // this, - // [this, &process]() - // { - // QByteArray processData = process.readAllStandardError(); - - // if(!processData.isValidUtf8()) - // { - // qWarning() << QStringLiteral("Process output not valid UTF8 data"); - // return; - // } - - // setCompilerOutput(m_compilerOutput + processData); - // } - // ); - - process.start(qsb, arguments); - - if(!process.waitForStarted(3000)) - { - throw std::filesystem::filesystem_error - ( - QStringLiteral("Could not start preprocessor").toStdString(), - std::error_code - ( - errno, - std::system_category() - ) - ); - } - - if(!process.waitForFinished()) - { - throw std::filesystem::filesystem_error - ( - QStringLiteral("Process timeout for preprocessor").toStdString(), - std::error_code - ( - errno, - std::system_category() - ) - ); - } - - if(process.exitCode() != 0) - { - throw std::ios_base::failure - ( - QStringLiteral("Preprocessor exited abnormally").toStdString(), - std::error_code - ( - process.exitCode(), - std::system_category() - ) - ); - } - - if(!QFile::remove(uri.toLocalFile())) - { - throw std::filesystem::filesystem_error - ( - QStringLiteral("Could not cleanup").toStdString(), - std::error_code - ( - errno, - std::system_category() - ) - ); - } + run(QStringLiteral("mkdir"), arguments); } -auto PackCompiler::removeDirectory(const QUrl &uri) -> bool +auto PackCompiler::removeDirectory(const QUrl &uri) noexcept(false) -> void { QStringList arguments = { @@ -1010,25 +354,180 @@ auto PackCompiler::removeDirectory(const QUrl &uri) -> bool uri.toLocalFile() }; + run(QStringLiteral("rm"), arguments); +} + +auto PackCompiler::getStageFromSuffix(const QString &suffix) const -> QShader::Stage +{ + return m_shaderStages.value(suffix.toLower(), QShader::FragmentStage); +} + +#ifdef HAS_OPENGL +auto PackCompiler::openGlVersion() const -> qint32 +{ + qint32 glVersion = 330; + + QString versionString = QString::fromUtf8(glGetString(GL_SHADING_LANGUAGE_VERSION)); + QStringList parts = versionString.split(QChar(' '), Qt::SkipEmptyParts); + + if(parts.count() >= 1) + { + QString version = parts.at(0); + version.remove(QChar('.')); + + bool okay = true; + qint32 temp = version.toInt(&okay); + + if(okay) + { + glVersion = temp; + } + } + + return glVersion; +} +#endif + +auto PackCompiler::validateUri(const QUrl &uri) const noexcept(false) -> void +{ + if(!uri.isValid() || !uri.isLocalFile()) + { + throw file_exception + ( + QStringLiteral("URI %1 is not a local file uri").arg + ( + uri.toString() + ).toStdString(), + std::error_code + ( + ENOENT, + std::system_category() + ) + ); + } +} + +auto PackCompiler::run(const QString &command, const QStringList &arguments, const QByteArray &data) noexcept(false) -> QByteArray +{ QProcess process; - process.start(QStringLiteral("rm"), arguments); + process.start(command, arguments); if(!process.waitForStarted(3000)) { - return false; + throw process_exception + ( + QStringLiteral("Could not start process for %1").arg(command).toStdString(), + process.errorString().toStdString() + ); + } + + if(data.length() > 0) + { + process.write(data); + process.closeWriteChannel(); } if(!process.waitForFinished()) { - return false; + throw process_exception + ( + QStringLiteral("Process timeout for %1").arg(command).toStdString(), + process.errorString().toStdString() + ); } if(process.exitCode() != 0) { - return false; + throw process_exception + ( + QStringLiteral("Process exited abnormally: %1").arg + ( + process.readAllStandardError() + ).toStdString(), + process.errorString().toStdString(), + std::error_code + ( + process.exitCode(), + std::system_category() + ) + ); } - return true; + return process.readAllStandardOutput(); +} + +auto PackCompiler::readFile(const QUrl &uri) -> QByteArray +{ + validateUri(uri); + QByteArray data; + QFile file(uri.toLocalFile()); + + if(!file.open(QFile::ReadOnly)) + { + throw file_exception + ( + QStringLiteral("Could not file").toStdString(), + std::error_code + ( + errno, + std::system_category() + ) + ); + } + + data = file.readAll(); + + if(data.length() != file.size()) + { + throw file_exception + ( + QStringLiteral("File read error").toStdString(), + std::error_code + ( + EIO, + std::system_category() + ) + ); + } + + file.close(); + return data; +} + +auto PackCompiler::writeFile(const QUrl &uri, const QByteArray &data) noexcept(false) -> void +{ + validateUri(uri); + + QFile file(uri.toLocalFile()); + if(!file.open(QFile::ReadWrite)) + { + throw file_exception + ( + QStringLiteral("Could not file").toStdString(), + std::error_code + ( + errno, + std::system_category() + ) + ); + } + + file.write(data); + + if(data.length() != file.size()) + { + throw file_exception + ( + QStringLiteral("File write error").toStdString(), + std::error_code + ( + EIO, + std::system_category() + ) + ); + } + + file.close(); } auto PackCompiler::setProgress(qreal progress) -> void @@ -1130,4 +629,4 @@ auto PackCompiler::incrementCompileStep() -> void ( static_cast(m_currentStep) / m_totalSteps ); -} \ No newline at end of file +} diff --git a/KomplexHubPlugin/packcompiler.h b/KomplexHubPlugin/packcompiler.h index 4d5dddb..0e37d66 100644 --- a/KomplexHubPlugin/packcompiler.h +++ b/KomplexHubPlugin/packcompiler.h @@ -36,6 +36,8 @@ #include #include #include +#include +#include #include #include #include @@ -43,21 +45,42 @@ #include #include #include "common/komplex_global.h" + +#if __has_include("vulkan/vulkan_core.h") +#define HAS_VULKAN +#include +#endif + +#if __has_include("GL/gl.h") +#define HAS_OPENGL +#include +#endif + +struct KOMPLEX_EXPORT BuildContext +{ + QMap commonData; + QPromise promise; +}; + /** * @brief The PackCompiler class * [1] Extract Pack (GZip) * [2] Verify directory is a pack and has shaders - * [3] Load shader code - * [4] Append Common and Global files, if they exist - * [5] Add uniform and replace variable use - * [6] Run C Preprocessor (cpp -P) to process macros - * [7] Append version macro - * [8] Compile with Qt Shader Baker + * [3] Load Common shader code + * [4] Walk the shader directory + * [5] Load shader code + * [6] Append type specific common code and global code + * [7] Replace uniform variable use + * [8] Add uniform header + * [9] Add uniform footer + * [10] Run preprocessor to process macros + * [11] Append version macro + * [12] Compile with QShaderBaker + * [13] Save */ -class KOMPLEX_EXPORT PackCompiler : public QObject +class PackCompiler : public QObject { Q_OBJECT - QML_ELEMENT public: enum State @@ -70,7 +93,6 @@ public: Q_ENUM(State) explicit PackCompiler(QObject *parent = nullptr); - ~PackCompiler() override; auto compilerOutput() const -> const QString & { return m_compilerOutput; } auto progress() -> qreal { return m_progress; } @@ -78,65 +100,11 @@ public: auto totalSteps() -> qint64 { return m_totalSteps; } auto errorTitle() const -> const QString & { return m_errorTitle; } auto errorMessage() const -> const QString & { return m_errorMessage; } - auto reset() -> void; - - auto process(const QUrl &uri) -> QFuture; - auto status() const -> const QString & { return m_status; } auto state() const -> State { return m_state; } + auto reset() -> void; -protected: - /** - * @brief prepareShaders - * Scans the pack's shader directory and sends each file to - * prepareFile() - * @param uri - * Directory to prepare - */ - auto prepareShaders(const QUrl &uri) noexcept(false) -> void; - - /** - * @brief prepareFile - * Prepares the individual file by appending the header, footer, - * common files and replacing standard variable names - * @param uri - */ - auto prepareFile(const QUrl &uri) noexcept(false) -> void; - - /** - * @brief validateDirectory - * - * @param uri - * @return true - * @return false - */ - auto validateDirectory(const QUrl &uri) -> bool; - - auto createDirectory(const QUrl &uri) -> void; - - auto validateUri(const QUrl &uri) noexcept(false) -> void; - auto extract(const QUrl &sourceUri) noexcept(false) -> QUrl; - auto copyFile(const QUrl &sourceUri, const QUrl &destinationUri) noexcept(false) -> void; - auto compile(const QUrl &uri) noexcept(false) -> void; - auto preprocess(const QUrl &uri) noexcept(false) -> void; - auto appendVersion(const QUrl &uri) noexcept(false) -> void; - auto compileShader(const QUrl &uri) noexcept(false) -> void; - auto loadCommonFragmentData(const QUrl &uri) -> QByteArray; - auto loadCommonVertexData(const QUrl &uri) -> QByteArray; - auto loadGlobalData(const QUrl &uri) -> QByteArray; - - auto setError(const QString &title, const QString &message, const QUrl &uri = {}) -> void; - auto setCompilerOutput(const QString &compilerOutput) -> void; - auto setState(State state) -> void; - auto setProgress(qreal progress) -> void; - auto setTotalSteps(qint64 steps) -> void; - auto setCurrentStep(qint64 step) -> void; - - auto setStatus(const QString &status) -> void; - - auto incrementCompileStep() -> void; - - auto removeDirectory(const QUrl &uri) -> bool; + auto build(const QUrl &uri) -> QUrl; signals: auto compilerOutputChanged() -> void; @@ -147,38 +115,213 @@ signals: auto progressChanged() -> void; auto currentStepChanged() -> void; auto totalStepsChanged() -> void; - auto compileComplete(const QUrl &uri) -> void; - auto statusChanged() -> void; auto packReady(const QString &) -> void; auto errorStepChanged() -> void; auto errorRatingChanged() -> void; private: + //[1] + /** + * @brief extract + * Extracts the packfile and deletes the source + * @param uri of the tar.gz pack + * @return uri of the extracted directory + */ + auto extract(const QUrl &uri) noexcept(false) -> const QUrl; + + //[2] + /** + * @brief validatePack + * Validates that the directory is a valid pack and has shaders to compile + * @param uri of the directory provided by extract + */ + auto validatePack(const QUrl &uri) const noexcept(false) -> void; + + //[3] + /** + * @brief loadCommonData + * Loads the common and global shader code from the shader directory of the uri + * @param uri + */ + auto loadCommonData(const QUrl &uri) -> QMap; + + //[4-5] + /** + * @brief processShaders + * Walks the shader directory, loads the code and calls 6-13 for each + * @param uri + * @param context + */ + auto processShaders(const QUrl &uri, BuildContext *context) -> void; + + //[6] + /** + * @brief appendCommonData + * Appends the type specific common code to the shader data + * @param data + * Pointer to shader data + * @param suffix + * Shader file suffix + * @param context + * Pointer to build context object + */ + auto appendCommonData(QByteArray *data, const QByteArray &suffix, BuildContext *context) noexcept(true) -> void; + + //[7] + /** + * @brief replaceUniformVariables + * Replaces standard variables such as iFrame with their uniform buffer name + * @param data + * Pointer to the shader data + */ + auto replaceUniformVariables(QByteArray *data) noexcept(true) -> void; + + //[8] + /** + * @brief appendUniformHeader + * Adds header and uniform buffer to the shader data + * @param data + * Pointer to the shader data + */ + auto appendUniformHeader(QByteArray *data) noexcept(true) -> void; + + //[9] + /** + * @brief appendUniformFooter + * Adds the uniform footer to the shader data + * @param data + * Pointer to the shader data + */ + auto appendUniformFooter(QByteArray *data) noexcept(true) -> void; + + //[10] + /** + * @brief preprocess + * I have had issues with defines in code so passing it to a c/c++ + * preprocessor is required. I am now using clang for this as clang + * is more modern and can use stdin + * @param data + */ + auto preprocess(QByteArray *data) noexcept(false) -> void; + + //[11] + /** + * @brief appendVersion + * The version header confuses the preprocessor, so add it at the end + * @param data + */ + auto appendVersion(QByteArray *data) noexcept(true) -> void; + + //[12] + /** + * @brief compile + * Compiles the shader data in memory and returns the shader object. + * @param data + * Shader data + * @param filename + * Filename of the shader + * @param stage + * Type of shader, based on suffix of original file + * @return + * Compiled shader object + */ + auto compile(const QByteArray &data, const QString &filename, const QShader::Stage stage) noexcept(false) -> QShader; + + //[13] + auto save(const QUrl &uri, const QShader &shader) noexcept(false) -> void; + + //helpers + auto validateUri(const QUrl &uri) const noexcept(false) -> void; + auto run(const QString &command, const QStringList &arguments, const QByteArray &data = {}) noexcept(false) -> QByteArray; + auto readFile(const QUrl &uri) noexcept(false) -> QByteArray; + auto writeFile(const QUrl &uri, const QByteArray &data) noexcept(false) -> void; + auto copyFile(const QUrl &sourceUri, const QUrl &destinationUri) noexcept(false) -> void; + auto createDirectory(const QUrl &uri) noexcept(false) -> void; + auto removeDirectory(const QUrl &uri) noexcept(false) -> void; + auto getStageFromSuffix(const QString &suffix) const -> QShader::Stage; + + auto setError(const QString &title, const QString &message, const QUrl &uri = {}) -> void; + auto setCompilerOutput(const QString &compilerOutput) -> void; + auto setState(State state) -> void; + auto setProgress(qreal progress) -> void; + auto setTotalSteps(qint64 steps) -> void; + auto setCurrentStep(qint64 step) -> void; + auto setStatus(const QString &status) -> void; + auto incrementCompileStep() -> void; + +#ifdef HAS_VULKAN + auto vulkanVersion() const -> qint32; +#endif + +#ifdef HAS_OPENGL + auto openGlVersion() const -> qint32; +#endif + + //members static inline const QStringList m_updateVariables + { + QStringLiteral("iTime"), + QStringLiteral("iTimeDelta"), + QStringLiteral("iFrameRate"), + QStringLiteral("iSampleRate"), + QStringLiteral("iFrame"), + QStringLiteral("iDate"), + QStringLiteral("iMouse"), + QStringLiteral("iResolution"), + QStringLiteral("iColorTheme"), + QStringLiteral("iChannelTime"), + QStringLiteral("iChannelResolution") + }; + + static inline const QStringList m_shaderNameFilters + { + QStringLiteral("*.frag"), + QStringLiteral("*.vert"), + QStringLiteral("*.tese"), + QStringLiteral("*.tesc"), + QStringLiteral("*.geom"), + QStringLiteral("*.comp"), + QStringLiteral("*.glsl") + }; + + static inline const QMap m_shaderStages + { { - QStringLiteral("iTime"), - QStringLiteral("iTimeDelta"), - QStringLiteral("iFrameRate"), - QStringLiteral("iSampleRate"), - QStringLiteral("iFrame"), - QStringLiteral("iDate"), - QStringLiteral("iMouse"), - QStringLiteral("iResolution"), - QStringLiteral("iColorTheme"), - QStringLiteral("iChannelTime"), - QStringLiteral("iChannelResolution") - }; + QStringLiteral("frag"), + QShader::FragmentStage + }, + { + QStringLiteral("vert"), + QShader::VertexStage + }, + { + QStringLiteral("tese"), + QShader::TessellationEvaluationStage + }, + { + QStringLiteral("tesc"), + QShader::TessellationControlStage + }, + { + QStringLiteral("geom"), + QShader::GeometryStage + }, + { + QStringLiteral("comp"), + QShader::ComputeStage + } + }; static inline const QByteArray m_version - { - R"(#version 450)" - }; + { + R"(#version 450)" + }; - static inline const QString m_header - { - R"(layout(location = 0) in vec2 qt_TexCoord0; + static inline const QByteArray m_header + { + R"(layout(location = 0) in vec2 qt_TexCoord0; layout(location = 0) out vec4 fragColor; layout(std140, binding = 0) uniform buf { @@ -203,16 +346,20 @@ layout(binding = 3) uniform sampler2D iChannel2; layout(binding = 4) uniform sampler2D iChannel3; vec2 fragCoord = vec2(qt_TexCoord0.x, 1.0 - qt_TexCoord0.y) * ubuf.iResolution.xy;)" - }; + }; - static inline const QString m_footer - { - R"(void main() { + static inline const QByteArray m_footer + { + R"( +void main() { vec4 color = vec4(0.0); mainImage(color, fragCoord); fragColor = color; })" - }; + }; + + QMap m_commonData; + QMap m_shaders; QString m_compilerOutput; QString m_errorMessage; @@ -227,34 +374,10 @@ vec2 fragCoord = vec2(qt_TexCoord0.x, 1.0 - qt_TexCoord0.y) * ubuf.iResolution.x QMutex m_downloadMutex; + QFuture m_buildFuture; + static inline QList m_variableExpressions; - static inline const QRegularExpression m_commonFragmentExpression = QRegularExpression - ( - QString("^common\\.frag$"), - QRegularExpression::CaseInsensitiveOption - ); - - static inline const QRegularExpression m_commonVertexExpression = QRegularExpression - ( - QString("^common\\.vert$"), - QRegularExpression::CaseInsensitiveOption - ); - - static inline const QRegularExpression m_fragmentExpression = QRegularExpression - ( - QString("^.{1,}\\.frag$"), - QRegularExpression::CaseInsensitiveOption | - QRegularExpression::DotMatchesEverythingOption - ); - - static inline const QRegularExpression m_vertexExpression = QRegularExpression - ( - QString("^.{1,}\\.vert$"), - QRegularExpression::CaseInsensitiveOption | - QRegularExpression::DotMatchesEverythingOption - ); - Q_PROPERTY(QString compilerOutput READ compilerOutput WRITE setCompilerOutput NOTIFY compilerOutputChanged FINAL) Q_PROPERTY(QString errorTitle READ errorTitle NOTIFY errorTitleChanged FINAL) Q_PROPERTY(QString errorMessage READ errorMessage NOTIFY errorMessageChanged FINAL)