qtaptestlogger.cpp

Go to the documentation of this file.

// Copyright (C) 2022 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include "qtaptestlogger_p.h"
 
#include "qtestlog_p.h"
#include "qtestresult_p.h"
#include "qtestassert.h"
 
#if QT_CONFIG(regularexpression)
#  include <QtCore/qregularexpression.h>
#endif
 
QT_BEGIN_NAMESPACE
 
using namespace Qt::StringLiterals;
 
QTapTestLogger::QTapTestLogger(const char *filename)
    : QAbstractTestLogger(filename)
{
}
 
QTapTestLogger::~QTapTestLogger() = default;
 
void QTapTestLogger::startLogging()
{
    QAbstractTestLogger::startLogging();
 
    QTestCharBuffer preamble;
    QTest::qt_asprintf(&preamble, "TAP version 13\n"
        // By convention, test suite names are output as diagnostics lines
        // This is a pretty poor convention, as consumers will then treat
        // actual diagnostics, e.g. qDebug, as test suite names o_O
        "# %s\n", QTestResult::currentTestObjectName());
    outputString(preamble.data());
}
 
void QTapTestLogger::stopLogging()
{
    const int total = QTestLog::totalCount();
 
    QTestCharBuffer testPlanAndStats;
    QTest::qt_asprintf(&testPlanAndStats,
        "1..%d\n" // The plan (last non-diagnostic line)
        "# tests %d\n"
        "# pass %d\n"
        "# fail %d\n",
    total, total, QTestLog::passCount(), QTestLog::failCount());
    outputString(testPlanAndStats.data());
 
    QAbstractTestLogger::stopLogging();
}
 
void QTapTestLogger::enterTestFunction(const char *function)
{
    m_firstExpectedFail.clear();
    Q_ASSERT(!m_gatherMessages);
    Q_ASSERT(m_comments.isEmpty());
    Q_ASSERT(m_messages.isEmpty());
    m_gatherMessages = function != nullptr;
}
 
void QTapTestLogger::enterTestData(QTestData *data)
{
    m_firstExpectedFail.clear();
    if (!m_messages.isEmpty() || !m_comments.isEmpty())
        flushMessages();
    m_gatherMessages = data != nullptr;
}
 
using namespace QTestPrivate;
 
void QTapTestLogger::outputTestLine(bool ok, int testNumber, const QTestCharBuffer &directive)
{
    QTestCharBuffer testIdentifier;
    QTestPrivate::generateTestIdentifier(&testIdentifier, TestFunction | TestDataTag);
 
    QTestCharBuffer testLine;
    QTest::qt_asprintf(&testLine, "%s %d - %s%s\n", ok ? "ok" : "not ok",
                       testNumber, testIdentifier.data(), directive.constData());
 
    outputString(testLine.data());
}
 
// The indent needs to be two spaces for maximum compatibility.
// This matches the width of the "- " prefix on a list item's first line.
#define YAML_INDENT "  "
 
void QTapTestLogger::outputBuffer(const QTestCharBuffer &buffer)
{
    auto isComment = [&buffer]() {
        return buffer.constData()[strlen(YAML_INDENT)] == '#';
    };
    if (!m_gatherMessages)
        outputString(buffer.constData());
    else
        QTestPrivate::appendCharBuffer(isComment() ? &m_comments : &m_messages, buffer);
}
 
void QTapTestLogger::beginYamlish()
{
    outputString(YAML_INDENT "---\n");
}
 
void QTapTestLogger::endYamlish()
{
    // Flush any accumulated messages:
    if (!m_messages.isEmpty()) {
        outputString(YAML_INDENT "extensions:\n");
        outputString(YAML_INDENT YAML_INDENT "messages:\n");
        outputString(m_messages.constData());
        m_messages.clear();
    }
    outputString(YAML_INDENT "...\n");
}
 
void QTapTestLogger::flushComments()
{
    if (!m_comments.isEmpty()) {
        outputString(m_comments.constData());
        m_comments.clear();
    }
}
 
void QTapTestLogger::flushMessages()
{
    /* A _data() function's messages show up here. */
    QTestCharBuffer dataLine;
    QTest::qt_asprintf(&dataLine, "ok %d - %s() # Data prepared\n",
                       QTestLog::totalCount(), QTestResult::currentTestFunction());
    outputString(dataLine.constData());
    flushComments();
    if (!m_messages.isEmpty()) {
        beginYamlish();
        endYamlish();
    }
}
 
void QTapTestLogger::addIncident(IncidentTypes type, const char *description,
                                 const char *file, int line)
{
    const bool isExpectedFail = type == XFail || type == BlacklistedXFail;
    const bool ok = (m_firstExpectedFail.isEmpty()
                     && (type == Pass || type == BlacklistedPass || type == Skip
                         || type == XPass || type == BlacklistedXPass));
 
    const char *const incident = [type](const char *priorXFail) {
        switch (type) {
        // We treat expected or blacklisted failures/passes as TODO-failures/passes,
        // which should be treated as soft issues by consumers. Not all do though :/
        case BlacklistedPass:
            if (priorXFail[0] != '\0')
                return priorXFail;
            Q_FALLTHROUGH();
        case XFail: case BlacklistedXFail:
        case XPass: case BlacklistedXPass:
        case BlacklistedFail:
            return "TODO";
        case Skip:
            return "SKIP";
        case Pass:
            if (priorXFail[0] != '\0')
                return priorXFail;
            Q_FALLTHROUGH();
        case Fail:
            break;
        }
        return static_cast<const char *>(nullptr);
    }(m_firstExpectedFail.constData());
 
    QTestCharBuffer directive;
    if (incident) {
        QTest::qt_asprintf(&directive, "%s%s%s%s",
                           isExpectedFail ? "" : " # ", incident,
                           description && description[0] ? " " : "", description);
    }
 
    if (!isExpectedFail) {
        m_gatherMessages = false;
        outputTestLine(ok, QTestLog::totalCount(), directive);
    } else if (m_gatherMessages && m_firstExpectedFail.isEmpty()) {
        QTestPrivate::appendCharBuffer(&m_firstExpectedFail, directive);
    }
    flushComments();
 
    if (!ok || !m_messages.isEmpty()) {
        // All failures need a diagnostics section to not confuse consumers.
        // We also need a diagnostics section when we have messages to report.
        if (isExpectedFail) {
            QTestCharBuffer message;
            if (m_gatherMessages) {
                QTest::qt_asprintf(&message, YAML_INDENT YAML_INDENT "- severity: xfail\n"
                                   YAML_INDENT YAML_INDENT YAML_INDENT "message:%s\n",
                                   directive.constData() + 4);
            } else {
                QTest::qt_asprintf(&message, YAML_INDENT "# xfail:%s\n", directive.constData() + 4);
            }
            outputBuffer(message);
        } else {
            beginYamlish();
        }
 
        if (!isExpectedFail || m_gatherMessages) {
            const char *indent = isExpectedFail ? YAML_INDENT YAML_INDENT YAML_INDENT : YAML_INDENT;
            if (!ok) {
#if QT_CONFIG(regularexpression)
                enum class OperationType {
                    Unknown,
                    Compare,    /* Plain old QCOMPARE */
                    Verify,     /* QVERIFY */
                    CompareOp,  /* QCOMPARE_OP */
                };
 
                // This is fragile, but unfortunately testlib doesn't plumb
                // the expected and actual values to the loggers (yet).
                static const QRegularExpression verifyRegex(
                            u"^'(?<actualexpression>.*)' returned "
                            "(?<actual>\\w+)\\. \\((?<message>.*)\\)$"_s);
 
                static const QRegularExpression compareRegex(
                            u"^(?<message>.*)\n"
                            "\\s*Actual\\s+\\((?<actualexpression>.*)\\)\\s*: (?<actual>.*)\n"
                            "\\s*Expected\\s+\\((?<expectedexpresssion>.*)\\)\\s*: "
                            "(?<expected>.*)$"_s);
 
                static const QRegularExpression compareOpRegex(
                            u"^(?<message>.*)\n"
                            "\\s*Computed\\s+\\((?<actualexpression>.*)\\)\\s*: (?<actual>.*)\n"
                            "\\s*Baseline\\s+\\((?<expectedexpresssion>.*)\\)\\s*: "
                            "(?<expected>.*)$"_s);
 
                const QString descriptionString = QString::fromUtf8(description);
                QRegularExpressionMatch match = verifyRegex.match(descriptionString);
 
                OperationType opType = OperationType::Unknown;
                if (match.hasMatch())
                    opType = OperationType::Verify;
 
                if (opType == OperationType::Unknown) {
                    match = compareRegex.match(descriptionString);
                    if (match.hasMatch())
                        opType = OperationType::Compare;
                }
 
                if (opType == OperationType::Unknown) {
                    match = compareOpRegex.match(descriptionString);
                    if (match.hasMatch())
                        opType = OperationType::CompareOp;
                }
 
                if (opType != OperationType::Unknown) {
                    QString message = match.captured(u"message");
                    QLatin1StringView comparisonType;
                    QString expected;
                    QString actual;
                    const auto parenthesize = [&match](QLatin1StringView key) -> QString {
                        return " ("_L1 % match.captured(key) % u')';
                    };
                    const QString actualExpression = parenthesize("actualexpression"_L1);
 
                    if (opType == OperationType::Verify) {
                        comparisonType = "QVERIFY"_L1;
                        actual = match.captured(u"actual").toLower() % actualExpression;
                        expected = (actual.startsWith("true "_L1) ? "false"_L1 : "true"_L1)
                                % actualExpression;
                        if (message.isEmpty())
                            message = u"Verification failed"_s;
                    } else if (opType == OperationType::Compare) {
                        comparisonType = "QCOMPARE"_L1;
                        expected = match.captured(u"expected")
                            % parenthesize("expectedexpresssion"_L1);
                        actual = match.captured(u"actual") % actualExpression;
                    } else {
                        struct ComparisonInfo {
                            const char *comparisonType;
                            const char *comparisonStringOp;
                        };
                        // get a proper comparison type based on the error message
                        const auto info = [](const QString &err) -> ComparisonInfo {
                            if (err.contains("different"_L1))
                                return { "QCOMPARE_NE", "!= " };
                            else if (err.contains("less than or equal to"_L1))
                                return { "QCOMPARE_LE", "<= " };
                            else if (err.contains("greater than or equal to"_L1))
                                return { "QCOMPARE_GE", ">= " };
                            else if (err.contains("less than"_L1))
                                return { "QCOMPARE_LT", "< " };
                            else if (err.contains("greater than"_L1))
                                return { "QCOMPARE_GT", "> " };
                            else if (err.contains("to be equal to"_L1))
                                return { "QCOMPARE_EQ", "== " };
                            else
                                return { "Unknown", "" };
                        }(message);
                        comparisonType = QLatin1StringView(info.comparisonType);
                        expected = QLatin1StringView(info.comparisonStringOp)
                                    % match.captured(u"expected")
                                    % parenthesize("expectedexpresssion"_L1);
                        actual = match.captured(u"actual") % actualExpression;
                    }
 
                    QTestCharBuffer diagnosticsYamlish;
                    QTest::qt_asprintf(&diagnosticsYamlish,
                                       "%stype: %s\n"
                                       "%smessage: %s\n"
                                       // Some consumers understand 'wanted/found', others need
                                       // 'expected/actual', so be compatible with both.
                                       "%swanted: %s\n"
                                       "%sfound: %s\n"
                                       "%sexpected: %s\n"
                                       "%sactual: %s\n",
                                       indent, comparisonType.latin1(),
                                       indent, qPrintable(message),
                                       indent, qPrintable(expected), indent, qPrintable(actual),
                                       indent, qPrintable(expected), indent, qPrintable(actual)
                    );
 
                    outputBuffer(diagnosticsYamlish);
                } else
#endif
                if (description && !incident) {
                    QTestCharBuffer unparsableDescription;
                    QTest::qt_asprintf(&unparsableDescription, YAML_INDENT "# %s\n", description);
                    outputBuffer(unparsableDescription);
                }
            }
 
            if (file) {
                QTestCharBuffer location;
                QTest::qt_asprintf(&location,
                                   // The generic 'at' key is understood by most consumers.
                                   "%sat: %s::%s() (%s:%d)\n"
 
                                   // The file and line keys are for consumers that are able
                                   // to read more granular location info.
                                   "%sfile: %s\n"
                                   "%sline: %d\n",
 
                                   indent, QTestResult::currentTestObjectName(),
                                   QTestResult::currentTestFunction(),
                                   file, line, indent, file, indent, line
                    );
                outputBuffer(location);
            }
        }
 
        if (!isExpectedFail)
            endYamlish();
    }
}
 
void QTapTestLogger::addMessage(MessageTypes type, const QString &message,
                                const char *file, int line)
{
    Q_UNUSED(file);
    Q_UNUSED(line);
    const char *const flavor = [type]() {
        switch (type) {
        case QDebug: return "debug";
        case QInfo: return "info";
        case QWarning: return "warning";
        case QCritical: return "critical";
        case QFatal: return "fatal";
        // Handle internal messages as comments
        case Info: return "# inform";
        case Warn: return "# warn";
        }
        return "unrecognised message";
    }();
 
    QTestCharBuffer diagnostic;
    if (!m_gatherMessages) {
        QTest::qt_asprintf(&diagnostic, "%s%s: %s\n",
                           flavor[0] == '#' ? "" : "# ",
                           flavor, qPrintable(message));
        outputString(diagnostic.constData());
    } else if (flavor[0] == '#') {
        QTest::qt_asprintf(&diagnostic, YAML_INDENT "%s: %s\n",
                           flavor, qPrintable(message));
        QTestPrivate::appendCharBuffer(&m_comments, diagnostic);
    } else {
        // These shall appear in a messages: sub-block of the extensions: block,
        // so triple-indent.
        QTest::qt_asprintf(&diagnostic, YAML_INDENT YAML_INDENT "- severity: %s\n"
                           YAML_INDENT YAML_INDENT YAML_INDENT "message: %s\n",
                           flavor, qPrintable(message));
        QTestPrivate::appendCharBuffer(&m_messages, diagnostic);
    }
}
 
QT_END_NAMESPACE