qcborstreamreader.cpp

Go to the documentation of this file.

// Copyright (C) 2020 Intel Corporation.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include "qcborstreamreader.h"
 
#define CBOR_NO_ENCODER_API
#include <private/qcborcommon_p.h>
 
#include <private/qbytearray_p.h>
#include <private/qnumeric_p.h>
#include <private/qstringconverter_p.h>
#include <qiodevice.h>
#include <qdebug.h>
#include <qstack.h>
#include <qvarlengtharray.h>
 
QT_BEGIN_NAMESPACE
 
static bool qt_cbor_decoder_can_read(void *token, size_t len);
static void qt_cbor_decoder_advance(void *token, size_t len);
static void *qt_cbor_decoder_read(void *token, void *userptr, size_t offset, size_t len);
static CborError qt_cbor_decoder_transfer_string(void *token, const void **userptr, size_t offset, size_t len);
 
#define CBOR_PARSER_READER_CONTROL              1
#define CBOR_PARSER_CAN_READ_BYTES_FUNCTION     qt_cbor_decoder_can_read
#define CBOR_PARSER_ADVANCE_BYTES_FUNCTION      qt_cbor_decoder_advance
#define CBOR_PARSER_TRANSFER_STRING_FUNCTION    qt_cbor_decoder_transfer_string
#define CBOR_PARSER_READ_BYTES_FUNCTION         qt_cbor_decoder_read
 
QT_WARNING_PUSH
QT_WARNING_DISABLE_MSVC(4334) // '<<': result of 32-bit shift implicitly converted to 64 bits (was 64-bit shift intended?)
 
#include <cborparser.c>
 
QT_WARNING_POP
 
static CborError _cbor_value_dup_string(const CborValue *, void **, size_t *, CborValue *)
{
    Q_UNREACHABLE_RETURN(CborErrorInternalError);
}
[[maybe_unused]] static CborError cbor_value_get_half_float_as_float(const CborValue *, float *)
{
    Q_UNREACHABLE_RETURN(CborErrorInternalError);
}
 
// confirm our constants match TinyCBOR's
static_assert(int(QCborStreamReader::UnsignedInteger) == CborIntegerType);
static_assert(int(QCborStreamReader::ByteString) == CborByteStringType);
static_assert(int(QCborStreamReader::TextString) == CborTextStringType);
static_assert(int(QCborStreamReader::Array) == CborArrayType);
static_assert(int(QCborStreamReader::Map) == CborMapType);
static_assert(int(QCborStreamReader::Tag) == CborTagType);
static_assert(int(QCborStreamReader::SimpleType) == CborSimpleType);
static_assert(int(QCborStreamReader::HalfFloat) == CborHalfFloatType);
static_assert(int(QCborStreamReader::Float) == CborFloatType);
static_assert(int(QCborStreamReader::Double) == CborDoubleType);
static_assert(int(QCborStreamReader::Invalid) == CborInvalidType);
 
class QCborStreamReaderPrivate
{
public:
    enum {
        // 9 bytes is the maximum size for any integer, floating point or
        // length in CBOR.
        MaxCborIndividualSize = 9,
        IdealIoBufferSize = 256
    };
 
    QIODevice *device;
    QByteArray buffer;
    QStack<CborValue> containerStack;
 
    CborParser parser;
    CborValue currentElement;
    QCborError lastError = {};
 
    QByteArray::size_type bufferStart = 0;
    bool corrupt = false;
 
    QCborStreamReaderPrivate(const QByteArray &data)
        : device(nullptr), buffer(data)
    {
        initDecoder();
    }
 
    QCborStreamReaderPrivate(QIODevice *device)
    {
        setDevice(device);
    }
 
    ~QCborStreamReaderPrivate()
    {
    }
 
    void setDevice(QIODevice *dev)
    {
        buffer.clear();
        device = dev;
        initDecoder();
    }
 
    void initDecoder()
    {
        containerStack.clear();
        bufferStart = 0;
        if (device) {
            buffer.clear();
            buffer.reserve(IdealIoBufferSize);      // sets the CapacityReserved flag
        }
 
        preread();
        if (CborError err = cbor_parser_init_reader(nullptr, &parser, &currentElement, this))
            handleError(err);
        else
            lastError = { QCborError::NoError };
    }
 
    char *bufferPtr()
    {
        Q_ASSERT(buffer.isDetached());
        return const_cast<char *>(buffer.constData()) + bufferStart;
    }
 
    void preread()
    {
        if (device && buffer.size() - bufferStart < MaxCborIndividualSize) {
            // load more, but only if there's more to be read
            qint64 avail = device->bytesAvailable();
            Q_ASSERT(avail >= buffer.size());
            if (avail == buffer.size())
                return;
 
            if (bufferStart)
                device->skip(bufferStart);  // skip what we've already parsed
 
            if (buffer.size() != IdealIoBufferSize)
                buffer.resize(IdealIoBufferSize);
 
            bufferStart = 0;
            qint64 read = device->peek(bufferPtr(), IdealIoBufferSize);
            if (read < 0)
                buffer.clear();
            else if (read != IdealIoBufferSize)
                buffer.truncate(read);
        }
    }
 
    void handleError(CborError err) noexcept
    {
        Q_ASSERT(err);
 
        // is the error fatal?
        if (err != CborErrorUnexpectedEOF)
            corrupt = true;
 
        lastError = QCborError { QCborError::Code(int(err)) };
    }
 
    struct ReadStringChunk {
        union {
            char *ptr;
            QByteArray *array;
            QString *string;
        };
        enum { ByteArray = -1, String = -3 };
        qsizetype maxlen_or_type;
 
        ReadStringChunk(char *ptr, qsizetype maxlen) : ptr(ptr), maxlen_or_type(maxlen) {}
        ReadStringChunk(QByteArray *array) : array(array), maxlen_or_type(ByteArray) {}
        ReadStringChunk(QString *str) : string(str), maxlen_or_type(String) {}
        bool isString() const { return maxlen_or_type == String; }
        bool isByteArray() const { return maxlen_or_type == ByteArray; }
        bool isPlainPointer() const { return maxlen_or_type >= 0; }
    };
 
    static QCborStreamReader::StringResultCode appendStringChunk(QCborStreamReader &reader, QByteArray *data);
    QCborStreamReader::StringResult<qsizetype> readStringChunk(ReadStringChunk params);
    qsizetype readStringChunk_byte(ReadStringChunk params, qsizetype len);
    qsizetype readStringChunk_unicode(ReadStringChunk params, qsizetype utf8len);
    bool ensureStringIteration();
};
 
void qt_cbor_stream_set_error(QCborStreamReaderPrivate *d, QCborError error)
{
    d->handleError(CborError(error.c));
}
 
static inline bool qt_cbor_decoder_can_read(void *token, size_t len)
{
    Q_ASSERT(len <= QCborStreamReaderPrivate::MaxCborIndividualSize);
    auto self = static_cast<QCborStreamReaderPrivate *>(token);
 
    qint64 avail = self->buffer.size() - self->bufferStart;
    return len <= quint64(avail);
}
 
static void qt_cbor_decoder_advance(void *token, size_t len)
{
    Q_ASSERT(len <= QCborStreamReaderPrivate::MaxCborIndividualSize);
    auto self = static_cast<QCborStreamReaderPrivate *>(token);
    Q_ASSERT(len <= size_t(self->buffer.size() - self->bufferStart));
 
    self->bufferStart += int(len);
    self->preread();
}
 
static void *qt_cbor_decoder_read(void *token, void *userptr, size_t offset, size_t len)
{
    Q_ASSERT(len == 1 || len == 2 || len == 4 || len == 8);
    Q_ASSERT(offset == 0 || offset == 1);
    auto self = static_cast<const QCborStreamReaderPrivate *>(token);
 
    // we must have pre-read the data
    Q_ASSERT(len + offset <= size_t(self->buffer.size() - self->bufferStart));
    return memcpy(userptr, self->buffer.constData() + self->bufferStart + offset, len);
}
 
static CborError qt_cbor_decoder_transfer_string(void *token, const void **userptr, size_t offset, size_t len)
{
    auto self = static_cast<QCborStreamReaderPrivate *>(token);
    Q_ASSERT(offset <= size_t(self->buffer.size()));
    static_assert(sizeof(size_t) >= sizeof(QByteArray::size_type));
    static_assert(sizeof(size_t) == sizeof(qsizetype));
 
    // check that we will have enough data from the QIODevice before we advance
    // (otherwise, we'd lose the length information)
    qsizetype total;
    if (len > size_t(std::numeric_limits<QByteArray::size_type>::max())
            || qAddOverflow<qsizetype>(offset, len, &total))
        return CborErrorDataTooLarge;
 
    // our string transfer is just saving the offset to the userptr
    *userptr = reinterpret_cast<void *>(offset);
 
    qint64 avail = (self->device ? self->device->bytesAvailable() : self->buffer.size()) -
            self->bufferStart;
    return total > avail ? CborErrorUnexpectedEOF : CborNoError;
}
 
bool QCborStreamReaderPrivate::ensureStringIteration()
{
    if (currentElement.flags & CborIteratorFlag_IteratingStringChunks)
        return true;
 
    CborError err = cbor_value_begin_string_iteration(&currentElement);
    if (!err)
        return true;
    handleError(err);
    return false;
}
 
inline void QCborStreamReader::preparse()
{
    if (lastError() == QCborError::NoError) {
        type_ = cbor_value_get_type(&d->currentElement);
 
        if (type_ == CborInvalidType) {
            // We may have reached the end.
            if (d->device && d->containerStack.isEmpty()) {
                d->buffer.clear();
                if (d->bufferStart)
                    d->device->skip(d->bufferStart);
                d->bufferStart = 0;
            }
        } else {
            d->lastError = {};
            // Undo the type mapping that TinyCBOR does (we have an explicit type
            // for negative integer and we don't have separate types for Boolean,
            // Null and Undefined).
            if (type_ == CborBooleanType || type_ == CborNullType || type_ == CborUndefinedType) {
                type_ = CborSimpleType;
                value64 = quint8(d->buffer.at(d->bufferStart)) - CborSimpleType;
            } else {
                // Using internal TinyCBOR API!
                value64 = _cbor_value_extract_int64_helper(&d->currentElement);
 
                if (cbor_value_is_negative_integer(&d->currentElement))
                    type_ = quint8(QCborStreamReader::NegativeInteger);
            }
        }
    } else {
        type_ = Invalid;
    }
}
 
QCborStreamReader::QCborStreamReader()
    : d(new QCborStreamReaderPrivate({})), type_(Invalid)
{
}
 
QCborStreamReader::QCborStreamReader(const char *data, qsizetype len)
    : QCborStreamReader(QByteArray::fromRawData(data, len))
{
}
 
QCborStreamReader::QCborStreamReader(const quint8 *data, qsizetype len)
    : QCborStreamReader(QByteArray::fromRawData(reinterpret_cast<const char *>(data), len))
{
}
 
QCborStreamReader::QCborStreamReader(const QByteArray &data)
    : d(new QCborStreamReaderPrivate(data))
{
    preparse();
}
 
QCborStreamReader::QCborStreamReader(QIODevice *device)
    : d(new QCborStreamReaderPrivate(device))
{
    preparse();
}
 
QCborStreamReader::~QCborStreamReader()
{
}
 
void QCborStreamReader::setDevice(QIODevice *device)
{
    d->setDevice(device);
    preparse();
}
 
QIODevice *QCborStreamReader::device() const
{
    return d->device;
}
 
void QCborStreamReader::addData(const QByteArray &data)
{
    addData(data.constData(), data.size());
}
 
void QCborStreamReader::addData(const char *data, qsizetype len)
{
    if (!d->device) {
        if (len > 0)
            d->buffer.append(data, len);
        reparse();
    } else {
        qWarning("QCborStreamReader: addData() with device()");
    }
}
 
void QCborStreamReader::reparse()
{
    d->lastError = {};
    d->preread();
    if (CborError err = cbor_value_reparse(&d->currentElement))
        d->handleError(err);
    else
        preparse();
}
 
void QCborStreamReader::clear()
{
    setDevice(nullptr);
}
 
void QCborStreamReader::reset()
{
    if (d->device)
        d->device->reset();
    d->lastError = {};
    d->initDecoder();
    preparse();
}
 
QCborError QCborStreamReader::lastError()
{
    return d->lastError;
}
 
qint64 QCborStreamReader::currentOffset() const
{
    return (d->device ? d->device->pos() : 0) + d->bufferStart;
}
 
int QCborStreamReader::containerDepth() const
{
    return d->containerStack.size();
}
 
QCborStreamReader::Type QCborStreamReader::parentContainerType() const
{
    if (d->containerStack.isEmpty())
        return Invalid;
    return Type(cbor_value_get_type(&std::as_const(d->containerStack).top()));
}
 
bool QCborStreamReader::hasNext() const noexcept
{
    return cbor_value_is_valid(&d->currentElement) &&
            !cbor_value_at_end(&d->currentElement);
}
 
bool QCborStreamReader::next(int maxRecursion)
{
    if (lastError() != QCborError::NoError)
        return false;
 
    if (!hasNext()) {
        d->handleError(CborErrorAdvancePastEOF);
    } else if (maxRecursion < 0) {
        d->handleError(CborErrorNestingTooDeep);
    } else if (isContainer()) {
        // iterate over each element
        enterContainer();
        while (lastError() == QCborError::NoError && hasNext())
            next(maxRecursion - 1);
        if (lastError() == QCborError::NoError)
            leaveContainer();
    } else if (isByteArray()) {
        char c;
        StringResult<qsizetype> r;
        do {
            r = readStringChunk(&c, 1);
        } while (r.status == Ok);
    } else if (isString()) {
        // we need to use actual readString so we get UTF-8 validation
        StringResult<QString> r;
        do {
            r = readString();
        } while (r.status == Ok);
    } else {
        // fixed types
        CborError err = cbor_value_advance_fixed(&d->currentElement);
        if (err)
            d->handleError(err);
    }
 
    preparse();
    return d->lastError == QCborError::NoError;
}
 
bool QCborStreamReader::isLengthKnown() const noexcept
{
    return cbor_value_is_length_known(&d->currentElement);
}
 
quint64 QCborStreamReader::length() const
{
    CborError err;
    switch (type()) {
    case String:
    case ByteArray:
    case Map:
    case Array:
        if (isLengthKnown())
            return value64;
        err = CborErrorUnknownLength;
        break;
 
    default:
        err = CborErrorIllegalType;
        break;
    }
 
    d->handleError(err);
    return quint64(-1);
}
 
bool QCborStreamReader::_enterContainer_helper()
{
    d->containerStack.push(d->currentElement);
    CborError err = cbor_value_enter_container(&d->containerStack.top(), &d->currentElement);
    if (!err) {
        preparse();
        return true;
    }
    d->handleError(err);
    return false;
}
 
bool QCborStreamReader::leaveContainer()
{
    if (d->containerStack.isEmpty()) {
        qWarning("QCborStreamReader::leaveContainer: trying to leave top-level element");
        return false;
    }
    if (d->corrupt)
        return false;
 
    CborValue container = d->containerStack.pop();
    CborError err = cbor_value_leave_container(&container, &d->currentElement);
    d->currentElement = container;
    if (err) {
        d->handleError(err);
        return false;
    }
 
    preparse();
    return true;
}
 
QCborStreamReader::StringResult<QString> QCborStreamReader::_readString_helper()
{
    QCborStreamReader::StringResult<QString> result;
    auto r = d->readStringChunk(&result.data);
    result.status = r.status;
    if (r.status == Error) {
        result.data.clear();
    } else {
        Q_ASSERT(r.data == result.data.size());
        if (r.status == EndOfString && lastError() == QCborError::NoError)
            preparse();
    }
 
    return result;
}
 
QCborStreamReader::StringResult<QByteArray> QCborStreamReader::_readByteArray_helper()
{
    QCborStreamReader::StringResult<QByteArray> result;
    auto r = d->readStringChunk(&result.data);
    result.status = r.status;
    if (r.status == Error) {
        result.data.clear();
    } else {
        Q_ASSERT(r.data == result.data.size());
        if (r.status == EndOfString && lastError() == QCborError::NoError)
            preparse();
    }
 
    return result;
}
 
qsizetype QCborStreamReader::_currentStringChunkSize() const
{
    if (!d->ensureStringIteration())
        return -1;
 
    size_t len;
    CborError err = cbor_value_get_string_chunk_size(&d->currentElement, &len);
    if (err == CborErrorNoMoreStringChunks)
        return 0;           // not a real error
    else if (err)
        d->handleError(err);
    else if (qsizetype(len) < 0)
        d->handleError(CborErrorDataTooLarge);
    else
        return qsizetype(len);
    return -1;
}
 
QCborStreamReader::StringResult<qsizetype>
QCborStreamReader::readStringChunk(char *ptr, qsizetype maxlen)
{
    auto r = d->readStringChunk({ptr, maxlen});
    if (r.status == EndOfString && lastError() == QCborError::NoError)
        preparse();
    return r;
}
 
// used by qcborvalue.cpp
QCborStreamReader::StringResultCode qt_cbor_append_string_chunk(QCborStreamReader &reader, QByteArray *data)
{
    return QCborStreamReaderPrivate::appendStringChunk(reader, data);
}
 
inline QCborStreamReader::StringResultCode
QCborStreamReaderPrivate::appendStringChunk(QCborStreamReader &reader, QByteArray *data)
{
    auto status = reader.d->readStringChunk(data).status;
    if (status == QCborStreamReader::EndOfString && reader.lastError() == QCborError::NoError)
        reader.preparse();
    return status;
}
 
Q_NEVER_INLINE QCborStreamReader::StringResult<qsizetype>
QCborStreamReaderPrivate::readStringChunk(ReadStringChunk params)
{
    CborError err;
    size_t len;
    const void *content = nullptr;
    QCborStreamReader::StringResult<qsizetype> result;
    result.data = 0;
    result.status = QCborStreamReader::Error;
 
    lastError = {};
    if (!ensureStringIteration())
        return result;
 
    // Note: in the current implementation, the call into TinyCBOR below only
    // succeeds if we *already* have all the data in memory. That's obvious for
    // the case of direct memory (no QIODevice), whereas for QIODevices
    // qt_cbor_decoder_transfer_string() enforces that
    // QIODevice::bytesAvailable() be bigger than the amount we're about to
    // read.
#if 1
    // Using internal TinyCBOR API!
    err = _cbor_value_get_string_chunk(&currentElement, &content, &len, &currentElement);
#else
    // the above is effectively the same as:
    if (cbor_value_is_byte_string(&currentElement))
        err = cbor_value_get_byte_string_chunk(&currentElement, reinterpret_cast<const uint8_t **>(&content),
                                               &len, &currentElement);
    else
        err = cbor_value_get_text_string_chunk(&currentElement, reinterpret_cast<const char **>(&content),
                                               &len, &currentElement);
#endif
 
    // Range check: using implementation-defined behavior in converting an
    // unsigned value out of range of the destination signed type (same as
    // "len > size_t(std::numeric_limits<qsizetype>::max())", but generates
    // better code with ICC and MSVC).
    if (!err && qsizetype(len) < 0)
        err = CborErrorDataTooLarge;
 
    if (err) {
        if (err == CborErrorNoMoreStringChunks) {
            preread();
            err = cbor_value_finish_string_iteration(&currentElement);
            result.status = QCborStreamReader::EndOfString;
        }
        if (err)
            handleError(err);
        // caller musts call preparse()
        return result;
    }
 
    qptrdiff offset = qptrdiff(content);
    bufferStart += offset;
    if (device) {
        // This first skip can't fail because we've already read this many bytes.
        device->skip(bufferStart);
    }
 
    if (params.isString()) {
        // readString()
        result.data = readStringChunk_unicode(params, qsizetype(len));
    } else {
        // readByteArray() or readStringChunk()
        result.data = readStringChunk_byte(params, qsizetype(len));
    }
 
    if (result.data < 0)
        return result;      // error
 
    // adjust the buffers after we're done reading the string
    bufferStart += len;
    if (device) {
        qsizetype remainingInBuffer = buffer.size() - bufferStart;
 
        if (remainingInBuffer <= 0) {
            // We've read from the QIODevice more than what was in the buffer.
            buffer.truncate(0);
        } else {
            // There's still data buffered, but we need to move it around.
            char *ptr = buffer.data();
            memmove(ptr, ptr + bufferStart, remainingInBuffer);
            buffer.truncate(remainingInBuffer);
        }
 
        bufferStart = 0;
    }
 
    preread();
    result.status = QCborStreamReader::Ok;
    return result;
}
 
inline qsizetype
QCborStreamReaderPrivate::readStringChunk_byte(ReadStringChunk params, qsizetype len)
{
    qint64 actuallyRead;
    qsizetype toRead = qsizetype(len);
    qsizetype left = 0;     // bytes from the chunk not copied to the user buffer, to discard
    char *ptr = nullptr;
 
    if (params.isPlainPointer()) {
        left = toRead - params.maxlen_or_type;
        if (left < 0)
            left = 0;                           // buffer bigger than string
        else
            toRead = params.maxlen_or_type;     // buffer smaller than string
        ptr = params.ptr;
    } else if (params.isByteArray()) {
        // See note above on having ensured there is enough incoming data.
        auto oldSize = params.array->size();
        auto newSize = oldSize;
        if (qAddOverflow<decltype(newSize)>(oldSize, toRead, &newSize)) {
            handleError(CborErrorDataTooLarge);
            return -1;
        }
        QT_TRY {
            params.array->resize(newSize);
        } QT_CATCH (const std::bad_alloc &) {
            // the distinction between DataTooLarge and OOM is mostly for
            // compatibility with Qt 5; in Qt 6, we could consider everything
            // to be OOM.
            handleError(newSize > MaxByteArraySize ? CborErrorDataTooLarge: CborErrorOutOfMemory);
            return -1;
        }
 
        ptr = const_cast<char *>(params.array->constData()) + oldSize;
    }
 
    if (device) {
        actuallyRead = device->read(ptr, toRead);
 
        if (actuallyRead != toRead)  {
            actuallyRead = -1;
        } else if (left) {
            qint64 skipped = device->skip(left);
            if (skipped != left)
                actuallyRead = -1;
        }
 
        if (actuallyRead < 0) {
            handleError(CborErrorIO);
            return -1;
        }
    } else {
        actuallyRead = toRead;
        memcpy(ptr, buffer.constData() + bufferStart, toRead);
    }
 
    return actuallyRead;
}
 
inline qsizetype
QCborStreamReaderPrivate::readStringChunk_unicode(ReadStringChunk params, qsizetype utf8len)
{
    // See QUtf8::convertToUnicode() a detailed explanation of why this
    // conversion uses the same number of words or less.
    QChar *begin = nullptr;
    if (params.isString()) {
        QT_TRY {
            params.string->resize(utf8len);
        } QT_CATCH (const std::bad_alloc &) {
            if (utf8len > MaxStringSize)
                handleError(CborErrorDataTooLarge);
            else
                handleError(CborErrorOutOfMemory);
            return -1;
        }
 
        begin = const_cast<QChar *>(params.string->constData());
    }
 
    QChar *ptr = begin;
    QStringConverter::State cs(QStringConverter::Flag::Stateless);
    if (device == nullptr) {
        // Easy case: we can decode straight from the buffer we already have
        ptr = QUtf8::convertToUnicode(ptr, { buffer.constData() + bufferStart, utf8len }, &cs);
    } else {
        // read in chunks, to avoid creating large, intermediate buffers
        constexpr qsizetype StringChunkSize = 16384;
        qsizetype chunkSize = qMin(StringChunkSize, utf8len);
        QVarLengthArray<char> chunk(chunkSize);
 
        cs = { QStringConverter::Flag::ConvertInitialBom };
        while (utf8len > 0 && cs.invalidChars == 0) {
            qsizetype toRead = qMin(chunkSize, utf8len);
            qint64 actuallyRead = device->read(chunk.data(), toRead);
            if (actuallyRead == toRead)
                ptr = QUtf8::convertToUnicode(ptr, { chunk.data(), toRead }, &cs);
 
            if (actuallyRead != toRead) {
                handleError(CborErrorIO);
                return -1;
            }
            utf8len -= toRead;
        }
    }
 
    if (cs.invalidChars != 0 || cs.remainingChars != 0) {
        handleError(CborErrorInvalidUtf8TextString);
        return -1;
    }
 
    qsizetype size = ptr - begin;
    if (params.isString())
        params.string->truncate(size);
    return size;
}
 
QT_END_NAMESPACE
 
#include "moc_qcborstreamreader.cpp"