qaudioengine.cpp

Go to the documentation of this file.

// Copyright (C) 2022 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only
#include <qaudioengine_p.h>
#include <qspatialsound_p.h>
#include <qambientsound.h>
#include <qaudioroom_p.h>
#include <qaudiolistener.h>
#include <resonance_audio.h>
#include <qambisonicdecoder_p.h>
#include <qaudiodecoder.h>
#include <qmediadevices.h>
#include <qiodevice.h>
#include <qaudiosink.h>
#include <qdebug.h>
#include <qelapsedtimer.h>
 
#include <QFile>
 
QT_BEGIN_NAMESPACE
 
// We'd like to have short buffer times, so the sound adjusts itself to changes
// quickly, but times below 100ms seem to give stuttering on macOS.
// It might be possible to set this value lower on other OSes.
const int bufferTimeMs = 100;
 
class QAudioOutputStream : public QIODevice
{
    Q_OBJECT
public:
    explicit QAudioOutputStream(QAudioEnginePrivate *d)
        : d(d)
    {
        open(QIODevice::ReadOnly);
    }
    ~QAudioOutputStream();
 
    qint64 readData(char *data, qint64 len) override;
 
    qint64 writeData(const char *, qint64) override;
 
    qint64 size() const override { return 0; }
    qint64 bytesAvailable() const override {
        return std::numeric_limits<qint64>::max();
    }
    bool isSequential() const override {
        return true;
    }
    bool atEnd() const override {
        return false;
    }
    qint64 pos() const override {
        return m_pos;
    }
 
    Q_INVOKABLE void startOutput() {
        QMutexLocker l(&d->mutex);
        Q_ASSERT(!sink);
        QAudioFormat format;
        format.setChannelConfig(d->outputMode == QAudioEngine::Surround ?
                                    d->device.channelConfiguration() : QAudioFormat::ChannelConfigStereo);
        format.setSampleRate(d->sampleRate);
        format.setSampleFormat(QAudioFormat::Int16);
        d->ambisonicDecoder.reset(new QAmbisonicDecoder(QAmbisonicDecoder::HighQuality, format));
        sink.reset(new QAudioSink(d->device, format));
        sink->setBufferSize(d->sampleRate*bufferTimeMs/1000*sizeof(qint16)*format.channelCount());
        sink->start(this);
    }
 
    Q_INVOKABLE void stopOutput() {
        sink->stop();
        sink.reset();
        d->ambisonicDecoder.reset();
    }
 
    Q_INVOKABLE void restartOutput() {
        stopOutput();
        startOutput();
    }
 
    void setPaused(bool paused) {
        if (paused)
            sink->suspend();
        else
            sink->resume();
    }
 
private:
    qint64 m_pos = 0;
    QAudioEnginePrivate *d = nullptr;
    std::unique_ptr<QAudioSink> sink;
};
 
 
QAudioOutputStream::~QAudioOutputStream()
{
}
 
qint64 QAudioOutputStream::writeData(const char *, qint64)
{
    return 0;
}
 
qint64 QAudioOutputStream::readData(char *data, qint64 len)
{
    if (d->paused.loadRelaxed())
        return 0;
 
    d->updateRooms();
 
    int nChannels = d->ambisonicDecoder ? d->ambisonicDecoder->nOutputChannels() : 2;
    if (len < nChannels*int(sizeof(float))*QAudioEnginePrivate::bufferSize)
        return 0;
 
    short *fd = (short *)data;
    qint64 frames = len / nChannels / sizeof(short);
    bool ok = true;
    while (frames >= qint64(QAudioEnginePrivate::bufferSize)) {
        // Fill input buffers
        for (auto *source : std::as_const(d->sources)) {
            auto *sp = QSpatialSoundPrivate::get(source);
            float buf[QAudioEnginePrivate::bufferSize];
            sp->getBuffer(buf, QAudioEnginePrivate::bufferSize, 1);
            d->resonanceAudio->api->SetInterleavedBuffer(sp->sourceId, buf, 1, QAudioEnginePrivate::bufferSize);
        }
        for (auto *source : std::as_const(d->stereoSources)) {
            auto *sp = QAmbientSoundPrivate::get(source);
            float buf[2*QAudioEnginePrivate::bufferSize];
            sp->getBuffer(buf, QAudioEnginePrivate::bufferSize, 2);
            d->resonanceAudio->api->SetInterleavedBuffer(sp->sourceId, buf, 2, QAudioEnginePrivate::bufferSize);
        }
 
        if (d->ambisonicDecoder && d->outputMode == QAudioEngine::Surround) {
            const float *channels[QAmbisonicDecoder::maxAmbisonicChannels];
            const float *reverbBuffers[2];
            int nSamples = d->resonanceAudio->getAmbisonicOutput(channels, reverbBuffers, d->ambisonicDecoder->nInputChannels());
            Q_ASSERT(d->ambisonicDecoder->nOutputChannels() <= 8);
            d->ambisonicDecoder->processBufferWithReverb(channels, reverbBuffers, fd, nSamples);
        } else {
            ok = d->resonanceAudio->api->FillInterleavedOutputBuffer(2, QAudioEnginePrivate::bufferSize, fd);
            if (!ok) {
                qWarning() << "    Reading failed!";
                break;
            }
        }
        fd += nChannels*QAudioEnginePrivate::bufferSize;
        frames -= QAudioEnginePrivate::bufferSize;
    }
    const int bytesProcessed = ((char *)fd - data);
    m_pos += bytesProcessed;
    return bytesProcessed;
}
 
 
QAudioEnginePrivate::QAudioEnginePrivate()
{
    device = QMediaDevices::defaultAudioOutput();
    audioThread.setPriority(QThread::TimeCriticalPriority);
}
 
QAudioEnginePrivate::~QAudioEnginePrivate()
{
    delete resonanceAudio;
}
 
void QAudioEnginePrivate::addSpatialSound(QSpatialSound *sound)
{
    QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(sound);
 
    sd->sourceId = resonanceAudio->api->CreateSoundObjectSource(vraudio::kBinauralHighQuality);
    sources.append(sound);
}
 
void QAudioEnginePrivate::removeSpatialSound(QSpatialSound *sound)
{
    QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(sound);
 
    resonanceAudio->api->DestroySource(sd->sourceId);
    sd->sourceId = vraudio::ResonanceAudioApi::kInvalidSourceId;
    sources.removeOne(sound);
}
 
void QAudioEnginePrivate::addStereoSound(QAmbientSound *sound)
{
    QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(sound);
 
    sd->sourceId = resonanceAudio->api->CreateStereoSource(2);
    stereoSources.append(sound);
}
 
void QAudioEnginePrivate::removeStereoSound(QAmbientSound *sound)
{
    QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(sound);
 
    resonanceAudio->api->DestroySource(sd->sourceId);
    sd->sourceId = vraudio::ResonanceAudioApi::kInvalidSourceId;
    stereoSources.removeOne(sound);
}
 
void QAudioEnginePrivate::addRoom(QAudioRoom *room)
{
    rooms.append(room);
}
 
void QAudioEnginePrivate::removeRoom(QAudioRoom *room)
{
    rooms.removeOne(room);
}
 
void QAudioEnginePrivate::updateRooms()
{
    if (!roomEffectsEnabled)
        return;
 
    bool needUpdate = listenerPositionDirty;
    listenerPositionDirty = false;
 
    bool roomDirty = false;
    for (const auto &room : rooms) {
        auto *rd = QAudioRoomPrivate::get(room);
        if (rd->dirty) {
            roomDirty = true;
            rd->update();
            needUpdate = true;
        }
    }
 
    if (!needUpdate)
        return;
 
    QVector3D listenerPos = listenerPosition();
    float roomVolume = float(qInf());
    QAudioRoom *room = nullptr;
    // Find the smallest room that contains the listener and apply its room effects
    for (auto *r : std::as_const(rooms)) {
        QVector3D dim2 = r->dimensions()/2.;
        float vol = dim2.x()*dim2.y()*dim2.z();
        if (vol > roomVolume)
            continue;
        QVector3D dist = r->position() - listenerPos;
        // transform into room coordinates
        dist = r->rotation().rotatedVector(dist);
        if (qAbs(dist.x()) <= dim2.x() &&
            qAbs(dist.y()) <= dim2.y() &&
            qAbs(dist.z()) <= dim2.z()) {
            room = r;
            roomVolume = vol;
        }
    }
    if (room != currentRoom)
        roomDirty = true;
    const bool previousRoom = currentRoom;
    currentRoom = room;
 
    if (!roomDirty)
        return;
 
    // apply room to engine
    if (!currentRoom) {
        resonanceAudio->api->EnableRoomEffects(false);
        return;
    }
    if (!previousRoom)
        resonanceAudio->api->EnableRoomEffects(true);
 
    QAudioRoomPrivate *rp = QAudioRoomPrivate::get(room);
    resonanceAudio->api->SetReflectionProperties(rp->reflections);
    resonanceAudio->api->SetReverbProperties(rp->reverb);
 
    // update room effects for all sound sources
    for (auto *s : std::as_const(sources)) {
        auto *sp = QSpatialSoundPrivate::get(s);
        sp->updateRoomEffects();
    }
}
 
QVector3D QAudioEnginePrivate::listenerPosition() const
{
    return listener ? listener->position() : QVector3D();
}
 
 
QAudioEngine::QAudioEngine(int sampleRate, QObject *parent)
    : QObject(parent)
    , d(new QAudioEnginePrivate)
{
    d->sampleRate = sampleRate;
    d->resonanceAudio = new vraudio::ResonanceAudio(2, QAudioEnginePrivate::bufferSize, d->sampleRate);
}
 
QAudioEngine::~QAudioEngine()
{
    stop();
    delete d;
}
 
void QAudioEngine::setOutputMode(OutputMode mode)
{
    if (d->outputMode == mode)
        return;
    d->outputMode = mode;
    if (d->resonanceAudio->api)
        d->resonanceAudio->api->SetStereoSpeakerMode(mode != Headphone);
 
    QMetaObject::invokeMethod(d->outputStream.get(), "restartOutput", Qt::BlockingQueuedConnection);
 
    emit outputModeChanged();
}
 
QAudioEngine::OutputMode QAudioEngine::outputMode() const
{
    return d->outputMode;
}
 
int QAudioEngine::sampleRate() const
{
    return d->sampleRate;
}
 
void QAudioEngine::setOutputDevice(const QAudioDevice &device)
{
    if (d->device == device)
        return;
    if (d->resonanceAudio->api) {
        qWarning() << "Changing device on a running engine not implemented";
        return;
    }
    d->device = device;
    emit outputDeviceChanged();
}
 
QAudioDevice QAudioEngine::outputDevice() const
{
    return d->device;
}
 
void QAudioEngine::setMasterVolume(float volume)
{
    if (d->masterVolume == volume)
        return;
    d->masterVolume = volume;
    d->resonanceAudio->api->SetMasterVolume(volume);
    emit masterVolumeChanged();
}
 
float QAudioEngine::masterVolume() const
{
    return d->masterVolume;
}
 
void QAudioEngine::start()
{
    if (d->outputStream)
        // already started
        return;
 
    d->resonanceAudio->api->SetStereoSpeakerMode(d->outputMode != Headphone);
    d->resonanceAudio->api->SetMasterVolume(d->masterVolume);
 
    d->outputStream.reset(new QAudioOutputStream(d));
    d->outputStream->moveToThread(&d->audioThread);
    d->audioThread.start();
 
    QMetaObject::invokeMethod(d->outputStream.get(), "startOutput");
}
 
void QAudioEngine::stop()
{
    QMetaObject::invokeMethod(d->outputStream.get(), "stopOutput", Qt::BlockingQueuedConnection);
    d->outputStream.reset();
    d->audioThread.exit(0);
    d->audioThread.wait();
    delete d->resonanceAudio->api;
    d->resonanceAudio->api = nullptr;
}
 
void QAudioEngine::setPaused(bool paused)
{
    bool old = d->paused.fetchAndStoreRelaxed(paused);
    if (old != paused) {
        if (d->outputStream)
            d->outputStream->setPaused(paused);
        emit pausedChanged();
    }
}
 
bool QAudioEngine::paused() const
{
    return d->paused.loadRelaxed();
}
 
void QAudioEngine::setRoomEffectsEnabled(bool enabled)
{
    if (d->roomEffectsEnabled == enabled)
        return;
    d->roomEffectsEnabled = enabled;
    d->resonanceAudio->roomEffectsEnabled = enabled;
}
 
bool QAudioEngine::roomEffectsEnabled() const
{
    return d->roomEffectsEnabled;
}
 
void QAudioEngine::setDistanceScale(float scale)
{
    // multiply with 100, to get the conversion to meters that resonance audio uses
    scale /= 100.f;
    if (scale <= 0.0f) {
        qWarning() << "QAudioEngine: Invalid distance scale.";
        return;
    }
    if (scale == d->distanceScale)
        return;
    d->distanceScale = scale;
    emit distanceScaleChanged();
}
 
float QAudioEngine::distanceScale() const
{
    return d->distanceScale*100.f;
}
 
 
void QAmbientSoundPrivate::load()
{
    decoder.reset(new QAudioDecoder);
    buffers.clear();
    currentBuffer = 0;
    sourceDeviceFile.reset(nullptr);
    bufPos = 0;
    m_playing = false;
    m_loading = true;
    auto *ep = QAudioEnginePrivate::get(engine);
    QAudioFormat f;
    f.setSampleFormat(QAudioFormat::Float);
    f.setSampleRate(ep->sampleRate);
    f.setChannelConfig(nchannels == 2 ? QAudioFormat::ChannelConfigStereo : QAudioFormat::ChannelConfigMono);
    decoder->setAudioFormat(f);
    if (url.scheme().compare(u"qrc", Qt::CaseInsensitive) == 0) {
        auto qrcFile = std::make_unique<QFile>(u':' + url.path());
        if (!qrcFile->open(QFile::ReadOnly))
            return;
        sourceDeviceFile = std::move(qrcFile);
        decoder->setSourceDevice(sourceDeviceFile.get());
    } else {
        decoder->setSource(url);
    }
    connect(decoder.get(), &QAudioDecoder::bufferReady, this, &QAmbientSoundPrivate::bufferReady);
    connect(decoder.get(), &QAudioDecoder::finished, this, &QAmbientSoundPrivate::finished);
    decoder->start();
}
 
void QAmbientSoundPrivate::getBuffer(float *buf, int nframes, int channels)
{
    Q_ASSERT(channels == nchannels);
    QMutexLocker l(&mutex);
    if (!m_playing || currentBuffer >= buffers.size()) {
        memset(buf, 0, channels * nframes * sizeof(float));
    } else {
        int frames = nframes;
        float *ff = buf;
        while (frames) {
            if (currentBuffer < buffers.size()) {
                const QAudioBuffer &b = buffers.at(currentBuffer);
//            qDebug() << s << b.format().sampleRate() << b.format().channelCount() << b.format().sampleFormat();
                auto *f = b.constData<float>() + bufPos*nchannels;
                int toCopy = qMin(b.frameCount() - bufPos, frames);
                memcpy(ff, f, toCopy*sizeof(float)*nchannels);
                ff += toCopy*nchannels;
                frames -= toCopy;
                bufPos += toCopy;
                Q_ASSERT(bufPos <= b.frameCount());
                if (bufPos == b.frameCount()) {
                    ++currentBuffer;
                    bufPos = 0;
                }
            } else {
                // no more data available
                if (m_loading)
                    qDebug() << "underrun" << frames << "frames when loading" << url;
                memset(ff, 0, frames * channels * sizeof(float));
                ff += frames * channels;
                frames = 0;
            }
            if (!m_loading) {
                if (currentBuffer == buffers.size()) {
                    currentBuffer = 0;
                    ++m_currentLoop;
                }
                if (m_loops > 0 && m_currentLoop >= m_loops) {
                    m_playing = false;
                    m_currentLoop = 0;
                }
            }
        }
        Q_ASSERT(ff - buf == channels*nframes);
    }
}
 
void QAmbientSoundPrivate::bufferReady()
{
    QMutexLocker l(&mutex);
    auto b = decoder->read();
//    qDebug() << "read buffer" << b.format() << b.startTime() << b.duration();
    buffers.append(b);
    if (m_autoPlay)
        m_playing = true;
}
 
void QAmbientSoundPrivate::finished()
{
    m_loading = false;
}
 
QT_END_NAMESPACE
 
#include "moc_qaudioengine.cpp"
#include "qaudioengine.moc"