Add support for audio playback via audout

Adds an audio manager to state to manage the creation of audio tracks and an audout service implementation that interfaces with it.
This commit is contained in:
Billy Laws 2020-01-02 20:19:34 +00:00 committed by ◱ PixelyIon
parent 08bbc66b09
commit 93206d5a3c
15 changed files with 513 additions and 1 deletions

4
.gitmodules vendored
View File

@ -4,3 +4,7 @@
[submodule "app/libraries/fmt"]
path = app/libraries/fmt
url = https://github.com/fmtlib/fmt
[submodule "app/libraries/oboe"]
path = app/libraries/oboe
url = https://github.com/google/oboe
branch = 1.3-stable

View File

@ -3,6 +3,7 @@
<component name="VcsDirectoryMappings">
<mapping directory="" vcs="Git" />
<mapping directory="$PROJECT_DIR$/app/libraries/fmt" vcs="Git" />
<mapping directory="$PROJECT_DIR$/app/libraries/oboe" vcs="Git" />
<mapping directory="$PROJECT_DIR$/app/libraries/tinyxml2" vcs="Git" />
</component>
</project>

View File

@ -15,12 +15,16 @@ endif()
set(CMAKE_POLICY_DEFAULT_CMP0048 OLD)
add_subdirectory("libraries/tinyxml2")
add_subdirectory("libraries/fmt")
add_subdirectory("libraries/oboe")
include_directories (libraries/oboe/include)
set(CMAKE_POLICY_DEFAULT_CMP0048 NEW)
include_directories(${source_DIR}/skyline)
add_library(skyline SHARED
${source_DIR}/main.cpp
${source_DIR}/skyline/audio.cpp
${source_DIR}/skyline/audio/track.cpp
${source_DIR}/skyline/common.cpp
${source_DIR}/skyline/nce/guest.S
${source_DIR}/skyline/nce/guest.cpp
@ -47,6 +51,7 @@ add_library(skyline SHARED
${source_DIR}/skyline/services/serviceman.cpp
${source_DIR}/skyline/services/sm/sm.cpp
${source_DIR}/skyline/services/fatal/fatal.cpp
${source_DIR}/skyline/services/audout/audout.cpp
${source_DIR}/skyline/services/set/sys.cpp
${source_DIR}/skyline/services/apm/apm.cpp
${source_DIR}/skyline/services/am/applet.cpp
@ -59,5 +64,5 @@ add_library(skyline SHARED
${source_DIR}/skyline/services/vi/vi_m.cpp
)
target_link_libraries(skyline vulkan android fmt tinyxml2)
target_link_libraries(skyline vulkan android fmt tinyxml2 oboe)
target_compile_options(skyline PRIVATE -Wno-c++17-extensions)

1
app/libraries/oboe Submodule

@ -0,0 +1 @@
Subproject commit 56854b88dd54a8bf7c511800ecf9f991e02cf3de

View File

@ -0,0 +1,64 @@
#include "audio.h"
namespace skyline::audio {
Audio::Audio(const DeviceState &state) : state(state), oboe::AudioStreamCallback() {
oboe::AudioStreamBuilder builder;
builder.setChannelCount(constant::ChannelCount)
->setSampleRate(constant::SampleRate)
->setFormat(constant::PcmFormat)
->setCallback(this)
->openManagedStream(outputStream);
outputStream->requestStart();
}
std::shared_ptr<AudioTrack> Audio::OpenTrack(int channelCount, int sampleRate, const std::function<void()> &releaseCallback) {
std::shared_ptr<AudioTrack> track = std::make_shared<AudioTrack>(channelCount, sampleRate, releaseCallback);
audioTracks.push_back(track);
return track;
}
void Audio::CloseTrack(std::shared_ptr<AudioTrack> &track) {
audioTracks.erase(std::remove(audioTracks.begin(), audioTracks.end(), track), audioTracks.end());
track.reset();
}
oboe::DataCallbackResult Audio::onAudioReady(oboe::AudioStream *audioStream, void *audioData, int32_t numFrames) {
i16 *destBuffer = static_cast<i16 *>(audioData);
int setIndex = 0;
size_t sampleI16Size = static_cast<size_t>(numFrames) * audioStream->getChannelCount();
for (auto &track : audioTracks) {
if (track->playbackState == AudioOutState::Stopped)
continue;
track->bufferLock.lock();
std::queue<i16> &srcBuffer = track->sampleQueue;
size_t amount = std::min(srcBuffer.size(), sampleI16Size);
for (size_t i = 0; i < amount; i++) {
if (setIndex == i) {
destBuffer[i] = srcBuffer.front();
setIndex++;
} else {
destBuffer[i] += srcBuffer.front();
}
srcBuffer.pop();
}
track->sampleCounter += amount;
track->CheckReleasedBuffers();
track->bufferLock.unlock();
}
if (sampleI16Size > setIndex)
memset(destBuffer, 0, (sampleI16Size - setIndex) * 2);
return oboe::DataCallbackResult::Continue;
}
}

View File

@ -0,0 +1,46 @@
#pragma once
#include <queue>
#include <oboe/Oboe.h>
#include <kernel/types/KEvent.h>
#include <audio/track.h>
#include "common.h"
namespace skyline::audio {
/**
* @brief The Audio class is used to mix audio from all tracks
*/
class Audio : public oboe::AudioStreamCallback {
private:
const DeviceState &state; //!< The state of the device
oboe::ManagedStream outputStream; //!< The output oboe audio stream
std::vector<std::shared_ptr<audio::AudioTrack>> audioTracks; //!< Vector containing a pointer of every open audio track
public:
Audio(const DeviceState &state);
/**
* @brief Opens a new track that can be used to play sound
* @param channelCount The amount channels that are present in the track
* @param sampleRate The sample rate of the track
* @param releaseCallback The callback to call when a buffer has been released
* @return A shared pointer to a new AudioTrack object
*/
std::shared_ptr<AudioTrack> OpenTrack(int channelCount, int sampleRate, const std::function<void()> &releaseCallback);
/**
* @brief Closes a track and frees its data
* @param track The track to close
*/
void CloseTrack(std::shared_ptr<AudioTrack> &track);
/**
* @brief The callback oboe uses to get audio sample data
* @param audioStream The audio stream we are being called by
* @param audioData The raw audio sample data
* @param numFrames The amount of frames the sample data needs to contain
*/
oboe::DataCallbackResult onAudioReady(oboe::AudioStream *audioStream, void *audioData, int32_t numFrames);
};
}

View File

@ -0,0 +1,43 @@
#pragma once
#include <oboe/Oboe.h>
#include <common.h>
namespace skyline::audio {
/**
* @brief The available PCM stream formats
*/
enum class PcmFormat : u8 {
Invalid = 0, //!< An invalid PCM format
Int8 = 1, //!< 8 bit integer PCM
Int16 = 2, //!< 16 bit integer PCM
Int24 = 3, //!< 24 bit integer PCM
Int32 = 4, //!< 32 bit integer PCM
PcmFloat = 5, //!< Floating point PCM
AdPcm = 6 //!< Adaptive differential PCM
};
/**
* @brief The state of an audio track
*/
enum class AudioOutState : u8 {
Started = 0, //!< Stream is started and is playing
Stopped = 1, //!< Stream is stopped, there are no samples left to play
Paused = 2 //!< Stream is paused, some samples may not have been played yet
};
/**
* @brief Stores various information about pushed buffers
*/
struct BufferIdentifier {
u64 tag; //!< The tag of the buffer
u64 finalSample; //!< The final sample this buffer will be played in
bool released; //!< Whether the buffer has been released
};
namespace constant {
constexpr int SampleRate = 48000; //!< The sampling rate to use for the oboe audio output
constexpr int ChannelCount = 2; //!< The amount of channels to use for the oboe audio output
constexpr oboe::AudioFormat PcmFormat = oboe::AudioFormat::I16; //!< The pcm data format to use for the oboe audio output
};
}

View File

@ -0,0 +1,71 @@
#include "track.h"
namespace skyline::audio {
AudioTrack::AudioTrack(int channelCount, int sampleRate, const std::function<void()> &releaseCallback) : channelCount(channelCount), sampleRate(sampleRate), releaseCallback(releaseCallback) {
if (sampleRate != constant::SampleRate)
throw exception("Unsupported audio sample rate: {}", sampleRate);
if (channelCount != constant::ChannelCount)
throw exception("Unsupported quantity of audio channels: {}", channelCount);
}
void AudioTrack::Stop() {
while (!identifierQueue.end()->released);
playbackState = AudioOutState::Stopped;
}
bool AudioTrack::ContainsBuffer(u64 tag) {
// Iterate from front of queue as we don't want released samples
for (auto identifier = identifierQueue.crbegin(); identifier != identifierQueue.crend(); ++identifier) {
if (identifier->released)
return false;
if (identifier->tag == tag)
return true;
}
return false;
}
std::vector<u64> AudioTrack::GetReleasedBuffers(u32 max) {
std::vector<u64> bufferIds;
for (u32 i = 0; i < max; i++) {
if (!identifierQueue.back().released)
break;
bufferIds.push_back(identifierQueue.back().tag);
identifierQueue.pop_back();
}
return bufferIds;
}
void AudioTrack::AppendBuffer(const std::vector<i16> &sampleData, u64 tag) {
BufferIdentifier identifier;
identifier.released = false;
identifier.tag = tag;
if (identifierQueue.empty())
identifier.finalSample = sampleData.size();
else
identifier.finalSample = sampleData.size() + identifierQueue.front().finalSample;
bufferLock.lock();
identifierQueue.push_front(identifier);
for (auto &sample : sampleData)
sampleQueue.push(sample);
bufferLock.unlock();
}
void AudioTrack::CheckReleasedBuffers() {
for (auto &identifier : identifierQueue) {
if (identifier.finalSample <= sampleCounter && !identifier.released) {
releaseCallback();
identifier.released = true;
}
}
}
}

View File

@ -0,0 +1,73 @@
#pragma once
#include <queue>
#include <kernel/types/KEvent.h>
#include <common.h>
#include "common.h"
namespace skyline::audio {
/**
* @brief The AudioTrack class manages the buffers for an audio stream
*/
class AudioTrack {
private:
const std::function<void()> releaseCallback; //!< Callback called when a buffer has been played
std::deque<BufferIdentifier> identifierQueue; //!< Queue of all appended buffer identifiers
int channelCount; //!< The amount channels present in the track
int sampleRate; //!< The sample rate of the track
public:
std::queue<i16> sampleQueue; //!< Queue of all appended buffer data
skyline::Mutex bufferLock; //!< Buffer access lock
AudioOutState playbackState{AudioOutState::Stopped}; //!< The current state of playback
u64 sampleCounter{}; //!< A counter used for tracking buffer status
/**
* @param channelCount The amount channels that will be present in the track
* @param sampleRate The sample rate to use for the track
* @param releaseCallback A callback to call when a buffer has been played
*/
AudioTrack(int channelCount, int sampleRate, const std::function<void()> &releaseCallback);
/**
* @brief Starts audio playback using data from appended buffers.
*/
inline void Start() {
playbackState = AudioOutState::Started;
}
/**
* @brief Stops audio playback. This waits for audio playback to finish before returning.
*/
void Stop();
/**
* @brief Checks if a buffer has been released
* @param tag The tag of the buffer to check
* @return True if the given buffer hasn't been released
*/
bool ContainsBuffer(u64 tag);
/**
* @brief Gets the IDs of all newly released buffers
* @param max The maximum amount of buffers to return
* @return A vector containing the identifiers of the buffers
*/
std::vector<u64> GetReleasedBuffers(u32 max);
/**
* @brief Appends audio samples to the output buffer
* @param sampleData Reference to a vector containing I16 format pcm data
* @param tag The tag of the buffer
*/
void AppendBuffer(const std::vector<i16> &sampleData, u64 tag);
/**
* @brief Checks if any buffers have been released and calls the appropriate callback for them
*/
void CheckReleasedBuffers();
};
}

View File

@ -1,6 +1,7 @@
#include "common.h"
#include "nce.h"
#include "gpu.h"
#include "audio.h"
#include <kernel/types/KThread.h>
#include <tinyxml2.h>
@ -132,6 +133,7 @@ namespace skyline {
// We assign these later as they use the state in their constructor and we don't want null pointers
nce = std::move(std::make_shared<NCE>(*this));
gpu = std::move(std::make_shared<gpu::GPU>(*this));
audio = std::move(std::make_shared<audio::Audio>(*this));
}
thread_local std::shared_ptr<kernel::type::KThread> DeviceState::thread = nullptr;

View File

@ -369,6 +369,9 @@ namespace skyline {
}
class OS;
}
namespace audio {
class Audio;
}
/**
* @brief This struct is used to hold the state of a device
@ -382,6 +385,7 @@ namespace skyline {
thread_local static ThreadContext *ctx; //!< This holds the context of the thread
std::shared_ptr<NCE> nce; //!< This holds a reference to the NCE class
std::shared_ptr<gpu::GPU> gpu; //!< This holds a reference to the GPU class
std::shared_ptr<audio::Audio> audio; //!< This holds a reference to the Audio class
std::shared_ptr<JvmManager> jvmManager; //!< This holds a reference to the JvmManager class
std::shared_ptr<Settings> settings; //!< This holds a reference to the Settings class
std::shared_ptr<Logger> logger; //!< This holds a reference to the Logger class

View File

@ -0,0 +1,103 @@
#include "audout.h"
#include <kernel/types/KProcess.h>
namespace skyline::service::audout {
audoutU::audoutU(const DeviceState &state, ServiceManager &manager) : BaseService(state, manager, false, Service::audout_u, {
{0x0, SFUNC(audoutU::ListAudioOuts)},
{0x1, SFUNC(audoutU::OpenAudioOut)}
}) {}
void audoutU::ListAudioOuts(type::KSession &session, ipc::IpcRequest &request, ipc::IpcResponse &response) {
state.process->WriteMemory(reinterpret_cast<void *>(const_cast<char *>(constant::DefaultAudioOutName.data())),
request.outputBuf.at(0).address, constant::DefaultAudioOutName.size());
}
void audoutU::OpenAudioOut(type::KSession &session, ipc::IpcRequest &request, ipc::IpcResponse &response) {
u32 sampleRate = request.Pop<u32>();
request.Pop<u16>(); // Channel count is stored in the upper half of a u32
u16 channelCount = request.Pop<u16>();
state.logger->Debug("audoutU: Opening an IAudioOut with sample rate: {}, channel count: {}", sampleRate, channelCount);
sampleRate = sampleRate ? sampleRate : audio::constant::SampleRate;
channelCount = channelCount ? channelCount : static_cast<u16>(audio::constant::ChannelCount);
manager.RegisterService(std::make_shared<IAudioOut>(state, manager, channelCount, sampleRate), session, response);
response.Push<u32>(sampleRate);
response.Push<u16>(channelCount);
response.Push<u16>(0);
response.Push(static_cast<u32>(audio::PcmFormat::Int16));
response.Push(static_cast<u32>(audio::AudioOutState::Stopped));
}
IAudioOut::IAudioOut(const DeviceState &state, ServiceManager &manager, int channelCount, int sampleRate) : releaseEvent(std::make_shared<type::KEvent>(state)), BaseService(state, manager, false, Service::audout_IAudioOut, {
{0x0, SFUNC(IAudioOut::GetAudioOutState)},
{0x1, SFUNC(IAudioOut::StartAudioOut)},
{0x2, SFUNC(IAudioOut::StopAudioOut)},
{0x3, SFUNC(IAudioOut::AppendAudioOutBuffer)},
{0x4, SFUNC(IAudioOut::RegisterBufferEvent)},
{0x5, SFUNC(IAudioOut::GetReleasedAudioOutBuffer)},
{0x6, SFUNC(IAudioOut::ContainsAudioOutBuffer)}
}) {
track = state.audio->OpenTrack(channelCount, sampleRate, [this]() { this->releaseEvent->Signal(); });
}
IAudioOut::~IAudioOut() {
state.audio->CloseTrack(track);
}
void IAudioOut::GetAudioOutState(type::KSession &session, ipc::IpcRequest &request, ipc::IpcResponse &response) {
response.Push(static_cast<u32>(track->playbackState));
}
void IAudioOut::StartAudioOut(type::KSession &session, ipc::IpcRequest &request, ipc::IpcResponse &response) {
state.logger->Debug("IAudioOut: Start playback");
track->Start();
}
void IAudioOut::StopAudioOut(type::KSession &session, ipc::IpcRequest &request, ipc::IpcResponse &response) {
state.logger->Debug("IAudioOut: Stop playback");
track->Stop();
}
void IAudioOut::AppendAudioOutBuffer(type::KSession &session, ipc::IpcRequest &request, ipc::IpcResponse &response) {
struct Data {
u64 nextBufferPtr;
u64 sampleBufferPtr;
u64 sampleCapacity;
u64 sampleSize;
u64 sampleOffset;
} data = state.process->GetObject<Data>(request.inputBuf.at(0).address);
u64 tag = request.Pop<u64>();
state.logger->Debug("IAudioOut: Appending buffer with address: 0x{:X}, size: 0x{:X}", data.sampleBufferPtr, data.sampleSize);
tmpSampleBuffer.resize(data.sampleSize / sizeof(i16));
state.process->ReadMemory(tmpSampleBuffer.data(), data.sampleBufferPtr, data.sampleSize);
track->AppendBuffer(tmpSampleBuffer, tag);
}
void IAudioOut::RegisterBufferEvent(type::KSession &session, ipc::IpcRequest &request, ipc::IpcResponse &response) {
auto handle = state.process->InsertItem(releaseEvent);
state.logger->Debug("Audout Buffer Release Event Handle: 0x{:X}", handle);
response.copyHandles.push_back(handle);
}
void IAudioOut::GetReleasedAudioOutBuffer(type::KSession &session, ipc::IpcRequest &request, ipc::IpcResponse &response) {
u32 maxCount = static_cast<u32>(request.outputBuf.at(0).size >> 3);
std::vector<u64> releasedBuffers = track->GetReleasedBuffers(maxCount);
u32 count = static_cast<u32>(releasedBuffers.size());
// Fill rest of output buffer with zeros
releasedBuffers.resize(maxCount, 0);
state.process->WriteMemory(releasedBuffers.data(), request.outputBuf.at(0).address, request.outputBuf.at(0).size);
response.Push<u32>(count);
}
void IAudioOut::ContainsAudioOutBuffer(type::KSession &session, ipc::IpcRequest &request, ipc::IpcResponse &response) {
u64 tag = request.Pop<u64>();
response.Push(static_cast<u32>(track->ContainsBuffer(tag)));
}
}

View File

@ -0,0 +1,87 @@
#pragma once
#include <audio.h>
#include <services/base_service.h>
#include <services/serviceman.h>
#include <kernel/types/KEvent.h>
namespace skyline::service::audout {
namespace constant {
constexpr std::string_view DefaultAudioOutName = "DeviceOut"; //!< The default audio output device name
};
/**
* @brief audout:u or IAudioOutManager is used to manage audio outputs (https://switchbrew.org/wiki/Audio_services#audout:u)
*/
class audoutU : public BaseService {
public:
audoutU(const DeviceState &state, ServiceManager &manager);
/**
* @brief Returns a list of all available audio outputs (https://switchbrew.org/wiki/Audio_services#ListAudioOuts)
*/
void ListAudioOuts(type::KSession &session, ipc::IpcRequest &request, ipc::IpcResponse &response);
/**
* @brief Creates a new audoutU::IAudioOut object and returns a handle to it (https://switchbrew.org/wiki/Audio_services#OpenAudioOut)
*/
void OpenAudioOut(type::KSession &session, ipc::IpcRequest &request, ipc::IpcResponse &response);
};
/**
* @brief IAudioOut is a service opened when OpenAudioOut is called by audout (https://switchbrew.org/wiki/Audio_services#IAudioOut)
*/
class IAudioOut : public BaseService {
private:
std::shared_ptr<audio::AudioTrack> track; //!< The audio track associated with the audio out
std::shared_ptr<type::KEvent> releaseEvent; //!< The KEvent that is signalled when a buffer has been released
std::vector<i16> tmpSampleBuffer; //!< A temporary buffer used to store sample data in AppendAudioOutBuffer
public:
/**
* @param channelCount The channel count of the audio data the audio out will be fed
* @param sampleRate The sample rate of the audio data the audio out will be fed
*/
IAudioOut(const DeviceState &state, ServiceManager &manager, int channelCount, int sampleRate);
/**
* @brief Closes the audio track
*/
~IAudioOut();
/**
* @brief Returns the playback state of the audio output (https://switchbrew.org/wiki/Audio_services#GetAudioOutState)
*/
void GetAudioOutState(type::KSession &session, ipc::IpcRequest &request, ipc::IpcResponse &response);
/**
* @brief Starts playback using data from appended samples (https://switchbrew.org/wiki/Audio_services#StartAudioOut)
*/
void StartAudioOut(type::KSession &session, ipc::IpcRequest &request, ipc::IpcResponse &response);
/**
* @brief Stops playback of audio, waits for all samples to be released (https://switchbrew.org/wiki/Audio_services#StartAudioOut)
*/
void StopAudioOut(type::KSession &session, ipc::IpcRequest &request, ipc::IpcResponse &response);
/**
* @brief Appends sample data to the output buffer (https://switchbrew.org/wiki/Audio_services#AppendAudioOutBuffer)
*/
void AppendAudioOutBuffer(type::KSession &session, ipc::IpcRequest &request, ipc::IpcResponse &response);
/**
* @brief Returns a handle to the sample release KEvent (https://switchbrew.org/wiki/Audio_services#AppendAudioOutBuffer)
*/
void RegisterBufferEvent(type::KSession &session, ipc::IpcRequest &request, ipc::IpcResponse &response);
/**
* @brief Returns the IDs of all pending released buffers (https://switchbrew.org/wiki/Audio_services#GetReleasedAudioOutBuffer)
*/
void GetReleasedAudioOutBuffer(type::KSession &session, ipc::IpcRequest &request, ipc::IpcResponse &response);
/**
* @brief Checks if the given buffer ID is in the playback queue (https://switchbrew.org/wiki/Audio_services#ContainsAudioOutBuffer)
*/
void ContainsAudioOutBuffer(type::KSession &session, ipc::IpcRequest &request, ipc::IpcResponse &response);
};
}

View File

@ -36,6 +36,8 @@ namespace skyline::service {
am_ILibraryAppletCreator,
am_IDebugFunctions,
am_IAppletCommonFunctions,
audout_u,
audout_IAudioOut,
hid,
hid_IAppletResource,
time,
@ -75,6 +77,8 @@ namespace skyline::service {
{"am:IApplicationFunctions", Service::am_IApplicationFunctions},
{"am:IDebugFunctions", Service::am_IDebugFunctions},
{"am:IAppletCommonFunctions", Service::am_IAppletCommonFunctions},
{"audout:u", Service::audout_u},
{"audout:IAudioOut", Service::audout_IAudioOut},
{"hid", Service::hid},
{"hid:IAppletResource", Service::hid_IAppletResource},
{"time:s", Service::time},

View File

@ -5,6 +5,7 @@
#include "apm/apm.h"
#include "am/applet.h"
#include "am/appletController.h"
#include "audout/audout.h"
#include "fatal/fatal.h"
#include "hid/hid.h"
#include "time/timesrv.h"
@ -82,6 +83,9 @@ namespace skyline::service {
case Service::am_IAppletCommonFunctions:
serviceObj = std::make_shared<am::IAppletCommonFunctions>(state, *this);
break;
case Service::audout_u:
serviceObj = std::make_shared<audout::audoutU>(state, *this);
break;
case Service::hid:
serviceObj = std::make_shared<hid::hid>(state, *this);
break;