From b7e88aaae0813cebea334fe6715980c74dbd97b4 Mon Sep 17 00:00:00 2001 From: "Ryan C. Gordon" Date: Wed, 16 May 2018 02:03:06 -0400 Subject: [PATCH] audio: Added ARM NEON versions of audio converters. These are _much_ faster than the scalar equivalents on the Raspberry Pi that I tested on. Often 3x to 4x as fast! --- docs/README-raspberrypi.md | 10 + include/SDL_cpuinfo.h | 3 + src/audio/SDL_audiotypecvt.c | 544 ++++++++++++++++++++++++++++++++++- 3 files changed, 555 insertions(+), 2 deletions(-) diff --git a/docs/README-raspberrypi.md b/docs/README-raspberrypi.md index fbcffa1ff..5e23ad5e0 100644 --- a/docs/README-raspberrypi.md +++ b/docs/README-raspberrypi.md @@ -27,6 +27,16 @@ OpenGL ES 2.x, it usually comes pre-installed, but in any case: sudo apt-get install libraspberrypi0 libraspberrypi-bin libraspberrypi-dev + +================================================================================ + NEON +================================================================================ + +If your Pi has NEON support, make sure you add -mfpu=neon to your CFLAGS so +that SDL will select some otherwise-disabled highly-optimized code. The +original Pi units don't have NEON, the Pi2 probably does, and the Pi3 +definitely does. + ================================================================================ Cross compiling from x86 Linux ================================================================================ diff --git a/include/SDL_cpuinfo.h b/include/SDL_cpuinfo.h index acdb1a872..ee3a47e84 100644 --- a/include/SDL_cpuinfo.h +++ b/include/SDL_cpuinfo.h @@ -57,6 +57,9 @@ #undef bool #endif #endif +#if defined(__ARM_NEON__) && !defined(SDL_DISABLE_ARM_NEON_H) +#include +#endif #if defined(__3dNOW__) && !defined(SDL_DISABLE_MM3DNOW_H) #include #endif diff --git a/src/audio/SDL_audiotypecvt.c b/src/audio/SDL_audiotypecvt.c index 736d46b7c..29cc24d03 100644 --- a/src/audio/SDL_audiotypecvt.c +++ b/src/audio/SDL_audiotypecvt.c @@ -25,8 +25,9 @@ #include "SDL_cpuinfo.h" #include "SDL_assert.h" -/* !!! FIXME: write NEON code. */ -#define HAVE_NEON_INTRINSICS 0 +#ifdef __ARM_NEON__ +#define HAVE_NEON_INTRINSICS 1 +#endif #ifdef __SSE2__ #define HAVE_SSE2_INTRINSICS 1 @@ -850,6 +851,538 @@ SDL_Convert_F32_to_S32_SSE2(SDL_AudioCVT *cvt, SDL_AudioFormat format) #endif +#if HAVE_NEON_INTRINSICS +static void SDLCALL +SDL_Convert_S8_to_F32_NEON(SDL_AudioCVT *cvt, SDL_AudioFormat format) +{ + const Sint8 *src = ((const Sint8 *) (cvt->buf + cvt->len_cvt)) - 1; + float *dst = ((float *) (cvt->buf + cvt->len_cvt * 4)) - 1; + int i; + + LOG_DEBUG_CONVERT("AUDIO_S8", "AUDIO_F32 (using NEON)"); + + /* Get dst aligned to 16 bytes (since buffer is growing, we don't have to worry about overreading from src) */ + for (i = cvt->len_cvt; i && (((size_t) (dst-15)) & 15); --i, --src, --dst) { + *dst = ((float) *src) * DIVBY128; + } + + src -= 15; dst -= 15; /* adjust to read NEON blocks from the start. */ + SDL_assert(!i || ((((size_t) dst) & 15) == 0)); + + /* Make sure src is aligned too. */ + if ((((size_t) src) & 15) == 0) { + /* Aligned! Do NEON blocks as long as we have 16 bytes available. */ + const int8_t *mmsrc = (const int8_t *) src; + const float32x4_t divby128 = vdupq_n_f32(DIVBY128); + while (i >= 16) { /* 16 * 8-bit */ + const int8x16_t bytes = vld1q_s8(mmsrc); /* get 16 sint8 into a NEON register. */ + const int16x8_t int16hi = vmovl_s8(vget_high_s8(bytes)); /* convert top 8 bytes to 8 int16 */ + const int16x8_t int16lo = vmovl_s8(vget_low_s8(bytes)); /* convert bottom 8 bytes to 8 int16 */ + /* split int16 to two int32, then convert to float, then multiply to normalize, store. */ + vst1q_f32(dst, vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(int16hi))), divby128)); + vst1q_f32(dst+4, vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(int16hi))), divby128)); + vst1q_f32(dst+8, vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(int16lo))), divby128)); + vst1q_f32(dst+12, vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(int16lo))), divby128)); + i -= 16; mmsrc -= 16; dst -= 16; + } + + src = (const Sint8 *) mmsrc; + } + + src += 15; dst += 15; /* adjust for any scalar finishing. */ + + /* Finish off any leftovers with scalar operations. */ + while (i) { + *dst = ((float) *src) * DIVBY128; + i--; src--; dst--; + } + + cvt->len_cvt *= 4; + if (cvt->filters[++cvt->filter_index]) { + cvt->filters[cvt->filter_index](cvt, AUDIO_F32SYS); + } +} + +static void SDLCALL +SDL_Convert_U8_to_F32_NEON(SDL_AudioCVT *cvt, SDL_AudioFormat format) +{ + const Uint8 *src = ((const Uint8 *) (cvt->buf + cvt->len_cvt)) - 1; + float *dst = ((float *) (cvt->buf + cvt->len_cvt * 4)) - 1; + int i; + + LOG_DEBUG_CONVERT("AUDIO_U8", "AUDIO_F32 (using NEON)"); + + /* Get dst aligned to 16 bytes (since buffer is growing, we don't have to worry about overreading from src) */ + for (i = cvt->len_cvt; i && (((size_t) (dst-15)) & 15); --i, --src, --dst) { + *dst = (((float) *src) * DIVBY128) - 1.0f; + } + + src -= 15; dst -= 15; /* adjust to read NEON blocks from the start. */ + SDL_assert(!i || ((((size_t) dst) & 15) == 0)); + + /* Make sure src is aligned too. */ + if ((((size_t) src) & 15) == 0) { + /* Aligned! Do NEON blocks as long as we have 16 bytes available. */ + const uint8_t *mmsrc = (const uint8_t *) src; + const float32x4_t divby128 = vdupq_n_f32(DIVBY128); + const float32x4_t one = vdupq_n_f32(1.0f); + while (i >= 16) { /* 16 * 8-bit */ + const uint8x16_t bytes = vld1q_u8(mmsrc); /* get 16 uint8 into a NEON register. */ + const uint16x8_t uint16hi = vmovl_u8(vget_high_u8(bytes)); /* convert top 8 bytes to 8 uint16 */ + const uint16x8_t uint16lo = vmovl_u8(vget_low_u8(bytes)); /* convert bottom 8 bytes to 8 uint16 */ + /* split uint16 to two uint32, then convert to float, then multiply to normalize, subtract to adjust for sign, store. */ + vst1q_f32(dst, vmlsq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(uint16hi))), divby128, one)); + vst1q_f32(dst+4, vmlsq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(uint16hi))), divby128, one)); + vst1q_f32(dst+8, vmlsq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(uint16lo))), divby128, one)); + vst1q_f32(dst+12, vmlsq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(uint16lo))), divby128, one)); + i -= 16; mmsrc -= 16; dst -= 16; + } + + src = (const Uint8 *) mmsrc; + } + + src += 15; dst += 15; /* adjust for any scalar finishing. */ + + /* Finish off any leftovers with scalar operations. */ + while (i) { + *dst = (((float) *src) * DIVBY128) - 1.0f; + i--; src--; dst--; + } + + cvt->len_cvt *= 4; + if (cvt->filters[++cvt->filter_index]) { + cvt->filters[cvt->filter_index](cvt, AUDIO_F32SYS); + } +} + +static void SDLCALL +SDL_Convert_S16_to_F32_NEON(SDL_AudioCVT *cvt, SDL_AudioFormat format) +{ + const Sint16 *src = ((const Sint16 *) (cvt->buf + cvt->len_cvt)) - 1; + float *dst = ((float *) (cvt->buf + cvt->len_cvt * 2)) - 1; + int i; + + LOG_DEBUG_CONVERT("AUDIO_S16", "AUDIO_F32 (using NEON)"); + + /* Get dst aligned to 16 bytes (since buffer is growing, we don't have to worry about overreading from src) */ + for (i = cvt->len_cvt / sizeof (Sint16); i && (((size_t) (dst-7)) & 15); --i, --src, --dst) { + *dst = ((float) *src) * DIVBY32768; + } + + src -= 7; dst -= 7; /* adjust to read NEON blocks from the start. */ + SDL_assert(!i || ((((size_t) dst) & 15) == 0)); + + /* Make sure src is aligned too. */ + if ((((size_t) src) & 15) == 0) { + /* Aligned! Do NEON blocks as long as we have 16 bytes available. */ + const float32x4_t divby32768 = vdupq_n_f32(DIVBY32768); + while (i >= 8) { /* 8 * 16-bit */ + const int16x8_t ints = vld1q_s16((int16_t const *) src); /* get 8 sint16 into a NEON register. */ + /* split int16 to two int32, then convert to float, then multiply to normalize, store. */ + vst1q_f32(dst, vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(ints))), divby32768)); + vst1q_f32(dst+4, vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(ints))), divby32768)); + i -= 8; src -= 8; dst -= 8; + } + } + + src += 7; dst += 7; /* adjust for any scalar finishing. */ + + /* Finish off any leftovers with scalar operations. */ + while (i) { + *dst = ((float) *src) * DIVBY32768; + i--; src--; dst--; + } + + cvt->len_cvt *= 2; + if (cvt->filters[++cvt->filter_index]) { + cvt->filters[cvt->filter_index](cvt, AUDIO_F32SYS); + } +} + +static void SDLCALL +SDL_Convert_U16_to_F32_NEON(SDL_AudioCVT *cvt, SDL_AudioFormat format) +{ + const Uint16 *src = ((const Uint16 *) (cvt->buf + cvt->len_cvt)) - 1; + float *dst = ((float *) (cvt->buf + cvt->len_cvt * 2)) - 1; + int i; + + LOG_DEBUG_CONVERT("AUDIO_U16", "AUDIO_F32 (using NEON)"); + + /* Get dst aligned to 16 bytes (since buffer is growing, we don't have to worry about overreading from src) */ + for (i = cvt->len_cvt / sizeof (Sint16); i && (((size_t) (dst-7)) & 15); --i, --src, --dst) { + *dst = (((float) *src) * DIVBY32768) - 1.0f; + } + + src -= 7; dst -= 7; /* adjust to read NEON blocks from the start. */ + SDL_assert(!i || ((((size_t) dst) & 15) == 0)); + + /* Make sure src is aligned too. */ + if ((((size_t) src) & 15) == 0) { + /* Aligned! Do NEON blocks as long as we have 16 bytes available. */ + const float32x4_t divby32768 = vdupq_n_f32(DIVBY32768); + const float32x4_t one = vdupq_n_f32(1.0f); + while (i >= 8) { /* 8 * 16-bit */ + const uint16x8_t uints = vld1q_u16((uint16_t const *) src); /* get 8 uint16 into a NEON register. */ + /* split uint16 to two int32, then convert to float, then multiply to normalize, subtract for sign, store. */ + vst1q_f32(dst, vmlsq_f32(one, vcvtq_f32_u32(vmovl_u16(vget_low_u16(uints))), divby32768)); + vst1q_f32(dst+4, vmlsq_f32(one, vcvtq_f32_u32(vmovl_u16(vget_high_u16(uints))), divby32768)); + i -= 8; src -= 8; dst -= 8; + } + } + + src += 7; dst += 7; /* adjust for any scalar finishing. */ + + /* Finish off any leftovers with scalar operations. */ + while (i) { + *dst = (((float) *src) * DIVBY32768) - 1.0f; + i--; src--; dst--; + } + + cvt->len_cvt *= 2; + if (cvt->filters[++cvt->filter_index]) { + cvt->filters[cvt->filter_index](cvt, AUDIO_F32SYS); + } +} + +static void SDLCALL +SDL_Convert_S32_to_F32_NEON(SDL_AudioCVT *cvt, SDL_AudioFormat format) +{ + const Sint32 *src = (const Sint32 *) cvt->buf; + float *dst = (float *) cvt->buf; + int i; + + LOG_DEBUG_CONVERT("AUDIO_S32", "AUDIO_F32 (using NEON)"); + + /* Get dst aligned to 16 bytes */ + for (i = cvt->len_cvt / sizeof (Sint32); i && (((size_t) dst) & 15); --i, ++src, ++dst) { + *dst = ((float) (*src>>8)) * DIVBY8388607; + } + + SDL_assert(!i || ((((size_t) dst) & 15) == 0)); + SDL_assert(!i || ((((size_t) src) & 15) == 0)); + + { + /* Aligned! Do NEON blocks as long as we have 16 bytes available. */ + const float32x4_t divby8388607 = vdupq_n_f32(DIVBY8388607); + const int32_t *mmsrc = (const int32_t *) src; + while (i >= 4) { /* 4 * sint32 */ + /* shift out lowest bits so int fits in a float32. Small precision loss, but much faster. */ + vst1q_f32(dst, vmulq_f32(vcvtq_f32_s32(vshrq_n_s32(vld1q_s32(mmsrc), 8)), divby8388607)); + i -= 4; mmsrc += 4; dst += 4; + } + src = (const Sint32 *) mmsrc; + } + + /* Finish off any leftovers with scalar operations. */ + while (i) { + *dst = ((float) (*src>>8)) * DIVBY8388607; + i--; src++; dst++; + } + + if (cvt->filters[++cvt->filter_index]) { + cvt->filters[cvt->filter_index](cvt, AUDIO_F32SYS); + } +} + +static void SDLCALL +SDL_Convert_F32_to_S8_NEON(SDL_AudioCVT *cvt, SDL_AudioFormat format) +{ + const float *src = (const float *) cvt->buf; + Sint8 *dst = (Sint8 *) cvt->buf; + int i; + + LOG_DEBUG_CONVERT("AUDIO_F32", "AUDIO_S8 (using NEON)"); + + /* Get dst aligned to 16 bytes */ + for (i = cvt->len_cvt / sizeof (float); i && (((size_t) dst) & 15); --i, ++src, ++dst) { + const float sample = *src; + if (sample >= 1.0f) { + *dst = 127; + } else if (sample <= -1.0f) { + *dst = -128; + } else { + *dst = (Sint8)(sample * 127.0f); + } + } + + SDL_assert(!i || ((((size_t) dst) & 15) == 0)); + + /* Make sure src is aligned too. */ + if ((((size_t) src) & 15) == 0) { + /* Aligned! Do NEON blocks as long as we have 16 bytes available. */ + const float32x4_t one = vdupq_n_f32(1.0f); + const float32x4_t negone = vdupq_n_f32(-1.0f); + const float32x4_t mulby127 = vdupq_n_f32(127.0f); + int8_t *mmdst = (int8_t *) dst; + while (i >= 16) { /* 16 * float32 */ + const int32x4_t ints1 = vcvtq_s32_f32(vmulq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src)), one), mulby127)); /* load 4 floats, clamp, convert to sint32 */ + const int32x4_t ints2 = vcvtq_s32_f32(vmulq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src+4)), one), mulby127)); /* load 4 floats, clamp, convert to sint32 */ + const int32x4_t ints3 = vcvtq_s32_f32(vmulq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src+8)), one), mulby127)); /* load 4 floats, clamp, convert to sint32 */ + const int32x4_t ints4 = vcvtq_s32_f32(vmulq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src+12)), one), mulby127)); /* load 4 floats, clamp, convert to sint32 */ + const int8x8_t i8lo = vmovn_s16(vcombine_s16(vmovn_s32(ints1), vmovn_s32(ints2))); /* narrow to sint16, combine, narrow to sint8 */ + const int8x8_t i8hi = vmovn_s16(vcombine_s16(vmovn_s32(ints3), vmovn_s32(ints4))); /* narrow to sint16, combine, narrow to sint8 */ + vst1q_s8(mmdst, vcombine_s8(i8lo, i8hi)); /* combine to int8x16_t, store out */ + i -= 16; src += 16; mmdst += 16; + } + dst = (Sint8 *) mmdst; + } + + /* Finish off any leftovers with scalar operations. */ + while (i) { + const float sample = *src; + if (sample >= 1.0f) { + *dst = 127; + } else if (sample <= -1.0f) { + *dst = -128; + } else { + *dst = (Sint8)(sample * 127.0f); + } + i--; src++; dst++; + } + + cvt->len_cvt /= 4; + if (cvt->filters[++cvt->filter_index]) { + cvt->filters[cvt->filter_index](cvt, AUDIO_S8); + } +} + +static void SDLCALL +SDL_Convert_F32_to_U8_NEON(SDL_AudioCVT *cvt, SDL_AudioFormat format) +{ + const float *src = (const float *) cvt->buf; + Uint8 *dst = (Uint8 *) cvt->buf; + int i; + + LOG_DEBUG_CONVERT("AUDIO_F32", "AUDIO_U8 (using NEON)"); + + /* Get dst aligned to 16 bytes */ + for (i = cvt->len_cvt / sizeof (float); i && (((size_t) dst) & 15); --i, ++src, ++dst) { + const float sample = *src; + if (sample >= 1.0f) { + *dst = 255; + } else if (sample <= -1.0f) { + *dst = 0; + } else { + *dst = (Uint8)((sample + 1.0f) * 127.0f); + } + } + + SDL_assert(!i || ((((size_t) dst) & 15) == 0)); + + /* Make sure src is aligned too. */ + if ((((size_t) src) & 15) == 0) { + /* Aligned! Do NEON blocks as long as we have 16 bytes available. */ + const float32x4_t one = vdupq_n_f32(1.0f); + const float32x4_t negone = vdupq_n_f32(-1.0f); + const float32x4_t mulby127 = vdupq_n_f32(127.0f); + uint8_t *mmdst = (uint8_t *) dst; + while (i >= 16) { /* 16 * float32 */ + const uint32x4_t uints1 = vcvtq_u32_f32(vmulq_f32(vaddq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src)), one), one), mulby127)); /* load 4 floats, clamp, convert to uint32 */ + const uint32x4_t uints2 = vcvtq_u32_f32(vmulq_f32(vaddq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src+4)), one), one), mulby127)); /* load 4 floats, clamp, convert to uint32 */ + const uint32x4_t uints3 = vcvtq_u32_f32(vmulq_f32(vaddq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src+8)), one), one), mulby127)); /* load 4 floats, clamp, convert to uint32 */ + const uint32x4_t uints4 = vcvtq_u32_f32(vmulq_f32(vaddq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src+12)), one), one), mulby127)); /* load 4 floats, clamp, convert to uint32 */ + const uint8x8_t ui8lo = vmovn_u16(vcombine_u16(vmovn_u32(uints1), vmovn_u32(uints2))); /* narrow to uint16, combine, narrow to uint8 */ + const uint8x8_t ui8hi = vmovn_u16(vcombine_u16(vmovn_u32(uints3), vmovn_u32(uints4))); /* narrow to uint16, combine, narrow to uint8 */ + vst1q_u8(mmdst, vcombine_u8(ui8lo, ui8hi)); /* combine to uint8x16_t, store out */ + i -= 16; src += 16; mmdst += 16; + } + + dst = (Uint8 *) mmdst; + } + + /* Finish off any leftovers with scalar operations. */ + while (i) { + const float sample = *src; + if (sample >= 1.0f) { + *dst = 255; + } else if (sample <= -1.0f) { + *dst = 0; + } else { + *dst = (Uint8)((sample + 1.0f) * 127.0f); + } + i--; src++; dst++; + } + + cvt->len_cvt /= 4; + if (cvt->filters[++cvt->filter_index]) { + cvt->filters[cvt->filter_index](cvt, AUDIO_U8); + } +} + +static void SDLCALL +SDL_Convert_F32_to_S16_NEON(SDL_AudioCVT *cvt, SDL_AudioFormat format) +{ + const float *src = (const float *) cvt->buf; + Sint16 *dst = (Sint16 *) cvt->buf; + int i; + + LOG_DEBUG_CONVERT("AUDIO_F32", "AUDIO_S16 (using NEON)"); + + /* Get dst aligned to 16 bytes */ + for (i = cvt->len_cvt / sizeof (float); i && (((size_t) dst) & 15); --i, ++src, ++dst) { + const float sample = *src; + if (sample >= 1.0f) { + *dst = 32767; + } else if (sample <= -1.0f) { + *dst = -32768; + } else { + *dst = (Sint16)(sample * 32767.0f); + } + } + + SDL_assert(!i || ((((size_t) dst) & 15) == 0)); + + /* Make sure src is aligned too. */ + if ((((size_t) src) & 15) == 0) { + /* Aligned! Do NEON blocks as long as we have 16 bytes available. */ + const float32x4_t one = vdupq_n_f32(1.0f); + const float32x4_t negone = vdupq_n_f32(-1.0f); + const float32x4_t mulby32767 = vdupq_n_f32(32767.0f); + int16_t *mmdst = (int16_t *) dst; + while (i >= 8) { /* 8 * float32 */ + const int32x4_t ints1 = vcvtq_s32_f32(vmulq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src)), one), mulby32767)); /* load 4 floats, clamp, convert to sint32 */ + const int32x4_t ints2 = vcvtq_s32_f32(vmulq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src+4)), one), mulby32767)); /* load 4 floats, clamp, convert to sint32 */ + vst1q_s16(mmdst, vcombine_s16(vmovn_s32(ints1), vmovn_s32(ints2))); /* narrow to sint16, combine, store out. */ + i -= 8; src += 8; mmdst += 8; + } + dst = (Sint16 *) mmdst; + } + + /* Finish off any leftovers with scalar operations. */ + while (i) { + const float sample = *src; + if (sample >= 1.0f) { + *dst = 32767; + } else if (sample <= -1.0f) { + *dst = -32768; + } else { + *dst = (Sint16)(sample * 32767.0f); + } + i--; src++; dst++; + } + + cvt->len_cvt /= 2; + if (cvt->filters[++cvt->filter_index]) { + cvt->filters[cvt->filter_index](cvt, AUDIO_S16SYS); + } +} + +static void SDLCALL +SDL_Convert_F32_to_U16_NEON(SDL_AudioCVT *cvt, SDL_AudioFormat format) +{ + const float *src = (const float *) cvt->buf; + Uint16 *dst = (Uint16 *) cvt->buf; + int i; + + LOG_DEBUG_CONVERT("AUDIO_F32", "AUDIO_U16 (using NEON)"); + + /* Get dst aligned to 16 bytes */ + for (i = cvt->len_cvt / sizeof (float); i && (((size_t) dst) & 15); --i, ++src, ++dst) { + const float sample = *src; + if (sample >= 1.0f) { + *dst = 65535; + } else if (sample <= -1.0f) { + *dst = 0; + } else { + *dst = (Uint16)((sample + 1.0f) * 32767.0f); + } + } + + SDL_assert(!i || ((((size_t) dst) & 15) == 0)); + + /* Make sure src is aligned too. */ + if ((((size_t) src) & 15) == 0) { + /* Aligned! Do NEON blocks as long as we have 16 bytes available. */ + const float32x4_t one = vdupq_n_f32(1.0f); + const float32x4_t negone = vdupq_n_f32(-1.0f); + const float32x4_t mulby32767 = vdupq_n_f32(32767.0f); + uint16_t *mmdst = (uint16_t *) dst; + while (i >= 8) { /* 8 * float32 */ + const uint32x4_t uints1 = vcvtq_u32_f32(vmulq_f32(vaddq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src)), one), one), mulby32767)); /* load 4 floats, clamp, convert to uint32 */ + const uint32x4_t uints2 = vcvtq_u32_f32(vmulq_f32(vaddq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src+4)), one), one), mulby32767)); /* load 4 floats, clamp, convert to uint32 */ + vst1q_u16(mmdst, vcombine_u16(vmovn_u32(uints1), vmovn_u32(uints2))); /* narrow to uint16, combine, store out. */ + i -= 8; src += 8; mmdst += 8; + } + dst = (Uint16 *) mmdst; + } + + /* Finish off any leftovers with scalar operations. */ + while (i) { + const float sample = *src; + if (sample >= 1.0f) { + *dst = 65535; + } else if (sample <= -1.0f) { + *dst = 0; + } else { + *dst = (Uint16)((sample + 1.0f) * 32767.0f); + } + i--; src++; dst++; + } + + cvt->len_cvt /= 2; + if (cvt->filters[++cvt->filter_index]) { + cvt->filters[cvt->filter_index](cvt, AUDIO_U16SYS); + } +} + +static void SDLCALL +SDL_Convert_F32_to_S32_NEON(SDL_AudioCVT *cvt, SDL_AudioFormat format) +{ + const float *src = (const float *) cvt->buf; + Sint32 *dst = (Sint32 *) cvt->buf; + int i; + + LOG_DEBUG_CONVERT("AUDIO_F32", "AUDIO_S32 (using NEON)"); + + /* Get dst aligned to 16 bytes */ + for (i = cvt->len_cvt / sizeof (float); i && (((size_t) dst) & 15); --i, ++src, ++dst) { + const float sample = *src; + if (sample >= 1.0f) { + *dst = 2147483647; + } else if (sample <= -1.0f) { + *dst = -2147483648; + } else { + *dst = ((Sint32)(sample * 8388607.0f)) << 8; + } + } + + SDL_assert(!i || ((((size_t) dst) & 15) == 0)); + SDL_assert(!i || ((((size_t) src) & 15) == 0)); + + { + /* Aligned! Do NEON blocks as long as we have 16 bytes available. */ + const float32x4_t one = vdupq_n_f32(1.0f); + const float32x4_t negone = vdupq_n_f32(-1.0f); + const float32x4_t mulby8388607 = vdupq_n_f32(8388607.0f); + int32_t *mmdst = (int32_t *) dst; + while (i >= 4) { /* 4 * float32 */ + vst1q_s32(mmdst, vshlq_n_s32(vcvtq_s32_f32(vmulq_f32(vminq_f32(vmaxq_f32(negone, vld1q_f32(src)), one), mulby8388607)), 8)); + i -= 4; src += 4; mmdst += 4; + } + dst = (Sint32 *) mmdst; + } + + /* Finish off any leftovers with scalar operations. */ + while (i) { + const float sample = *src; + if (sample >= 1.0f) { + *dst = 2147483647; + } else if (sample <= -1.0f) { + *dst = -2147483648; + } else { + *dst = ((Sint32)(sample * 8388607.0f)) << 8; + } + i--; src++; dst++; + } + + if (cvt->filters[++cvt->filter_index]) { + cvt->filters[cvt->filter_index](cvt, AUDIO_S32SYS); + } +} +#endif + + + void SDL_ChooseAudioConverters(void) { static SDL_bool converters_chosen = SDL_FALSE; @@ -878,6 +1411,13 @@ void SDL_ChooseAudioConverters(void) } #endif +#if HAVE_NEON_INTRINSICS + if (SDL_HasNEON()) { + SET_CONVERTER_FUNCS(NEON); + return; + } +#endif + #if NEED_SCALAR_CONVERTER_FALLBACKS SET_CONVERTER_FUNCS(Scalar); #endif