Use an array of ALvoice pointers for the active voices

This allows growing the array atomically with the mixer since the ALvoice
objects themselves don't move, and a new larger array of them can be swapped in
without blocking the mixer.

5 files changed, 130 insertions, 112 deletions

diff --git a/al/source.cpp b/al/source.cpp
index ace69310..9d6fd8be 100644
--- a/al/source.cpp
+++ b/al/source.cpp
@@ -78,13 +78,14 @@ using std::chrono::nanoseconds;
 
 ALvoice *GetSourceVoice(ALsource *source, ALCcontext *context)
 {
+    auto voicelist = context->getVoicesSpan();
     ALuint idx{source->VoiceIdx};
-    if(idx < context->mVoices.size())
+    if(idx < voicelist.size())
     {
         ALuint sid{source->id};
-        ALvoice &voice = context->mVoices[idx];
-        if(voice.mSourceID.load(std::memory_order_acquire) == sid)
-            return &voice;
+        ALvoice *voice = voicelist[idx];
+        if(voice->mSourceID.load(std::memory_order_acquire) == sid)
+            return voice;
     }
     source->VoiceIdx = INVALID_VOICE_IDX;
     return nullptr;
@@ -2679,21 +2680,27 @@ START_API_FUNC
     }
 
     /* Count the number of reusable voices. */
+    auto voicelist = context->getVoicesSpan();
     size_t free_voices{0};
-    for(const ALvoice &voice : context->mVoices)
+    for(const ALvoice *voice : voicelist)
     {
-        free_voices += (voice.mPlayState.load(std::memory_order_acquire) == ALvoice::Stopped
-            && voice.mSourceID.load(std::memory_order_relaxed) == 0u
-            && voice.mPendingStop.load(std::memory_order_relaxed) == false);
+        free_voices += (voice->mPlayState.load(std::memory_order_acquire) == ALvoice::Stopped
+            && voice->mSourceID.load(std::memory_order_relaxed) == 0u
+            && voice->mPendingStop.load(std::memory_order_relaxed) == false);
         if(free_voices == srchandles.size())
             break;
     }
     if UNLIKELY(srchandles.size() != free_voices)
     {
-        BackendLockGuard __{*device->Backend};
-        /* Increase the number of voices to handle the request. */
-        const size_t need_voices{srchandles.size() - free_voices};
-        context->mVoices.resize(context->mVoices.size() + need_voices);
+        const size_t inc_amount{srchandles.size() - free_voices};
+        auto &allvoices = *context->mVoices.load(std::memory_order_relaxed);
+        if(inc_amount > allvoices.size() - voicelist.size())
+        {
+            /* Increase the number of voices to handle the request. */
+            context->allocVoices(inc_amount > (allvoices.size() - voicelist.size()));
+        }
+        context->mActiveVoiceCount.fetch_add(inc_amount, std::memory_order_release);
+        voicelist = context->getVoicesSpan();
     }
 
     VoiceChange *tail{}, *cur{};
@@ -2765,17 +2772,18 @@ START_API_FUNC
         }
 
         /* Look for an unused voice to play this source with. */
-        auto find_voice = [](const ALvoice &v) noexcept -> bool
+        ALuint vidx{0};
+        for(ALvoice *v : voicelist)
         {
-            return v.mPlayState.load(std::memory_order_acquire) == ALvoice::Stopped
-                && v.mSourceID.load(std::memory_order_relaxed) == 0u
-                && v.mPendingStop.load(std::memory_order_relaxed) == false;
+            if(v->mPlayState.load(std::memory_order_acquire) == ALvoice::Stopped
+                && v->mSourceID.load(std::memory_order_relaxed) == 0u
+                && v->mPendingStop.load(std::memory_order_relaxed) == false)
+            {
+                voice = v;
+                break;
+            }
+            ++vidx;
         };
-        auto voices_end = context->mVoices.data() + context->mVoices.size();
-        voice = std::find_if(context->mVoices.data(), voices_end, find_voice);
-        assert(voice != voices_end);
-
-        auto vidx = static_cast<ALuint>(std::distance(context->mVoices.data(), voice));
 
         /* A source that's not playing or paused has any offset applied when it
          * starts playing.
@@ -3352,13 +3360,14 @@ ALsource::~ALsource()
 void UpdateAllSourceProps(ALCcontext *context)
 {
     std::lock_guard<std::mutex> _{context->mSourceLock};
-    std::for_each(context->mVoices.begin(), context->mVoices.end(),
-        [context](ALvoice &voice) -> void
+    auto voicelist = context->getVoicesSpan();
+    std::for_each(voicelist.begin(), voicelist.end(),
+        [context](ALvoice *voice) -> void
         {
-            ALuint sid{voice.mSourceID.load(std::memory_order_acquire)};
+            ALuint sid{voice->mSourceID.load(std::memory_order_acquire)};
             ALsource *source = sid ? LookupSource(context, sid) : nullptr;
             if(source && !source->PropsClean.test_and_set(std::memory_order_acq_rel))
-                UpdateSourceProps(source, &voice, context);
+                UpdateSourceProps(source, voice, context);
         }
     );
 }
diff --git a/alc/alc.cpp b/alc/alc.cpp
index 46e930ad..bc411a75 100644
--- a/alc/alc.cpp
+++ b/alc/alc.cpp
@@ -1653,12 +1653,44 @@ void ALCcontext::allocVoiceChanges(size_t addcount)
         for(size_t i{1};i < clustersize;++i)
             cluster[i-1].mNext.store(std::addressof(cluster[i]), std::memory_order_relaxed);
         cluster[clustersize-1].mNext.store(mVoiceChangeTail, std::memory_order_relaxed);
-        mVoiceChangeCluster.emplace_back(std::move(cluster));
-        mVoiceChangeTail = mVoiceChangeCluster.back().get();
+        mVoiceChangeClusters.emplace_back(std::move(cluster));
+        mVoiceChangeTail = mVoiceChangeClusters.back().get();
         --addcount;
     }
 }
 
+void ALCcontext::allocVoices(size_t addcount)
+{
+    constexpr size_t clustersize{4};
+    /* Convert element count to cluster count. */
+    addcount = (addcount+(clustersize-1)) / clustersize;
+
+    if(addcount >= std::numeric_limits<int>::max()/clustersize - mVoiceClusters.size())
+        throw std::runtime_error{"Allocating too many voices"};
+
+    auto newarray = ALvoiceArray::Create((mVoiceClusters.size()+addcount) * clustersize);
+    while(addcount)
+    {
+        mVoiceClusters.emplace_back(VoiceCluster{new ALvoice[clustersize]});
+        --addcount;
+    }
+
+    auto voice_iter = newarray->begin();
+    for(VoiceCluster &cluster : mVoiceClusters)
+    {
+        for(size_t i{0};i < clustersize;++i)
+            *(voice_iter++) = &cluster[i];
+    }
+
+    if(auto *oldvoices = mVoices.exchange(newarray.release(), std::memory_order_acq_rel))
+    {
+        ALuint refcount;
+        while((refcount=mDevice->MixCount.load(std::memory_order_acquire))&1)
+            std::this_thread::yield();
+        delete oldvoices;
+    }
+}
+
 
 /* alcSetError
  *
@@ -2277,45 +2309,47 @@ static ALCenum UpdateDeviceParams(ALCdevice *device, const ALCint *attrList)
         {
             const ALuint num_sends{device->NumAuxSends};
             /* Clear extraneous property set sends. */
-            auto clear_sends = [num_sends](ALvoice &voice) -> void
+            auto clear_sends = [num_sends](ALvoice *voice) -> void
             {
-                std::fill(std::begin(voice.mProps.Send)+num_sends, std::end(voice.mProps.Send),
+                std::fill(std::begin(voice->mProps.Send)+num_sends, std::end(voice->mProps.Send),
                     ALvoiceProps::SendData{});
 
-                std::fill(voice.mSend.begin()+num_sends, voice.mSend.end(), ALvoice::TargetData{});
+                std::fill(voice->mSend.begin()+num_sends, voice->mSend.end(),
+                    ALvoice::TargetData{});
                 auto clear_chan_sends = [num_sends](ALvoice::ChannelData &chandata) -> void
                 {
                     std::fill(chandata.mWetParams.begin()+num_sends, chandata.mWetParams.end(),
                         SendParams{});
                 };
-                std::for_each(voice.mChans.begin(), voice.mChans.end(), clear_chan_sends);
+                std::for_each(voice->mChans.begin(), voice->mChans.end(), clear_chan_sends);
             };
-            std::for_each(context->mVoices.begin(), context->mVoices.end(), clear_sends);
+            auto voicelist = context->getVoicesSpan();
+            std::for_each(voicelist.begin(), voicelist.end(), clear_sends);
         }
-        auto reset_voice = [device](ALvoice &voice) -> void
+        auto reset_voice = [device](ALvoice *voice) -> void
         {
-            delete voice.mUpdate.exchange(nullptr, std::memory_order_acq_rel);
+            delete voice->mUpdate.exchange(nullptr, std::memory_order_acq_rel);
 
             /* Force the voice to stopped if it was stopping. */
             ALvoice::State vstate{ALvoice::Stopping};
-            voice.mPlayState.compare_exchange_strong(vstate, ALvoice::Stopped,
+            voice->mPlayState.compare_exchange_strong(vstate, ALvoice::Stopped,
                 std::memory_order_acquire, std::memory_order_acquire);
-            if(voice.mSourceID.load(std::memory_order_relaxed) == 0u)
+            if(voice->mSourceID.load(std::memory_order_relaxed) == 0u)
                 return;
 
-            voice.mStep = 0;
-            voice.mFlags |= VOICE_IS_FADING;
+            voice->mStep = 0;
+            voice->mFlags |= VOICE_IS_FADING;
 
-            if((voice.mFmtChannels == FmtBFormat2D || voice.mFmtChannels == FmtBFormat3D)
-                && device->mAmbiOrder > voice.mAmbiOrder)
+            if((voice->mFmtChannels == FmtBFormat2D || voice->mFmtChannels == FmtBFormat3D)
+                && device->mAmbiOrder > voice->mAmbiOrder)
             {
-                const uint8_t *OrderFromChan{(voice.mFmtChannels == FmtBFormat2D) ?
+                const uint8_t *OrderFromChan{(voice->mFmtChannels == FmtBFormat2D) ?
                     AmbiIndex::OrderFrom2DChannel.data() :
                     AmbiIndex::OrderFromChannel.data()};
 
                 const BandSplitter splitter{400.0f / static_cast<float>(device->Frequency)};
 
-                const auto scales = BFormatDec::GetHFOrderScales(voice.mAmbiOrder,
+                const auto scales = BFormatDec::GetHFOrderScales(voice->mAmbiOrder,
                     device->mAmbiOrder);
                 auto init_ambi = [device,&scales,&OrderFromChan,splitter](ALvoice::ChannelData &chandata) -> void
                 {
@@ -2325,10 +2359,10 @@ static ALCenum UpdateDeviceParams(ALCdevice *device, const ALCint *attrList)
                     chandata.mDryParams = DirectParams{};
                     std::fill_n(chandata.mWetParams.begin(), device->NumAuxSends, SendParams{});
                 };
-                std::for_each(voice.mChans.begin(), voice.mChans.begin()+voice.mNumChannels,
+                std::for_each(voice->mChans.begin(), voice->mChans.begin()+voice->mNumChannels,
                     init_ambi);
 
-                voice.mFlags |= VOICE_IS_AMBISONIC;
+                voice->mFlags |= VOICE_IS_AMBISONIC;
             }
             else
             {
@@ -2339,10 +2373,10 @@ static ALCenum UpdateDeviceParams(ALCdevice *device, const ALCint *attrList)
                     chandata.mDryParams = DirectParams{};
                     std::fill_n(chandata.mWetParams.begin(), device->NumAuxSends, SendParams{});
                 };
-                std::for_each(voice.mChans.begin(), voice.mChans.begin()+voice.mNumChannels,
+                std::for_each(voice->mChans.begin(), voice->mChans.begin()+voice->mNumChannels,
                     clear_prevs);
 
-                voice.mFlags &= ~VOICE_IS_AMBISONIC;
+                voice->mFlags &= ~VOICE_IS_AMBISONIC;
             }
 
             if(device->AvgSpeakerDist > 0.0f)
@@ -2352,11 +2386,12 @@ static ALCenum UpdateDeviceParams(ALCdevice *device, const ALCint *attrList)
                     (device->AvgSpeakerDist * static_cast<float>(device->Frequency))};
                 auto init_nfc = [w1](ALvoice::ChannelData &chandata) -> void
                 { chandata.mDryParams.NFCtrlFilter.init(w1); };
-                std::for_each(voice.mChans.begin(), voice.mChans.begin()+voice.mNumChannels,
+                std::for_each(voice->mChans.begin(), voice->mChans.begin()+voice->mNumChannels,
                     init_nfc);
             }
         };
-        std::for_each(context->mVoices.begin(), context->mVoices.end(), reset_voice);
+        auto voicelist = context->getVoicesSpan();
+        std::for_each(voicelist.begin(), voicelist.end(), reset_voice);
         srclock.unlock();
 
         context->mPropsClean.test_and_set(std::memory_order_release);
@@ -2523,7 +2558,7 @@ ALCcontext::~ALCcontext()
     }
     TRACE("Freed %zu voice property object%s\n", count, (count==1)?"":"s");
 
-    mVoices.clear();
+    delete mVoices.exchange(nullptr, std::memory_order_relaxed);
 
     count = 0;
     ALlistenerProps *lprops{mListener.Params.Update.exchange(nullptr, std::memory_order_relaxed)};
@@ -2611,8 +2646,8 @@ void ALCcontext::init()
     StartEventThrd(this);
 
 
-    mVoices.reserve(256);
-    mVoices.resize(64);
+    allocVoices(256);
+    mActiveVoiceCount.store(64, std::memory_order_relaxed);
 }
 
 bool ALCcontext::deinit()
diff --git a/alc/alcontext.h b/alc/alcontext.h
index 59fda1a7..199f207d 100644
--- a/alc/alcontext.h
+++ b/alc/alcontext.h
@@ -145,7 +145,7 @@ struct ALCcontext : public al::intrusive_ref<ALCcontext> {
      * in clusters that are stored in a vector for easy automatic cleanup.
      */
     using VoiceChangeCluster = std::unique_ptr<VoiceChange[]>;
-    al::vector<VoiceChangeCluster> mVoiceChangeCluster;
+    al::vector<VoiceChangeCluster> mVoiceChangeClusters;
 
     /* The voice change tail is the beginning of the "free" elements, up to and
      * *excluding* the current. If tail==current, there's no free elements and
@@ -157,7 +157,26 @@ struct ALCcontext : public al::intrusive_ref<ALCcontext> {
 
     void allocVoiceChanges(size_t addcount);
 
-    al::vector<ALvoice> mVoices;
+
+    using VoiceCluster = std::unique_ptr<ALvoice[]>;
+    al::vector<VoiceCluster> mVoiceClusters;
+
+    using ALvoiceArray = al::FlexArray<ALvoice*>;
+    std::atomic<ALvoiceArray*> mVoices{};
+    std::atomic<size_t> mActiveVoiceCount{};
+
+    void allocVoices(size_t addcount);
+    al::span<ALvoice*> getVoicesSpan() const noexcept
+    {
+        return {mVoices.load(std::memory_order_relaxed)->data(),
+            mActiveVoiceCount.load(std::memory_order_relaxed)};
+    }
+    al::span<ALvoice*> getVoicesSpanAcquired() const noexcept
+    {
+        return {mVoices.load(std::memory_order_acquire)->data(),
+            mActiveVoiceCount.load(std::memory_order_acquire)};
+    }
+
 
     using ALeffectslotArray = al::FlexArray<ALeffectslot*>;
     std::atomic<ALeffectslotArray*> mActiveAuxSlots{nullptr};
diff --git a/alc/alu.cpp b/alc/alu.cpp
index 9039fb9c..4e554219 100644
--- a/alc/alu.cpp
+++ b/alc/alu.cpp
@@ -1663,7 +1663,7 @@ void ProcessVoiceChanges(ALCcontext *ctx)
 }
 
 void ProcessParamUpdates(ALCcontext *ctx, const ALeffectslotArray &slots,
-    const al::span<ALvoice> voices)
+    const al::span<ALvoice*> voices)
 {
     ProcessVoiceChanges(ctx);
 
@@ -1676,8 +1676,8 @@ void ProcessParamUpdates(ALCcontext *ctx, const ALeffectslotArray &slots,
         for(ALeffectslot *slot : slots)
             force |= CalcEffectSlotParams(slot, sorted_slots, ctx);
 
-        auto calc_params = [ctx,force](ALvoice &voice) -> void
-        { CalcSourceParams(&voice, ctx, force); };
+        auto calc_params = [ctx,force](ALvoice *voice) -> void
+        { CalcSourceParams(voice, ctx, force); };
         std::for_each(voices.begin(), voices.end(), calc_params);
     }
     IncrementRef(ctx->mUpdateCount);
@@ -1688,7 +1688,7 @@ void ProcessContext(ALCcontext *ctx, const ALuint SamplesToDo)
     ASSUME(SamplesToDo > 0);
 
     const ALeffectslotArray &auxslots = *ctx->mActiveAuxSlots.load(std::memory_order_acquire);
-    const al::span<ALvoice> voices{ctx->mVoices.data(), ctx->mVoices.size()};
+    const al::span<ALvoice*> voices{ctx->getVoicesSpanAcquired()};
 
     /* Process pending propery updates for objects on the context. */
     ProcessParamUpdates(ctx, auxslots, voices);
@@ -1702,10 +1702,10 @@ void ProcessContext(ALCcontext *ctx, const ALuint SamplesToDo)
         });
 
     /* Process voices that have a playing source. */
-    auto mix_voice = [SamplesToDo,ctx](ALvoice &voice) -> void
+    auto mix_voice = [SamplesToDo,ctx](ALvoice *voice) -> void
     {
-        const ALvoice::State vstate{voice.mPlayState.load(std::memory_order_acquire)};
-        if(vstate != ALvoice::Stopped) voice.mix(vstate, ctx, SamplesToDo);
+        const ALvoice::State vstate{voice->mPlayState.load(std::memory_order_acquire)};
+        if(vstate != ALvoice::Stopped) voice->mix(vstate, ctx, SamplesToDo);
     };
     std::for_each(voices.begin(), voices.end(), mix_voice);
 
@@ -2097,14 +2097,15 @@ void aluHandleDisconnect(ALCdevice *device, const char *msg, ...)
             }
         }
 
-        auto stop_voice = [](ALvoice &voice) -> void
+        auto voicelist = ctx->getVoicesSpanAcquired();
+        auto stop_voice = [](ALvoice *voice) -> void
         {
-            voice.mCurrentBuffer.store(nullptr, std::memory_order_relaxed);
-            voice.mLoopBuffer.store(nullptr, std::memory_order_relaxed);
-            voice.mSourceID.store(0u, std::memory_order_relaxed);
-            voice.mPlayState.store(ALvoice::Stopped, std::memory_order_release);
+            voice->mCurrentBuffer.store(nullptr, std::memory_order_relaxed);
+            voice->mLoopBuffer.store(nullptr, std::memory_order_relaxed);
+            voice->mSourceID.store(0u, std::memory_order_relaxed);
+            voice->mPlayState.store(ALvoice::Stopped, std::memory_order_release);
         };
-        std::for_each(ctx->mVoices.begin(), ctx->mVoices.end(), stop_voice);
+        std::for_each(voicelist.begin(), voicelist.end(), stop_voice);
     }
     IncrementRef(device->MixCount);
 }
diff --git a/alc/voice.h b/alc/voice.h
index e8daa0c7..555874c4 100644
--- a/alc/voice.h
+++ b/alc/voice.h
@@ -260,54 +260,8 @@ struct ALvoice {
 
     ALvoice() = default;
     ALvoice(const ALvoice&) = delete;
-    ALvoice(ALvoice&& rhs) noexcept { *this = std::move(rhs); }
     ~ALvoice() { delete mUpdate.exchange(nullptr, std::memory_order_acq_rel); }
     ALvoice& operator=(const ALvoice&) = delete;
-    ALvoice& operator=(ALvoice&& rhs) noexcept
-    {
-        ALvoiceProps *old_update{mUpdate.load(std::memory_order_relaxed)};
-        mUpdate.store(rhs.mUpdate.exchange(old_update, std::memory_order_relaxed),
-            std::memory_order_relaxed);
-
-        mProps = rhs.mProps;
-
-        mSourceID.store(rhs.mSourceID.load(std::memory_order_relaxed), std::memory_order_relaxed);
-        mPlayState.store(rhs.mPlayState.load(std::memory_order_relaxed),
-            std::memory_order_relaxed);
-        mPendingStop.store(rhs.mPendingStop.load(std::memory_order_relaxed),
-            std::memory_order_relaxed);
-
-        mPosition.store(rhs.mPosition.load(std::memory_order_relaxed), std::memory_order_relaxed);
-        mPositionFrac.store(rhs.mPositionFrac.load(std::memory_order_relaxed),
-            std::memory_order_relaxed);
-
-        mCurrentBuffer.store(rhs.mCurrentBuffer.load(std::memory_order_relaxed),
-            std::memory_order_relaxed);
-        mLoopBuffer.store(rhs.mLoopBuffer.load(std::memory_order_relaxed),
-            std::memory_order_relaxed);
-
-        mFmtChannels = rhs.mFmtChannels;
-        mFrequency = rhs.mFrequency;
-        mNumChannels = rhs.mNumChannels;
-        mSampleSize = rhs.mSampleSize;
-        mAmbiLayout = rhs.mAmbiLayout;
-        mAmbiScaling = rhs.mAmbiScaling;
-        mAmbiOrder = rhs.mAmbiOrder;
-
-        mStep = rhs.mStep;
-        mResampler = rhs.mResampler;
-
-        mResampleState = rhs.mResampleState;
-
-        mFlags = rhs.mFlags;
-        mNumCallbackSamples = rhs.mNumCallbackSamples;
-
-        mDirect = rhs.mDirect;
-        mSend = rhs.mSend;
-        mChans = rhs.mChans;
-
-        return *this;
-    }
 
     void mix(const State vstate, ALCcontext *Context, const ALuint SamplesToDo);
 };