Fix DSHOT circular DMA

src/main/drivers/light_ws2811strip.c

@@ -301,7 +301,7 @@ void ws2811UpdateStrip(uint16_t usedLedCount)
     ws2811RefillHalf(0);
     ws2811RefillHalf(1);
 
-    impl_timerPWMSetDMACircular(ws2811TCH, true, WS2811_CHUNK_BUFFER_SIZE);
+    impl_timerPWMSetDMACircular(ws2811TCH, true, ledStripDMABuffer, WS2811_CHUNK_BUFFER_SIZE);
 }
 
 void ws2811SetIdleHigh(bool high)

src/main/drivers/pwm_output.c

@@ -31,6 +31,7 @@
 #include "common/circular_queue.h"
 
 #include "drivers/io.h"
+#include "drivers/time.h"
 #include "drivers/timer.h"
 #include "drivers/pwm_mapping.h"
 #include "drivers/pwm_output.h"
@@ -49,9 +50,14 @@
 #define MULTISHOT_20US_MULT (MULTISHOT_TIMER_HZ * 20 / 1000000.0f / 1000.0f)
 
 #ifdef USE_DSHOT
+/* Timer clock per DSHOT rate: DSHOT_MOTOR_BITLENGTH (20) ticks make one bit, so
+ * 12 MHz / 20 = 600 kbit/s. */
 #define MOTOR_DSHOT600_HZ     12000000
 #define MOTOR_DSHOT300_HZ     6000000
 #define MOTOR_DSHOT150_HZ     3000000
+/* Fastest rate INAV supports; static buffers that depend on the bit period (keep-alive
+ * buffer) are sized from it, so update this when a faster rate is added. */
+#define MOTOR_DSHOT_FASTEST_HZ MOTOR_DSHOT600_HZ
 
 
 #define DSHOT_MOTOR_BIT_0       7
@@ -61,6 +67,18 @@
 #define DSHOT_DMA_BUFFER_SIZE   18 /* resolution + frame reset (2us) */
 #define MAX_DMA_TIMERS          8
 
+/* Keep-alive frame replayed by circular DMA while the CPU is stalled by a flash write:
+ * 16 data bits followed by an idle (line low) gap of DSHOT_KEEPALIVE_GAP_US.
+ * One DMA slot is one DSHOT bit period, so the gap needs gapUs * dshotHz / bitLength slots. */
+#define DSHOT_KEEPALIVE_GAP_US      40
+#define DSHOT_KEEPALIVE_SLOTS(dshotHz)  (16 + (DSHOT_KEEPALIVE_GAP_US * ((dshotHz) / 1000000) + DSHOT_MOTOR_BITLENGTH - 1) / DSHOT_MOTOR_BITLENGTH)
+/* The buffer is static, so it is sized for the fastest rate: the shorter the bit period,
+ * the more slots a 40 us gap needs (40 slots at DSHOT600, 28 at DSHOT300, 22 at DSHOT150).
+ * Only DSHOT_KEEPALIVE_SLOTS(actual rate) slots are used at run time. */
+#define DSHOT_KEEPALIVE_BUFFER_SIZE     DSHOT_KEEPALIVE_SLOTS(MOTOR_DSHOT_FASTEST_HZ)
+/* Bound for the waits at the keep-alive transitions: two keep-alive cycles at DSHOT150 */
+#define DSHOT_KEEPALIVE_WAIT_TIMEOUT_US 400
+
 #define DSHOT_COMMAND_DELAY_US 1000
 #define DSHOT_COMMAND_INTERVAL_US 10000
 #define DSHOT_COMMAND_QUEUE_LENGTH 8
@@ -73,6 +91,15 @@ typedef void (*pwmWriteFuncPtr)(uint8_t index, uint16_t value);  // function poi
     timerDMASafeType_t dmaBurstBuffer[MAX_DMA_TIMERS][DSHOT_DMA_BUFFER_SIZE * 4];
 #endif
 
+#ifdef USE_DSHOT
+// Every motor replays the same zero-throttle keep-alive frame, so one buffer feeds all DMA streams
+#ifdef USE_DSHOT_DMAR
+static DMA_RAM timerDMASafeType_t dshotKeepaliveBuffer[DSHOT_KEEPALIVE_BUFFER_SIZE * 4];
+#else
+static DMA_RAM timerDMASafeType_t dshotKeepaliveBuffer[DSHOT_KEEPALIVE_BUFFER_SIZE];
+#endif
+#endif
+
 typedef struct {
     TCH_t * tch;
     bool configured;
@@ -126,6 +153,7 @@ static uint8_t commandsBuff[DHSOT_COMMAND_QUEUE_SIZE];
 static currentExecutingCommand_t currentExecutingCommand;
 
 static uint16_t prepareDshotPacket(const uint16_t value, bool requestTelemetry);
+uint32_t getDshotHz(motorPwmProtocolTypes_e pwmProtocolType);
 #ifndef USE_DSHOT_DMAR
 static void loadDmaBufferDshot(timerDMASafeType_t *dmaBuffer, uint16_t packet);
 #else
@@ -238,6 +266,22 @@ void pwmEnableMotors(void)
     pwmMotorsEnabled = true;
 }
 
+#ifdef USE_DSHOT
+/*
+ * Wait (bounded) until the keep-alive stream of this port is sending its idle padding.
+ * CCR holds the slot last written by DMA: a bit length while a data bit is in flight,
+ * 0 in the padding. Waiting for the data -> padding transition leaves a full
+ * DSHOT_KEEPALIVE_GAP_US before the DMA wraps to the next frame, so the stream can be
+ * stopped with the last frame complete and the line low.
+ */
+static void dshotWaitForKeepalivePadding(const pwmOutputPort_t *port)
+{
+    const timeUs_t start = micros();
+    while (*port->ccr == 0 && (micros() - start) < DSHOT_KEEPALIVE_WAIT_TIMEOUT_US);
+    while (*port->ccr != 0 && (micros() - start) < DSHOT_KEEPALIVE_WAIT_TIMEOUT_US);
+}
+#endif
+
 void pwmSetMotorDMACircular(bool circular)
 {
 #ifdef USE_DSHOT
@@ -246,20 +290,28 @@ void pwmSetMotorDMACircular(bool circular)
     }
 
     int motorCount = getMotorCount();
+    const uint32_t dshotHz = getDshotHz(initMotorProtocol);
+    const uint32_t keepaliveSlots = DSHOT_KEEPALIVE_SLOTS(dshotHz);
 
     if (circular) {
-        // Load zero-throttle packets directly into DMA buffers,
-        // bypassing the rate limiter in pwmCompleteMotorUpdate()
+        // A frame started by pwmCompleteMotorUpdate() may still be in flight: let it finish
+        // and keep the line low for one full gap before the keep-alive stream starts
+        delayMicroseconds(DSHOT_DMA_BUFFER_SIZE * DSHOT_MOTOR_BITLENGTH * 1000000UL / dshotHz + DSHOT_KEEPALIVE_GAP_US);
+
+        // Load a zero-throttle packet into the shared keep-alive buffer. The padding slots
+        // must be zero (line low between frames); DMA_RAM is NOLOAD and not cleared at
+        // startup, so clear it explicitly.
         uint16_t packet = prepareDshotPacket(0, false);
+        ZERO_FARRAY(dshotKeepaliveBuffer);
+#ifdef USE_DSHOT_DMAR
         for (int i = 0; i < motorCount; i++) {
             if (motors[i].pwmPort && motors[i].pwmPort->configured) {
-#ifdef USE_DSHOT_DMAR
-                loadDmaBufferDshotStride(&motors[i].pwmPort->dmaBurstBuffer[motors[i].pwmPort->tch->timHw->channelIndex], 4, packet);
-#else
-                loadDmaBufferDshot(motors[i].pwmPort->dmaBuffer, packet);
-#endif
+                loadDmaBufferDshotStride(&dshotKeepaliveBuffer[motors[i].pwmPort->tch->timHw->channelIndex], 4, packet);
             }
         }
+#else
+        loadDmaBufferDshot(dshotKeepaliveBuffer, packet);
+#endif
     }
 
 #ifdef USE_DSHOT_DMAR
@@ -270,7 +322,12 @@ void pwmSetMotorDMACircular(bool circular)
         for (int m = 0; m < motorCount; m++) {
             if (motors[m].pwmPort && motors[m].pwmPort->configured && motors[m].pwmPort->tch
                 && motors[m].pwmPort->tch->timHw->tim == burstDmaTimer->timer) {
-                impl_pwmBurstDMASetCircular(burstDmaTimer, motors[m].pwmPort->tch, circular, DSHOT_DMA_BUFFER_SIZE * 4);
+                if (circular) {
+                    impl_pwmBurstDMASetCircular(burstDmaTimer, motors[m].pwmPort->tch, true, dshotKeepaliveBuffer, keepaliveSlots * 4);
+                } else {
+                    dshotWaitForKeepalivePadding(motors[m].pwmPort);
+                    impl_pwmBurstDMASetCircular(burstDmaTimer, motors[m].pwmPort->tch, false, burstDmaTimer->dmaBurstBuffer, DSHOT_DMA_BUFFER_SIZE * 4);
+                }
                 break;
             }
         }
@@ -279,10 +336,26 @@ void pwmSetMotorDMACircular(bool circular)
     // Per-channel DMA: one DMA stream per motor
     for (int i = 0; i < motorCount; i++) {
         if (motors[i].pwmPort && motors[i].pwmPort->configured && motors[i].pwmPort->tch) {
-            impl_timerPWMSetDMACircular(motors[i].pwmPort->tch, circular, DSHOT_DMA_BUFFER_SIZE);
+            if (circular) {
+                impl_timerPWMSetDMACircular(motors[i].pwmPort->tch, true, dshotKeepaliveBuffer, keepaliveSlots);
+            } else {
+                dshotWaitForKeepalivePadding(motors[i].pwmPort);
+                impl_timerPWMSetDMACircular(motors[i].pwmPort->tch, false, motors[i].pwmPort->dmaBuffer, DSHOT_DMA_BUFFER_SIZE);
+            }
         }
     }
 #endif
+
+    if (!circular) {
+        // The streams were stopped while sending padding, so CCR is already 0 and the lines
+        // stay low until pwmCompleteMotorUpdate() sends the next frame. Enforce it in case a
+        // wait above timed out; the timer would otherwise keep repeating the last bit.
+        for (int i = 0; i < motorCount; i++) {
+            if (motors[i].pwmPort && motors[i].pwmPort->configured) {
+                *motors[i].pwmPort->ccr = 0;
+            }
+        }
+    }
 #else
     UNUSED(circular);
 #endif

src/main/drivers/timer_impl.h

@@ -84,11 +84,11 @@ bool impl_timerPWMConfigChannelDMA(TCH_t * tch, void * dmaBuffer, uint8_t dmaBuf
 void impl_timerPWMPrepareDMA(TCH_t * tch, uint32_t dmaBufferElementCount);
 void impl_timerPWMStartDMA(TCH_t * tch);
 void impl_timerPWMStopDMA(TCH_t * tch);
-void impl_timerPWMSetDMACircular(TCH_t * tch, bool circular, uint32_t dmaBufferSize);
+void impl_timerPWMSetDMACircular(TCH_t * tch, bool circular, void * dmaBuffer, uint32_t dmaBufferSize);
 void impl_timerPWMSetDMARefillCallback(TCH_t * tch, timerDmaRefillFn * callback);
 
 #ifdef USE_DSHOT_DMAR
 bool impl_timerPWMConfigDMABurst(burstDmaTimer_t *burstDmaTimer, TCH_t * tch, void * dmaBuffer, uint8_t dmaBufferElementSize, uint32_t dmaBufferElementCount);
 void impl_pwmBurstDMAStart(burstDmaTimer_t * burstDmaTimer, uint32_t BurstLength);
-void impl_pwmBurstDMASetCircular(burstDmaTimer_t * burstDmaTimer, TCH_t * tch, bool circular, uint32_t dmaBufferSize);
+void impl_pwmBurstDMASetCircular(burstDmaTimer_t * burstDmaTimer, TCH_t * tch, bool circular, void * dmaBuffer, uint32_t dmaBufferSize);
 #endif

src/main/drivers/timer_impl_hal.c

@@ -540,7 +540,7 @@ void impl_pwmBurstDMAStart(burstDmaTimer_t * burstDmaTimer, uint32_t BurstLength
     LL_TIM_EnableDMAReq_CCx(burstDmaTimer->timer, burstDmaTimer->burstRequestSource);
 }
 
-void impl_pwmBurstDMASetCircular(burstDmaTimer_t * burstDmaTimer, TCH_t * tch, bool circular, uint32_t dmaBufferSize)
+void impl_pwmBurstDMASetCircular(burstDmaTimer_t * burstDmaTimer, TCH_t * tch, bool circular, void * dmaBuffer, uint32_t dmaBufferSize)
 {
     if (!tch->dma || !tch->dma->dma) {
         return;
@@ -564,7 +564,6 @@ void impl_pwmBurstDMASetCircular(burstDmaTimer_t * burstDmaTimer, TCH_t * tch, b
 
         if (circular) {
             LL_DMA_SetMode(burstDmaTimer->dma, burstDmaTimer->streamLL, LL_DMA_MODE_CIRCULAR);
-            LL_DMA_SetDataLength(burstDmaTimer->dma, burstDmaTimer->streamLL, dmaBufferSize);
             LL_DMA_DisableIT_TC(burstDmaTimer->dma, burstDmaTimer->streamLL);
             tch->dmaState = TCH_DMA_CIRCULAR;
         } else {
@@ -573,10 +572,18 @@ void impl_pwmBurstDMASetCircular(burstDmaTimer_t * burstDmaTimer, TCH_t * tch, b
             tch->dmaState = TCH_DMA_IDLE;
         }
 
+        // M0AR/NDTR writes are only accepted while EN = 0 (checked above)
+        LL_DMA_SetMemoryAddress(burstDmaTimer->dma, burstDmaTimer->streamLL, (uint32_t)dmaBuffer);
+        LL_DMA_SetDataLength(burstDmaTimer->dma, burstDmaTimer->streamLL, dmaBufferSize);
+
         __DSB();
 
-        LL_DMA_EnableStream(burstDmaTimer->dma, burstDmaTimer->streamLL);
-        LL_TIM_EnableDMAReq_CCx(burstDmaTimer->timer, burstDmaTimer->burstRequestSource);
+        // Normal mode: leave the stream stopped, as after a completed frame;
+        // the next frame is started by impl_pwmBurstDMAStart()
+        if (circular) {
+            LL_DMA_EnableStream(burstDmaTimer->dma, burstDmaTimer->streamLL);
+            LL_TIM_EnableDMAReq_CCx(burstDmaTimer->timer, burstDmaTimer->burstRequestSource);
+        }
     }
 }
 #endif
@@ -669,7 +676,7 @@ void impl_timerPWMStopDMA(TCH_t * tch)
     HAL_TIM_Base_Start(tch->timCtx->timHandle);
 }
 
-void impl_timerPWMSetDMACircular(TCH_t * tch, bool circular, uint32_t dmaBufferSize)
+void impl_timerPWMSetDMACircular(TCH_t * tch, bool circular, void * dmaBuffer, uint32_t dmaBufferSize)
 {
     if (!tch->dma || !tch->dma->dma) {
         return;
@@ -698,8 +705,6 @@ void impl_timerPWMSetDMACircular(TCH_t * tch, bool circular, uint32_t dmaBufferS
 
         if (circular) {
             LL_DMA_SetMode(dmaBase, streamLL, LL_DMA_MODE_CIRCULAR);
-            // Circular mode requires non-zero NDTR (STM32H7 RM constraint)
-            LL_DMA_SetDataLength(dmaBase, streamLL, dmaBufferSize);
             if (tch->dmaRefillCallback) {
                 // Refill consumer needs an IRQ every half-cycle to keep the
                 // buffer fed
@@ -718,10 +723,19 @@ void impl_timerPWMSetDMACircular(TCH_t * tch, bool circular, uint32_t dmaBufferS
             tch->dmaState = TCH_DMA_IDLE;
         }
 
+        // M0AR/NDTR writes are only accepted while EN = 0 (checked above);
+        // circular mode requires non-zero NDTR (STM32H7 RM constraint)
+        LL_DMA_SetMemoryAddress(dmaBase, streamLL, (uint32_t)dmaBuffer);
+        LL_DMA_SetDataLength(dmaBase, streamLL, dmaBufferSize);
+
         // Ensure register writes are visible to DMA before re-enabling
         __DSB();
 
-        LL_DMA_EnableStream(dmaBase, streamLL);
-        LL_TIM_EnableDMAReq_CCx(tch->timHw->tim, lookupDMASourceTable[tch->timHw->channelIndex]);
+        // Normal mode: leave the stream stopped, as after a completed frame; the next
+        // frame is started by impl_timerPWMPrepareDMA()/impl_timerPWMStartDMA()
+        if (circular) {
+            LL_DMA_EnableStream(dmaBase, streamLL);
+            LL_TIM_EnableDMAReq_CCx(tch->timHw->tim, lookupDMASourceTable[tch->timHw->channelIndex]);
+        }
     }
 }

src/main/drivers/timer_impl_stdperiph.c

@@ -492,7 +492,7 @@ void impl_pwmBurstDMAStart(burstDmaTimer_t * burstDmaTimer, uint32_t BurstLength
     TIM_DMACmd(burstDmaTimer->timer, burstDmaTimer->burstRequestSource, ENABLE);
 }
 
-void impl_pwmBurstDMASetCircular(burstDmaTimer_t * burstDmaTimer, TCH_t * tch, bool circular, uint32_t dmaBufferSize)
+void impl_pwmBurstDMASetCircular(burstDmaTimer_t * burstDmaTimer, TCH_t * tch, bool circular, void * dmaBuffer, uint32_t dmaBufferSize)
 {
     if (!tch->dma || !tch->dma->ref) {
         return;
@@ -516,7 +516,6 @@ void impl_pwmBurstDMASetCircular(burstDmaTimer_t * burstDmaTimer, TCH_t * tch, b
 
         if (circular) {
             burstDmaTimer->dmaBurstStream->CR |= DMA_SxCR_CIRC;
-            DMA_SetCurrDataCounter(burstDmaTimer->dmaBurstStream, dmaBufferSize);
             DMA_ITConfig(burstDmaTimer->dmaBurstStream, DMA_IT_TC, DISABLE);
             tch->dmaState = TCH_DMA_CIRCULAR;
         } else {
@@ -525,10 +524,18 @@ void impl_pwmBurstDMASetCircular(burstDmaTimer_t * burstDmaTimer, TCH_t * tch, b
             tch->dmaState = TCH_DMA_IDLE;
         }
 
+        // M0AR/NDTR writes are only accepted while EN = 0 (checked above)
+        burstDmaTimer->dmaBurstStream->M0AR = (uint32_t)dmaBuffer;
+        DMA_SetCurrDataCounter(burstDmaTimer->dmaBurstStream, dmaBufferSize);
+
         __DSB();
 
-        DMA_Cmd(burstDmaTimer->dmaBurstStream, ENABLE);
-        TIM_DMACmd(burstDmaTimer->timer, burstDmaTimer->burstRequestSource, ENABLE);
+        // Normal mode: leave the stream stopped, as after a completed frame;
+        // the next frame is started by impl_pwmBurstDMAStart()
+        if (circular) {
+            DMA_Cmd(burstDmaTimer->dmaBurstStream, ENABLE);
+            TIM_DMACmd(burstDmaTimer->timer, burstDmaTimer->burstRequestSource, ENABLE);
+        }
     }
 }
 #endif
@@ -595,7 +602,7 @@ void impl_timerPWMStopDMA(TCH_t * tch)
     TIM_Cmd(tch->timHw->tim, ENABLE);
 }
 
-void impl_timerPWMSetDMACircular(TCH_t * tch, bool circular, uint32_t dmaBufferSize)
+void impl_timerPWMSetDMACircular(TCH_t * tch, bool circular, void * dmaBuffer, uint32_t dmaBufferSize)
 {
     if (!tch->dma || !tch->dma->ref) {
         return;
@@ -621,7 +628,6 @@ void impl_timerPWMSetDMACircular(TCH_t * tch, bool circular, uint32_t dmaBufferS
 
         if (circular) {
             tch->dma->ref->CR |= DMA_SxCR_CIRC;
-            DMA_SetCurrDataCounter(tch->dma->ref, dmaBufferSize);
             if (tch->dmaRefillCallback) {
                 // Refill consumer needs an IRQ every half-cycle to keep the
                 // buffer fed
@@ -639,10 +645,18 @@ void impl_timerPWMSetDMACircular(TCH_t * tch, bool circular, uint32_t dmaBufferS
             tch->dmaState = TCH_DMA_IDLE;
         }
 
+        // M0AR/NDTR writes are only accepted while EN = 0 (checked above)
+        tch->dma->ref->M0AR = (uint32_t)dmaBuffer;
+        DMA_SetCurrDataCounter(tch->dma->ref, dmaBufferSize);
+
         // Ensure register writes are visible to DMA before re-enabling
         __DSB();
 
-        DMA_Cmd(tch->dma->ref, ENABLE);
-        TIM_DMACmd(tch->timHw->tim, lookupDMASourceTable[tch->timHw->channelIndex], ENABLE);
+        // Normal mode: leave the stream stopped, as after a completed frame; the next
+        // frame is started by impl_timerPWMPrepareDMA()/impl_timerPWMStartDMA()
+        if (circular) {
+            DMA_Cmd(tch->dma->ref, ENABLE);
+            TIM_DMACmd(tch->timHw->tim, lookupDMASourceTable[tch->timHw->channelIndex], ENABLE);
+        }
     }
 }

src/main/drivers/timer_impl_stdperiph_at32.c

@@ -443,7 +443,7 @@ void impl_timerPWMStopDMA(TCH_t * tch)
     tmr_counter_enable(tch->timHw->tim, TRUE);
 }
 
-void impl_timerPWMSetDMACircular(TCH_t * tch, bool circular, uint32_t dmaBufferSize)
+void impl_timerPWMSetDMACircular(TCH_t * tch, bool circular, void * dmaBuffer, uint32_t dmaBufferSize)
 {
     if (!tch->dma || !tch->dma->ref) {
         return;
@@ -469,7 +469,6 @@ void impl_timerPWMSetDMACircular(TCH_t * tch, bool circular, uint32_t dmaBufferS
 
         if (circular) {
             tch->dma->ref->ctrl_bit.lm = TRUE;
-            dma_data_number_set(tch->dma->ref, dmaBufferSize);
             if (tch->dmaRefillCallback) {
                 // Refill consumer needs an IRQ every half-cycle to keep the
                 // buffer fed
@@ -488,10 +487,18 @@ void impl_timerPWMSetDMACircular(TCH_t * tch, bool circular, uint32_t dmaBufferS
             tch->dmaState = TCH_DMA_IDLE;
         }
 
+        // Memory address / data count are only writable while the channel is disabled (checked above)
+        tch->dma->ref->maddr = (uint32_t)dmaBuffer;
+        dma_data_number_set(tch->dma->ref, dmaBufferSize);
+
         // Ensure register writes are visible to DMA before re-enabling
         __DSB();
 
-        dma_channel_enable(tch->dma->ref, TRUE);
-        tmr_dma_request_enable(tch->timHw->tim, lookupDMASourceTable[tch->timHw->channelIndex], TRUE);
+        // Normal mode: leave the channel stopped, as after a completed frame; the next
+        // frame is started by impl_timerPWMPrepareDMA()/impl_timerPWMStartDMA()
+        if (circular) {
+            dma_channel_enable(tch->dma->ref, TRUE);
+            tmr_dma_request_enable(tch->timHw->tim, lookupDMASourceTable[tch->timHw->channelIndex], TRUE);
+        }
     }
 }