diff --git a/src/cubeb_resampler_internal.h b/src/cubeb_resampler_internal.h
index 08e019c610..dfc93f15b4 100644
--- a/src/cubeb_resampler_internal.h
+++ b/src/cubeb_resampler_internal.h
@@ -211,9 +211,11 @@
     const size_t LATENCY_SAMPLES = 8192;
     T input_buffer[LATENCY_SAMPLES] = {};
     T output_buffer[LATENCY_SAMPLES] = {};
-    uint32_t input_frame_count = input_latency;
-    uint32_t output_frame_count = LATENCY_SAMPLES;
-    assert(input_latency * channels <= LATENCY_SAMPLES);
+    const uint32_t latency_frames =
+        LATENCY_SAMPLES / std::max<uint32_t>(channels, 1);
+    uint32_t input_frame_count = std::min(input_latency, latency_frames);
+    uint32_t output_frame_count = latency_frames;
+    assert(output_frame_count * channels <= LATENCY_SAMPLES);
     speex_resample(input_buffer, &input_frame_count, output_buffer,
                    &output_frame_count);
   }