Skip to content

Audio: MFCC: Add Voice Activity Detection based on Mel spectrum #10782

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lgirdwood merged 4 commits into from
May 28, 2026
Merged

Audio: MFCC: Add Voice Activity Detection based on Mel spectrum #10782

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d495cc1
Audio: MFCC: Add Voice Activity Detection based on Mel spectrum
singalsu May 12, 2026
1c5fb79
Audio: MFCC: Add Python script for speech to text with Whisper model
singalsu May 19, 2026
5537875
Audio: MFCC: Add VAD switch control notification to user space
singalsu May 19, 2026
2fa783a
Audio: MFCC: Fix memory leak on reset-prepare cycles
singalsu May 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/audio/mfcc/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,8 @@ if(CONFIG_COMP_MFCC STREQUAL "m" AND DEFINED CONFIG_LLEXT)
add_subdirectory(llext ${PROJECT_BINARY_DIR}/mfcc_llext)
add_dependencies(app mfcc)
else()
add_local_sources(sof mfcc.c mfcc_setup.c mfcc_common.c mfcc_generic.c mfcc_hifi4.c mfcc_hifi3.c)
add_local_sources(sof mfcc.c mfcc_setup.c mfcc_common.c mfcc_generic.c mfcc_hifi4.c mfcc_hifi3.c mfcc_vad.c)
if(CONFIG_IPC_MAJOR_4)
add_local_sources(sof mfcc_ipc4.c)
endif()
Comment thread
singalsu marked this conversation as resolved.
endif()
56 changes: 25 additions & 31 deletions src/audio/mfcc/mfcc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,36 +97,14 @@ static int mfcc_free(struct processing_module *mod)
struct mfcc_comp_data *cd = module_get_private_data(mod);

comp_info(mod->dev, "entry");
ipc_msg_free(cd->msg);
cd->msg = NULL;
mod_data_blob_handler_free(mod, cd->model_handler);
mfcc_free_buffers(mod);
mod_free(mod, cd);
return 0;
}

static int mfcc_get_config(struct processing_module *mod,
uint32_t config_id, uint32_t *data_offset_size,
uint8_t *fragment, size_t fragment_size)
{
struct sof_ipc_ctrl_data *cdata = (struct sof_ipc_ctrl_data *)fragment;
struct mfcc_comp_data *cd = module_get_private_data(mod);

comp_info(mod->dev, "entry");

return comp_data_blob_get_cmd(cd->model_handler, cdata, fragment_size);
}

static int mfcc_set_config(struct processing_module *mod, uint32_t config_id,
enum module_cfg_fragment_position pos, uint32_t data_offset_size,
const uint8_t *fragment, size_t fragment_size, uint8_t *response,
size_t response_size)
{
struct mfcc_comp_data *cd = module_get_private_data(mod);

comp_info(mod->dev, "entry");

return comp_data_blob_set(cd->model_handler, pos, data_offset_size,
fragment, fragment_size);
}

static int mfcc_process(struct processing_module *mod,
struct input_stream_buffer *input_buffers, int num_input_buffers,
Expand Down Expand Up @@ -187,22 +165,33 @@ static int mfcc_prepare(struct processing_module *mod,
audio_stream_get_channels(&sourceb->stream));
if (ret < 0) {
comp_err(dev, "setup failed.");
goto err;
return ret;
}
} else {
comp_err(dev, "configuration is missing.");
return -EINVAL;
}

cd->mfcc_func = mfcc_find_func(source_format, sink_format, mfcc_fm, ARRAY_SIZE(mfcc_fm));
if (!cd->mfcc_func) {
comp_err(dev, "No proc func");
ret = -EINVAL;
goto err;
mfcc_free_buffers(mod);
return -EINVAL;
}
Comment thread
singalsu marked this conversation as resolved.

return 0;
/* Initialize VAD switch control notification if enabled */
if (cd->config->enable_vad && cd->config->update_controls) {
if (!cd->msg) {
ret = mfcc_ipc_notification_init(mod);
if (ret < 0) {
mfcc_free_buffers(mod);
return ret;
}
}
}
Comment thread
singalsu marked this conversation as resolved.

err:
comp_set_state(dev, COMP_TRIGGER_RESET);
return ret;
cd->vad_prev = false;
return 0;
}

static int mfcc_reset(struct processing_module *mod)
Expand All @@ -211,6 +200,11 @@ static int mfcc_reset(struct processing_module *mod)

comp_info(mod->dev, "entry");

/* Free MFCC buffers to prevent leaks on reset->prepare cycles.
* mfcc_free_buffers() NULLs the pointers after free.
*/
mfcc_free_buffers(mod);

/* Reset to similar state as init() */
cd->mfcc_func = NULL;
return 0;
Expand Down
110 changes: 79 additions & 31 deletions src/audio/mfcc/mfcc_common.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,17 @@
#include <stddef.h>
#include <stdint.h>

#include <sof/audio/mfcc/mfcc_vad.h>

LOG_MODULE_REGISTER(mfcc_common, CONFIG_SOF_LOG_LEVEL);

/*
* The main processing function for MFCC
*/

static int mfcc_stft_process(const struct comp_dev *dev, struct mfcc_comp_data *cd)
static int mfcc_stft_process(struct processing_module *mod, struct mfcc_comp_data *cd)
{
const struct comp_dev *dev = mod->dev;
struct sof_mfcc_config *config = cd->config;
struct mfcc_state *state = &cd->state;
struct mfcc_buffer *buf = &state->buf;
Expand Down Expand Up @@ -169,6 +172,32 @@ static int mfcc_stft_process(const struct comp_dev *dev, struct mfcc_comp_data *

cc_count += state->dct.num_out;
}

/* Use hop counter for frame numbering (independent of VAD enable) */
state->header.frame_number = state->hop_count;

/* Run VAD on the mel log spectrum (available in both modes) */
if (config->enable_vad) {
mfcc_vad_update(&cd->vad, state->mel_log_32);

/* Populate data header for this output frame */
state->header.energy = cd->vad.energy;
state->header.noise_energy = cd->vad.noise_energy;
state->header.vad_flag = cd->vad.is_speech ? 1 : 0;
}

/* Increment hop counter at end of hop processing */
state->hop_count++;

/* Send notification when VAD state changes */
if (config->enable_vad && config->update_controls) {
bool vad_now = cd->vad.is_speech;

if (vad_now != cd->vad_prev) {
mfcc_send_vad_notification(mod, vad_now ? 1 : 0);
cd->vad_prev = vad_now;
}
}
}

return cc_count;
Expand Down Expand Up @@ -267,9 +296,8 @@ void mfcc_s16_default(struct processing_module *mod, struct input_stream_buffer
struct mfcc_comp_data *cd = module_get_private_data(mod);
struct mfcc_state *state = &cd->state;
struct mfcc_buffer *buf = &cd->state.buf;
uint32_t magic = MFCC_MAGIC;
int16_t *w_ptr = audio_stream_get_wptr(sink);
const int num_magic = 2;
const int num_header_s16 = sizeof(state->header) / sizeof(int16_t);
Copy link
Copy Markdown
Collaborator

@lyakh lyakh May 26, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is the division guaranteed to be exact or you do want rounding-down?

Copy link
Copy Markdown
Collaborator Author

@singalsu singalsu May 28, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is guaranteed manually with design of the struct. This part of code changes in next PR to MFCC #10814 with (more) final version of output code, so better to put review effort there.

int num_ceps;
int sink_samples;
int to_copy;
Expand All @@ -278,27 +306,35 @@ void mfcc_s16_default(struct processing_module *mod, struct input_stream_buffer
mfcc_source_copy_s16(bsource, buf, &state->emph, frames, state->source_channel);

/* Run STFT and processing after FFT: Mel auditory filter and DCT. */
num_ceps = mfcc_stft_process(mod->dev, cd);
num_ceps = mfcc_stft_process(mod, cd);

/* If new output produced, set up pointer into scratch data and mark magic pending */
/* If new output produced, set up pointer into scratch data and mark header pending */
if (num_ceps > 0) {
if (state->mel_only)
if (state->mel_only) {
state->out_data_ptr = state->mel_spectra->data;
else
} else {
state->out_data_ptr = state->cepstral_coef->data;
}

state->out_remain = num_ceps;
state->magic_pending = true;
state->header_pending = true;
}

/* Write to sink, limited by period size */
sink_samples = frames * audio_stream_get_channels(sink);

/* Write magic word first if pending */
if (state->magic_pending && sink_samples >= num_magic) {
w_ptr = mfcc_sink_copy_data_s16(sink, w_ptr, num_magic, (int16_t *)&magic);
sink_samples -= num_magic;
state->magic_pending = false;
/* Write data header first if pending */
if (state->header_pending) {
if (sink_samples < num_header_s16) {
/* Not enough sink space for header, defer entire frame */
mfcc_sink_copy_zero_s16(sink, w_ptr, sink_samples);
return;
}

w_ptr = mfcc_sink_copy_data_s16(sink, w_ptr, num_header_s16,
(int16_t *)&state->header);
sink_samples -= num_header_s16;
state->header_pending = false;
Comment thread
singalsu marked this conversation as resolved.
Comment thread
singalsu marked this conversation as resolved.
}

/* Write cepstral/mel data from scratch buffer */
Expand Down Expand Up @@ -363,9 +399,8 @@ void mfcc_s24_default(struct processing_module *mod, struct input_stream_buffer
struct mfcc_comp_data *cd = module_get_private_data(mod);
struct mfcc_state *state = &cd->state;
struct mfcc_buffer *buf = &cd->state.buf;
uint32_t magic = MFCC_MAGIC;
int32_t *w_ptr = audio_stream_get_wptr(sink);
const int num_magic = 1; /* one int32_t word for magic */
const int num_header_s32 = sizeof(state->header) / sizeof(int32_t);
Copy link
Copy Markdown
Collaborator

@lyakh lyakh May 26, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same about rounding

Copy link
Copy Markdown
Collaborator Author

@singalsu singalsu May 28, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, also by design a multiple of int32_t size.

int num_ceps;
int sink_samples;
int remain_s32;
Expand All @@ -376,7 +411,7 @@ void mfcc_s24_default(struct processing_module *mod, struct input_stream_buffer
mfcc_source_copy_s24(bsource, buf, &state->emph, frames, state->source_channel);

/* Run STFT and processing after FFT */
num_ceps = mfcc_stft_process(mod->dev, cd);
num_ceps = mfcc_stft_process(mod, cd);

/* If new output produced, set up pointer into scratch data */
if (num_ceps > 0) {
Expand All @@ -391,17 +426,24 @@ void mfcc_s24_default(struct processing_module *mod, struct input_stream_buffer
}

state->out_remain = num_ceps;
state->magic_pending = true;
state->header_pending = true;
}

/* Write to sink, limited by period size */
sink_samples = frames * audio_stream_get_channels(sink);

/* Write magic word first if pending */
if (state->magic_pending && sink_samples >= num_magic) {
w_ptr = mfcc_sink_copy_data_s32(sink, w_ptr, num_magic, (int32_t *)&magic);
sink_samples -= num_magic;
state->magic_pending = false;
/* Write data header first if pending */
if (state->header_pending) {
if (sink_samples < num_header_s32) {
/* Not enough sink space for header, defer entire frame */
mfcc_sink_copy_zero_s32(sink, w_ptr, sink_samples);
return;
}

w_ptr = mfcc_sink_copy_data_s32(sink, w_ptr, num_header_s32,
(int32_t *)&state->header);
Copy link
Copy Markdown
Collaborator

@lyakh lyakh May 26, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it looks like we expect the header size to be a multiple of 4 bytes, so maybe we could add a build-assertion and comments here to make it clear. Can be a follow-up, just checking that this is indeed the case

Copy link
Copy Markdown
Member

@lgirdwood lgirdwood May 26, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no assertions, return an error.

Copy link
Copy Markdown
Collaborator

@lyakh lyakh May 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@lgirdwood a build assertion, not a runtime one. These sizes are known at build time, so a build-time check can be added with no run-time cost

Copy link
Copy Markdown
Member

@lgirdwood lgirdwood May 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, build is fine.

Copy link
Copy Markdown
Collaborator Author

@singalsu singalsu May 28, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Tips how to add build time check and error for struct size (multiple of something) is welcome. I will still after #10814 try to come up with more generic audio feature stream for bespoke encoder with more header layers, e.g. with use of https://github.com/thesofproject/sof/blob/main/src/include/user/audio_feature.h . Now I'm focusing to make the things work well and to show improvement in power measurement vs. SOF PCM stream and things done in host user space.

sink_samples -= num_header_s32;
state->header_pending = false;
}

if (state->mel_only) {
Expand Down Expand Up @@ -443,9 +485,8 @@ void mfcc_s32_default(struct processing_module *mod, struct input_stream_buffer
struct mfcc_comp_data *cd = module_get_private_data(mod);
struct mfcc_state *state = &cd->state;
struct mfcc_buffer *buf = &cd->state.buf;
uint32_t magic = MFCC_MAGIC;
int32_t *w_ptr = audio_stream_get_wptr(sink);
const int num_magic = 1; /* one int32_t word for magic */
const int num_header_s32 = sizeof(state->header) / sizeof(int32_t);
int num_ceps;
int sink_samples;
int remain_s32;
Expand All @@ -455,7 +496,7 @@ void mfcc_s32_default(struct processing_module *mod, struct input_stream_buffer
mfcc_source_copy_s32(bsource, buf, &state->emph, frames, state->source_channel);

/* Run STFT and processing after FFT */
num_ceps = mfcc_stft_process(mod->dev, cd);
num_ceps = mfcc_stft_process(mod, cd);

/* If new output produced, set up pointer into scratch data */
if (num_ceps > 0) {
Expand All @@ -466,17 +507,24 @@ void mfcc_s32_default(struct processing_module *mod, struct input_stream_buffer
}

state->out_remain = num_ceps;
state->magic_pending = true;
state->header_pending = true;
}

/* Write to sink, limited by period size */
sink_samples = frames * audio_stream_get_channels(sink);

/* Write magic word first if pending */
if (state->magic_pending && sink_samples >= num_magic) {
w_ptr = mfcc_sink_copy_data_s32(sink, w_ptr, num_magic, (int32_t *)&magic);
sink_samples -= num_magic;
state->magic_pending = false;
/* Write data header first if pending */
if (state->header_pending) {
if (sink_samples < num_header_s32) {
/* Not enough sink space for header, defer entire frame */
mfcc_sink_copy_zero_s32(sink, w_ptr, sink_samples);
return;
}

w_ptr = mfcc_sink_copy_data_s32(sink, w_ptr, num_header_s32,
(int32_t *)&state->header);
sink_samples -= num_header_s32;
state->header_pending = false;
}

if (state->mel_only) {
Expand Down
Loading
Loading