WIP: recover audio stream after buffer overrun (c2d76746) · Commits · Elphel / elphel-apps-camogm

src/camogm.c

+5 −0

Original line number	Diff line number	Diff line
		@@ -1769,6 +1769,11 @@ int listener_loop(camogm_state *state)
		} else {
		D6(fprintf(debug_file, "not recording audio samples after this frame\n"));
		}

		// === debug code ===
		fprintf(debug_file, "video frames recorded: %d\n", state->frameno);
		// ===end of debug ===

		audio_process(&state->audio);
		}
		}

src/camogm_audio.c

+160 −23

Original line number	Diff line number	Diff line
		@@ -40,6 +40,10 @@ static void audio_deinit(struct audio *audio);
		static bool skip_audio(struct audio *audio, snd_pcm_uframes_t frames);
		static long frames_to_bytes(const struct audio *audio, long frames);
		static void record_buffer(struct audio *audio, int opt);
		static void recover_stream(struct audio *audio, snd_pcm_sframes_t err, snd_pcm_uframes_t xrun);
		static void dummy_read(struct audio *audio);
		static void write_silence(struct audio *audio);
		static int realloc_buffers(struct context_audio *ctx);

		/**
		* Initialize HW part of audio interface.
		@@ -114,12 +118,10 @@ void audio_init_hw(struct audio *audio, bool restart)
		break;
		}
		if (init_ok) {
		char tmp_buff[32];
		snd_pcm_prepare(audio->ctx_a.capture_hnd);
		snd_pcm_reset(audio->ctx_a.capture_hnd);
		audio_set_volume(audio->audio_volume);
		// read some frames to force the driver to start reporting correct number of available frames
		snd_pcm_readi(audio->ctx_a.capture_hnd, tmp_buff, 8);
		dummy_read(audio);
		snd_pcm_status_alloca(&status);
		snd_pcm_status(audio->ctx_a.capture_hnd, status);
		snd_pcm_status_get_tstamp(status, &audio_ts);
		@@ -180,6 +182,7 @@ void audio_init_sw(struct audio *audio, bool restart, int frames)
		float v_chunk_time; // duration of one video chunk, in seconds

		assert(audio->ctx_a.sbuffer == NULL);
		assert(audio->ctx_a.xrun_buffer == NULL);

		audio->ctx_a.sbuffer_pos = 0;

		@@ -213,11 +216,12 @@ void audio_init_sw(struct audio *audio, bool restart, int frames)
		audio->ctx_a.read_frames = def_buff_frames;
		buff_size = audio->ctx_a.sbuffer_len * audio->audio_channels * (snd_pcm_format_physical_width(audio->ctx_a.audio_format) / 8);
		audio->ctx_a.sbuffer = malloc(buff_size);
		if (audio->ctx_a.sbuffer == NULL) {
		audio->ctx_a.xrun_buffer = malloc(buff_size);
		if (audio->ctx_a.sbuffer == NULL \|\| audio->ctx_a.xrun_buffer == NULL) {
		audio->set_audio_enable = 0;
		audio->audio_enable = 0;
		snd_pcm_close(audio->ctx_a.capture_hnd);
		D0(fprintf(debug_file, "error: can not allocate %u bytes for audio buffer: %s\n", buff_size, strerror(errno)));
		D0(fprintf(debug_file, "error: can not allocate %u bytes for audio buffer: %s. Audio disabled\n", buff_size, strerror(errno)));
		}
		D6(fprintf(debug_file, "allocated audio buffer for %ld frames, read granularity is %ld frames\n",
		audio->ctx_a.sbuffer_len, audio->ctx_a.read_frames));
		@@ -256,14 +260,23 @@ void audio_process(struct audio *audio)
		snd_pcm_status_get_tstamp(status, &ts);
		avail = snd_pcm_status_get_avail(status);

		D6(fprintf(debug_file, "\navailable audio frames: %ld\n", avail));
		D6(fprintf(debug_file, "\navailable audio frames: %ld, audio timestamp: %ld:%06ld\n", avail, ts.tv_sec, ts.tv_usec));
		assert(audio->ctx_a.rem_samples >= 0);

		snd_pcm_uframes_t to_read = audio->ctx_a.read_frames; // length in audio frames
		if (avail >= audio->ctx_a.read_frames && audio->ctx_a.rem_samples == 0) {
		if (audio->ctx_a.xrun_append > 0) {
		// finish xrun recovery process and fill the buffer with new frames untill it is full
		to_read = audio->ctx_a.xrun_append;

		// === debug code ===
		fprintf(debug_file, "append %ld audio frames\n", to_read);
		// === end of debug ===
		}
		if (avail >= to_read && audio->ctx_a.rem_samples == 0) {
		if (skip_audio(audio, avail))
		continue;
		to_push_flag = AUDIO_PROCESS;
		audio->ctx_a.xrun_append = 0;
		}
		if (audio->ctx_a.rem_samples > 0) {
		if (audio->ctx_a.rem_samples > audio->ctx_a.read_frames) {
		@@ -282,8 +295,19 @@ void audio_process(struct audio *audio)
		}

		if (to_push_flag) {

		assert((to_read + audio->ctx_a.sbuffer_pos) <= audio->ctx_a.sbuffer_len);
		if ((to_read + audio->ctx_a.sbuffer_pos) > audio->ctx_a.sbuffer_len) {
		/* looks like we spent too much time somewhere and now driver has
		* more audio frames than we can store in buffer, but overrun has not occured.
		* We can not record all these frames as it is not proper time yet, but we can increase
		* buffer size and continue with a bigger buffer.
		*/
		int err_code = realloc_buffers(&audio->ctx_a);
		if (err_code < 0) {
		D0(fprintf(debug_file, "error (%d), could not reallocate audio buffer\n", err_code));
		audio->set_audio_enable = 0;
		audio_deinit(audio);
		}
		}

		char *buff_ptr = audio->ctx_a.sbuffer + frames_to_bytes(audio, audio->ctx_a.sbuffer_pos);
		slen = snd_pcm_readi(audio->ctx_a.capture_hnd, buff_ptr, to_read);
		@@ -293,20 +317,7 @@ void audio_process(struct audio *audio)
		record_buffer(audio, to_push_flag);
		}
		} else {
		// TODO: recovery below does not work as expected, snd_pcm_status_get_avail() always returns 0 after buffer overflow; need to be fixed
		if (slen == -EPIPE \|\| slen == -ESTRPIPE) {
		int err;
		D0(fprintf(debug_file, "snd_pcm_readi returned error: %ld\n", (long)slen));
		err = snd_pcm_recover(audio->ctx_a.capture_hnd, slen, 0);
		snd_pcm_reset(audio->ctx_a.capture_hnd);
		if (err != 0) {
		D0(fprintf(debug_file, "error: ALSA could not recover audio buffer, error code: %s\n", snd_strerror(err)));
		// TODO: restart audio interface
		break;
		} else {
		D0(fprintf(debug_file, "audio error recover complete, trying to restart the stream\n"));
		}
		}
		recover_stream(audio, slen, avail);
		}
		} else {
		// no audio frames for processing, return
		@@ -444,6 +455,9 @@ static void audio_deinit(struct audio *audio)
		free(audio->ctx_a.sbuffer);
		audio->ctx_a.sbuffer = NULL;
		audio->ctx_a.sbuffer_pos = 0;
		free(audio->ctx_a.xrun_buffer);
		audio->ctx_a.xrun_buffer = NULL;
		audio->ctx_a.xrun_pos = 0;

		gettimeofday(&tv, NULL);
		D4(fprintf(debug_file, "audio deinitialized at %ld:%06ld\n", tv.tv_sec, tv.tv_usec));
		@@ -498,6 +512,20 @@ static void record_buffer(struct audio *audio, int opt)
		long frames;
		long rem_frames;

		/* check if xrun has occurred and write audio frames that were saved before xrun,
		* then add silence equal in time to lost frames
		*/
		if(audio->ctx_a.lost_frames > 0) {
		_buf = audio->ctx_a.xrun_buffer;
		_buf_len = frames_to_bytes(audio, audio->ctx_a.xrun_pos);
		frames = audio->ctx_a.xrun_pos;
		audio->write_samples(audio, _buf, _buf_len, frames);
		audio->ctx_a.xrun_pos = 0;
		D6(fprintf(debug_file, "record %ld audio frames which were saved before xrun\n", frames));

		write_silence(audio);
		}

		_buf = audio->ctx_a.sbuffer;
		rem_frames = audio->ctx_a.sbuffer_pos;
		while (rem_frames >= audio->ctx_a.read_frames \|\| opt == AUDIO_LAST_CHUNK) {
		@@ -529,3 +557,112 @@ static void record_buffer(struct audio *audio, int opt)
		}
		audio->ctx_a.sbuffer_pos = rem_frames;
		}

		/**
		* Try to recover audio stream after buffer overflow.
		* @param audio pointer to a structure containing audio parameters and buffers
		* @param err error code received after overflow event
		* @param xrun the number of audio frames returned by snd_pcm_format_get_avail() after xrun
		*/
		static void recover_stream(struct audio *audio, snd_pcm_sframes_t err, snd_pcm_uframes_t xrun)
		{
		int ret;
		long prepend_frames;

		if (err == -EPIPE \|\| err == -ESTRPIPE) {
		D0(fprintf(debug_file, "snd_pcm_readi returned error: %ld\n", err));
		ret = snd_pcm_recover(audio->ctx_a.capture_hnd, err, 0);
		if (ret != 0) {
		D0(fprintf(debug_file, "error: ALSA could not recover audio stream, error code: %s\n", snd_strerror(err)));
		// TODO: complete restart of audio interface
		} else {
		if (audio->ctx_a.sbuffer_pos > 0) {
		/* buffer contains some data which was saved before xrun,
		* move the data to a temporary storage in order to free the buffer for current use
		*/
		size_t bytes = frames_to_bytes(audio, audio->ctx_a.sbuffer_pos);
		memcpy(audio->ctx_a.xrun_buffer, audio->ctx_a.sbuffer, bytes);
		audio->ctx_a.xrun_pos = audio->ctx_a.sbuffer_pos;
		audio->ctx_a.sbuffer_pos = 0;
		}

		dummy_read(audio);

		prepend_frames = xrun % audio->ctx_a.read_frames;
		audio->ctx_a.lost_frames = xrun - prepend_frames;
		snd_pcm_format_set_silence(audio->ctx_a.audio_format, audio->ctx_a.sbuffer, prepend_frames * audio->audio_channels);
		audio->ctx_a.sbuffer_pos = prepend_frames;
		audio->ctx_a.xrun_append = audio->ctx_a.read_frames - prepend_frames;
		D0(fprintf(debug_file, "audio error recover complete, trying to restart the stream\n"));

		// === debug code ===
		snd_pcm_status_t *s;
		snd_timestamp_t ts;
		snd_pcm_status_alloca(&s);
		snd_pcm_status(audio->ctx_a.capture_hnd, s);
		snd_pcm_status_get_tstamp(s, &ts);
		fprintf(debug_file, "xrun = %lu, prepend_frames = %ld, lost_frames = %ld, xrun_append = %ld\n",
		xrun, prepend_frames, audio->ctx_a.lost_frames, audio->ctx_a.xrun_append);
		fprintf(debug_file, "audio tstamp: %ld:%06ld\n", ts.tv_sec, ts.tv_usec);
		// === end of debug ===
		}
		}
		}

		/**
		* For some reason, ALSA reports incorrect number of frames (always 0) when audio stream has just started or
		* been recoverded after xrun. Reading small number of frames seems to restore normal operation.
		* @param audio pointer to a structure containing audio parameters and buffers
		*/
		static void dummy_read(struct audio *audio)
		{
		char tmp_buff[32];
		snd_pcm_readi(audio->ctx_a.capture_hnd, tmp_buff, 8);
		}

		/**
		* Pad audio stream with silence frames instead of lost frames after buffer overflow not lose sync with video.
		* This function reuses xrun_buffer for silence frames and data from this buffer should be
		* recorded by the moment.
		* @param audio pointer to a structure containing audio parameters and buffers
		* @return None
		*/
		static void write_silence(struct audio *audio)
		{
		void *_buf;
		long _buf_len;
		long rem_frames;

		_buf = audio->ctx_a.xrun_buffer;
		_buf_len = frames_to_bytes(audio, audio->ctx_a.read_frames);
		snd_pcm_format_set_silence(audio->ctx_a.audio_format, _buf, audio->ctx_a.read_frames * audio->audio_channels);
		rem_frames = audio->ctx_a.lost_frames;
		while (rem_frames >= audio->ctx_a.read_frames) {
		audio->write_samples(audio, _buf, _buf_len, audio->ctx_a.read_frames);
		rem_frames -= audio->ctx_a.read_frames;
		}
		D6(fprintf(debug_file, "recorded %ld audio frames of silence\n", audio->ctx_a.lost_frames));
		assert(rem_frames == 0);
		audio->ctx_a.lost_frames = 0;
		}

		/**
		* Allocate new audio buffer with double size of the previous buffer
		* @param ctx pointer to current audio context
		* @return 0 in case buffers were reallocated and negative error code otherwise
		*/
		static int realloc_buffers(struct context_audio *ctx)
		{
		int ret_val = 0;
		ssize_t new_size = snd_pcm_frames_to_bytes(ctx->capture_hnd, 2 * ctx->sbuffer_len);

		ctx->sbuffer = realloc(ctx->sbuffer, new_size);
		ctx->xrun_buffer = realloc(ctx->xrun_buffer, new_size);
		if (ctx->sbuffer == NULL \|\| ctx->xrun_buffer == NULL) {
		ret_val = -CAMOGM_FRAME_MALLOC;
		} else {
		ctx->sbuffer_len = 2 * ctx->sbuffer_len;
		D1(fprintf(debug_file, "audio buffer reallocated, new size is %ld frames\n", ctx->sbuffer_len));
		}
		return ret_val;
		}

src/camogm_audio.h

+6 −0

Original line number	Diff line number	Diff line
		@@ -51,6 +51,12 @@ struct context_audio {
		struct timeval time_last; ///< calculated time of last audio sample (this value is not taken from ALSA)
		long rem_samples; ///< remaining samples

		long lost_frames; ///< the number of frames lost after buffer overrun
		char *xrun_buffer; ///< temporary storage for the data saved in buffer befor xrun
		long xrun_pos; ///< number of samples in xrun buffer
		long xrun_append; ///< save in buffer this number of frames after xrun, all other frames in chunk
		///< will be silence

		snd_pcm_format_t audio_format; ///< format of audio samples as defined in 'enum snd_pcm_format_t'
		snd_pcm_t *capture_hnd; ///< ALSA PCM handle
		};

Admin message