Process audio between video frames (313f638e) · Commits · Elphel / elphel-apps-camogm

src/camogm.c

+64 −93

Original line number	Diff line number	Diff line
		@@ -168,6 +168,7 @@ static void camogm_set_dummy_read(camogm_state *state, int d);
		static void camogm_set_audio_state(camogm_state state, char args);
		static void camogm_set_audio_volume(camogm_state state, char args);
		static void camogm_set_audio_format(camogm_state state, char args);
		static void camogm_select_sync_port(camogm_state *state);

		void put_uint16(void *buf, u_int16_t val)
		{
		@@ -356,7 +357,11 @@ int camogm_start(camogm_state *state, bool restart)
		int * ifp_this = (int*)&(state->this_frame_params[port]);
		if (state->kml_enable) camogm_init_kml(); // do nothing

		audio_init(&state->audio, restart);
		if (state->frames_skip > 0) {
		state->audio.audio_enable = 0;
		D0(fprintf(debug_file, "Audio recording is not supported in time lapse mode, audio disabled\n"));
		}
		audio_init_hw(&state->audio, restart);
		if (state->format != state->set_format) {
		state->format = state->set_format;
		switch (state->format) {
		@@ -501,8 +506,12 @@ int camogm_start(camogm_state *state, bool restart)
		}
		}

		// init audio stuff
		camogm_select_sync_port(state);
		state->audio.frame_period_us = state->frame_period[state->audio.sync_port];
		audio_init_sw(&state->audio, restart, state->frames_per_chunk);

		// here we are ready to initialize Ogm (or other) file
		audio_start(&state->audio);
		switch (state->format) {
		case CAMOGM_FORMAT_NONE: rslt = 0; break;
		case CAMOGM_FORMAT_OGM: rslt = camogm_start_ogm(state); break;
		@@ -540,12 +549,6 @@ int sendImageFrame(camogm_state *state)
		int fp;
		int port = state->port_num;

		// === debug code ===
		struct timeval tv;
		gettimeofday(&tv, NULL);
		fprintf(debug_file, "start time %ld:%06ld\n", tv.tv_sec, tv.tv_usec);
		// === end of debug ===

		// This is probably needed only for Quicktime (not to exceed already allocated frame index)
		if (!state->rawdev_op && (state->frameno >= (state->max_frames))) {
		D3(fprintf(debug_file, "sendImageFrame:1: state->frameno(0x%x) >= state->max_frames(0x%x)\n", state->frameno, state->max_frames));
		@@ -630,11 +633,6 @@ int sendImageFrame(camogm_state *state)
		return -CAMOGM_FRAME_NOT_READY; // the required frame is not ready
		}

		// === debug code ===
		gettimeofday(&tv, NULL);
		fprintf(debug_file, " time %ld:%06ld ", tv.tv_sec, tv.tv_usec);
		// === end of debug ===

		D3(fprintf(debug_file, "_4_"));
		if (state->exif) {
		D3(fprintf(debug_file, "_5_"));
		@@ -697,10 +695,10 @@ int sendImageFrame(camogm_state *state)
		state->audio.ts_video.tv_usec = state->this_frame_params[port].timestamp_usec;
		int sync_ok = 1;
		// synchronize audio and video before recording has started, this need to be done only once
		if (state->audio.ctx_a.begin_of_stream_with_audio) {
		if (state->audio.begin_of_stream_with_audio) {
		D6(fprintf(debug_file, "\n"));
		if (state->audio.ctx_a.audio_trigger) {
		state->audio.ctx_a.audio_trigger = 0;
		if (state->audio.audio_trigger) {
		state->audio.audio_trigger = 0;
		// calculate how many audio frames we need to skip to synch with the next video frame
		state->audio.ts_video_start = state->audio.ts_audio;
		struct timeval tv = state->audio.ts_video; // next frame right after audio started
		@@ -708,18 +706,19 @@ int sendImageFrame(camogm_state *state)
		tv.tv_usec += state->frame_period[port];
		time_normalize(&tv);
		}
		struct timeval skip_audio_time = time_sub(&tv, &state->audio.ts_audio); // audio time we need to skip to the next frame
		struct timeval skip_audio_time; // audio time we need to skip to the next frame
		timersub(&tv, &state->audio.ts_audio, &skip_audio_time);
		unsigned long long skip_audio_us = time_to_us(&skip_audio_time);
		double s = state->audio.audio_rate;
		s /= 1000.0;
		s *= skip_audio_us;
		s /= 1000.0;
		state->audio.ctx_a.audio_skip_samples = (long) s;
		state->audio.audio_skip_samples = (long) s;
		state->audio.ctx_a.time_start = tv;
		D6(fprintf(debug_file , "audio started at: %ld:%06ld; we need to record it from: %ld:%06ld; audio_to_skip_us == %lld; "
		"audio samples to skip == %lld\n",
		D6(fprintf(debug_file , "audio started at: %ld:%06ld; we need to record it from: %ld:%06ld; skip_audio_us == %lld; "
		"audio samples to skip == %lu\n",
		state->audio.ts_audio.tv_sec, state->audio.ts_audio.tv_usec, tv.tv_sec, tv.tv_usec, skip_audio_us,
		state->audio.ctx_a.audio_skip_samples));
		state->audio.audio_skip_samples));
		}
		D6(fprintf(debug_file, "audio (start): %ld:%06ld; video (current): %ld:%06ld; frame period is: %d us\n",
		state->audio.ts_audio.tv_sec, state->audio.ts_audio.tv_usec, state->audio.ts_video.tv_sec, state->audio.ts_video.tv_usec,
		@@ -729,15 +728,11 @@ int sendImageFrame(camogm_state *state)
		sync_ok = 0;
		} else {
		D6(fprintf(debug_file, "save video frame with time: %ld:%06ld\n", state->audio.ts_video.tv_sec, state->audio.ts_video.tv_usec));
		state->audio.ctx_a.begin_of_stream_with_audio = 0;
		state->audio.begin_of_stream_with_audio = 0;
		state->chunk_frame_cntr = state->frames_per_chunk;
		}
		}

		// === debug code ===
		gettimeofday(&tv, NULL);
		fprintf(debug_file, " time %ld:%06ld ", tv.tv_sec, tv.tv_usec);
		// === end of debug ===

		if (sync_ok) {
		switch (state->format) {
		case CAMOGM_FORMAT_NONE: rslt = 0; break;
		@@ -751,11 +746,6 @@ int sendImageFrame(camogm_state *state)
		rslt = 0;
		}

		// === debug code ===
		gettimeofday(&tv, NULL);
		fprintf(debug_file, " time %ld:%06ld ", tv.tv_sec, tv.tv_usec);
		// === end of debug ===

		if (rslt) {
		D3(fprintf(debug_file, "sendImageFrame:12: camogm_frame_***() returned %d\n", rslt));
		return rslt;
		@@ -1444,14 +1434,7 @@ char * getLineFromPipe(FILE* npipe)
		if (!cmdbufp) cmdbuf[cmdbufp] = 0; //null-terminate first access (probably not needed for the static buffer
		nlp = strpbrk(cmdbuf, ";\n");
		if (!nlp) { //no complete string, try to read more

		// === debug code (around fread) ===
		struct timeval tv1, tv2;
		gettimeofday(&tv1, NULL);
		fl = fread(&cmdbuf[cmdbufp], 1, sizeof(cmdbuf) - cmdbufp - 1, npipe);
		gettimeofday(&tv2, NULL);
		fprintf(debug_file, "pipe read time: start %ld:%06ld, end %ld:%06ld\n", tv1.tv_sec, tv1.tv_usec, tv2.tv_sec, tv2.tv_usec);
		// === end of debug ===
		cmdbuf[cmdbufp + fl] = 0;
		// is there any complete string in a buffer after reading?
		nlp = strpbrk(&cmdbuf[cmdbufp], ";\n"); // there were no new lines before cmdbufp
		@@ -1761,57 +1744,30 @@ int listener_loop(camogm_state *state)
		} else if (state->prog_state == STATE_RUNNING) { // no commands in queue, started
		switch ((rslt = -sendImageFrame(state))) {
		case 0: {

		// === debug ===
		double fps = 1000000 / state->frame_period[state->port_num];
		double avg_rate = 0;
		double ratio = 0;
		struct timeval tv;
		int samples;
		if (state->frameno != 0) {
		avg_rate = ((double)(state->audio.audio_samples + state->audio.avail_samples) / (double)state->frameno) * fps;
		ratio = (double)state->audio.audio_rate / avg_rate;
		}
		samples = ((double)state->frameno / fps) * state->audio.audio_rate;
		fprintf(debug_file, "frames recorded: %d, average sampling rate: %f, ratio: %f, expected sample count: %d\n",
		state->frameno, avg_rate, ratio, samples);
		gettimeofday(&tv, NULL);
		fprintf(debug_file, "system time %ld:%06ld\n", tv.tv_sec, tv.tv_usec);
		// === end of debug ===

		// skip audio processing while sync video frame is not found
		if (state->format == CAMOGM_FORMAT_MOV && !state->audio.ctx_a.begin_of_stream_with_audio) {
		// skip audio processing while not in sync with video
		if (state->format == CAMOGM_FORMAT_MOV &&
		state->audio.audio_enable &&
		!state->audio.begin_of_stream_with_audio) {
		state->chunk_frame_cntr--;
		if (state->chunk_frame_cntr == 0) {
		state->audio.save_data = true;
		state->chunk_frame_cntr = state->frames_per_chunk;
		} else {
		D6(fprintf(debug_file, "not recording audio samples after this frame\n"));
		}
		audio_process(&state->audio);
		// === debug code ===
		float fps = 1000000 / state->frame_period[state->port_num];
		int samples;
		samples = ((float)state->frameno / fps) * state->audio.audio_rate;
		float r = (float)state->audio.audio_samples / (float)samples;
		fprintf(debug_file, "(recorded samples / expected samples) = %f\n", r);
		long calc_diff = samples - state->audio.calc_frames;
		fprintf(debug_file, "calc_frames_diff = %ld\n", calc_diff);
		// === end of debug ===
		state->audio.frame_period = state->frame_period[state->port_num];
		}
		}
		break; // frame sent OK, nothing to do (TODO: check file length/duration)
		case CAMOGM_FRAME_NOT_READY: // just wait for the frame to appear at the current pointer
		// we'll wait for a frame, not to waste resources. But if the compressor is stopped this program will not respond to any commands
		// TODO - add another wait with (short) timeout?
		if (state->audio.audio_enable == 0 \|\| state->audio.sleep_period_us == 0) {
		fp0 = lseek(state->fd_circ[curr_port], 0, SEEK_CUR);
		if (fp0 < 0) {
		D0(fprintf(debug_file, "%s:line %d got broken frame (%d) before waiting for ready\n", __FILE__, __LINE__, fp0));
		rslt = CAMOGM_FRAME_BROKEN;
		} else {

		// === debug code (around lseek) ===
		struct timeval tv1, tv2;
		gettimeofday(&tv1, NULL);
		fp1 = lseek(state->fd_circ[curr_port], LSEEK_CIRC_WAIT, SEEK_END);
		gettimeofday(&tv2, NULL);
		fprintf(debug_file,"time in sleep: start %ld:%06ld, end %ld:%06ld\n", tv1.tv_sec, tv1.tv_usec, tv2.tv_sec, tv2.tv_usec);
		// === end of debug ===

		if (fp1 < 0) {
		D0(fprintf(debug_file, "%s:line %d got broken frame (%d) while waiting for ready. Before that fp0=0x%x\n", __FILE__, __LINE__, fp1, fp0));
		rslt = CAMOGM_FRAME_BROKEN;
		@@ -1819,6 +1775,13 @@ int listener_loop(camogm_state *state)
		break;
		}
		}
		} else {
		// frame period is too long for sleep in lseek, use shorter sleep periods and process audio buffer in between
		audio_process(&state->audio);
		D6(fprintf(debug_file, "usleep(%u)\n", state->audio.sleep_period_us));
		usleep(state->audio.sleep_period_us);
		break;
		}
		// no break
		case CAMOGM_FRAME_CHANGED: // frame parameters have changed
		case CAMOGM_FRAME_NEXTFILE: // next file needed (need to switch to a new file (time/size exceeded limit)
		@@ -2094,12 +2057,6 @@ unsigned int select_port(camogm_state *state)
		D6(fprintf(debug_file, "Selecting sensor port, buffer free size: "));
		for (int i = 0; i < SENSOR_PORTS; i++) {

		// === debug code ===
		struct timeval tv;
		gettimeofday(&tv, NULL);
		fprintf(debug_file, " time: %ld:%06ld ", tv.tv_sec, tv.tv_usec);
		// === end of debug ===

		if (is_chn_active(state, i)) {
		file_pos = lseek(state->fd_circ[i], 0, SEEK_CUR);
		if (file_pos != -1) {
		@@ -2403,3 +2360,17 @@ static void get_fpga_time_w(const struct audio audio, struct timeval tv)
		port = state->port_num;
		*tv = get_fpga_time(state->fd_fparmsall[port], port);
		}

		/**
		* @brief Select sensor port to which audio will be synchronized.
		*
		* This function selects the first active sensor port and sets it as synchronization port for
		* audio stream. Audio data is recorded when new video frame from this port becomes available.
		* @param[in,out] state a pointer to a structure containing current state
		*/
		static void camogm_select_sync_port(camogm_state *state)
		{
		for (int i = 0; i < SENSOR_PORTS; i++)
		if (is_chn_active(state, i))
		state->audio.sync_port = i;
		}

src/camogm.h

+1 −0

Original line number	Diff line number	Diff line
		@@ -223,6 +223,7 @@ typedef struct {
		int max_frames;
		int set_max_frames;
		int frames_per_chunk; ///< QuickTime, the number of samples (images or audio samples) in a chunk
		unsigned int chunk_frame_cntr; ///< QuickTime, number of video frames recorded in current chunk
		int set_frames_per_chunk; ///< QuickTime, index for fast forward (sample-to-chunk atom)
		int frameno; ///< total image frame counter, does not include audio samples
		unsigned long *frame_lengths; ///< QuickTime; pointer to an array of frame lengths wich includes both image frames

src/camogm_audio.c

+198 −164

File changed.

Preview size limit exceeded, changes collapsed.

src/camogm_audio.h

+25 −21

Original line number	Diff line number	Diff line
		@@ -25,8 +25,8 @@
		#include <sys/time.h>
		#include <alsa/asoundlib.h>

		#define SAMPLE_RATE 44100
		#define SAMPLE_CHANNELS 2
		#define SAMPLE_RATE 44100 ///< default sampling rate
		#define SAMPLE_CHANNELS 2 ///< default number of audio channels
		#define SAMPLE_TIME 200 ///< restrict ALSA to have this period, in milliseconds
		#define BUFFER_TIME 1000 ///< approximate ALSA buffer duration, in milliseconds
		#define DEFAULT_SND_DEVICE "plughw:0,0"
		@@ -36,27 +36,34 @@
		#define AUDIO_RATE_MAX 44100
		#define DEFAULT_AUDIO_VOLUME 0xffff

		/**
		* @brief Audio recording context related to stream management.
		* Members of this structure should not be used outside audio module.
		*/
		struct context_audio {
		char *sbuffer; ///< buffer for audio samples
		long sbuffer_len; ///< the length of samples buffer in samples
		long sbuffer_len; ///< total length of audio buffer, in audio frames
		long sbuffer_pos; ///< pointer to current write position in audio buffer, in frames
		long read_frames; ///< read granularity, in frames
		long sample_time; ///< duration of one chunk of audio data, in ms

		struct timeval time_start; ///< start time, set only when stream starts and updated with each new file
		struct timeval time_last; ///< calculated time of last audio sample (this value is not taken from ALSA)
		long rem_samples; ///< remaining samples

		int begin_of_stream_with_audio; ///<
		int audio_trigger; ///< indicates the beginning of audio recording to make some initial set ups
		long long audio_skip_samples; ///<

		snd_pcm_format_t audio_format; ///< format of audio samples as defined in 'enum snd_pcm_format_t'
		snd_pcm_t *capture_hnd; ///< ALSA PCM handle
		};

		/**
		* @brief Various parameters related to audio recording.
		*/
		struct audio {
		int audio_enable; ///< flag indicating that audio is enabled
		int audio_rate; ///< sample rate, in Hz
		int audio_channels; ///< number of channels
		int audio_volume; ///< volume set in range [0..0xFFFF]
		int sync_port; ///< synch audio stream to this sensor port

		int set_audio_enable; ///< temporary storage for new value
		int set_audio_rate; ///< temporary storage for new value
		@@ -74,27 +81,24 @@ struct audio {
		struct timeval ts_audio; ///< time stamp when audio stream started
		struct timeval ts_video; ///< time stamp of each new frame
		struct timeval ts_video_start; ///< time stamp of starting video frame
		int frame_period; ///< video frame period, in microseconds
		snd_pcm_format_t audio_format; ///< format of audio samples as defined in 'enum snd_pcm_format_t'
		int frame_period_us; ///< video frame period measured for #sync_port, in microseconds

		unsigned long audio_skip_samples; ///< skip this number audio frames to sync to video
		int begin_of_stream_with_audio; ///< flag indicating that A/V sync is in progress
		int audio_trigger; ///< indicates the beginning of audio recording to make some initial set ups
		bool save_data; ///< flag indicating that audio data should be recorded, otherwise audio frames should be
		///< stored in buffer for delayed recording
		unsigned int sleep_period_us; ///< sleep period between frames while processing audio stream, in microseconds

		void (get_fpga_time)(const struct audio audio, struct timeval *tv);//< callback function which can get FPGA time
		int (write_samples)(struct audio audio, void *buff, long len, long slen); ///< callback function which actually write data to file, this must be set
		///< in the camogm_init_* function when appropriate format is selected
		// === debug ===
		struct timeval sf_timediff; // system to fpga time difference at the beginning of the stream
		struct timeval m_len;
		struct timeval sys_fpga_timediff;
		int avail_samples;
		long calc_frames; // calculated number of frames by current video frame
		struct timeval prev_ts_video;
		long long skip_samples;
		// === end of debug ===
		};

		void audio_init(struct audio *audio, bool restart);
		void audio_start(struct audio *audio);
		void audio_init_hw(struct audio *audio, bool restart);
		void audio_init_sw(struct audio *audio, bool restart, int frames);
		void audio_process(struct audio *audio);
		void audio_finish(struct audio *audio, bool reset);
		void audio_set_volume(int nvolume);
		unsigned long audio_get_hw_buffer_max(void);

		#endif /* _CAMOGM_AUDIO_H */

src/camogm_mov.c

+4 −17

Original line number	Diff line number	Diff line
		@@ -26,11 +26,7 @@
		#include <sys/types.h>
		#include <assert.h>

		// for debug only
		#include <math.h>

		#include "camogm_mov.h"
		#include "thelper.h"

		/** @brief QuickTime header length (w/o index tables) enough to accommodate static data */
		#define QUICKTIME_MIN_HEADER 0x300
		@@ -228,9 +224,9 @@ int camogm_frame_mov(camogm_state *state)

		/**
		* Write audio samples to file.
		* @param[in] buff pointer to buffer containing audio samples
		* @param[in] buff pointer to buffer containing audio frames
		* @param[in] len the size of buffer, in bytes
		* @param[in] slen the number of audio samples in buffer
		* @param[in] slen the number of audio frames in buffer
		* @return 0 if data was recorded successfully and negative error code otherwise
		*/
		static int camogm_audio_mov(struct audio audio, void buff, long len, long slen)
		@@ -240,7 +236,7 @@ static int camogm_audio_mov(struct audio audio, void buff, long len, long slen
		ssize_t wr_len;
		camogm_state *state = container_of(audio, camogm_state, audio);

		D6(fprintf(debug_file, "write audio sample, len = %ld, slen = %ld\n", len, slen));
		D6(fprintf(debug_file, "write audio chunk, len = %ld, slen = %ld\n", len, slen));

		wr_len = write(state->ivf, buff, len);
		if (wr_len < len) {
		@@ -289,19 +285,15 @@ int camogm_end_mov(camogm_state *state)
		q_template, // string containing header template
		state->ivf, // output file descriptor (opened)
		state->width, // width in pixels
		state->height,
		state->height, // height in pixels
		state->frameno, // the number of image frames
		state->frame_period[port] / (1000000 / timescale),
		state->frames_per_chunk,
		0, // frame size - will look in the table
		(int)((float)timescale / (state->timescale)),
		// state->frame_lengths, // array of frame lengths to build an index
		NULL, // array of frame lengths to build an index
		state->frame_data_start
		);
		// === debug code ===
		fprintf(debug_file, "total # of video frames: %d, total # of audio samples: %ld\n", state->frameno, state->audio.audio_samples);
		// === end of debug ===
		close(state->ivf);
		state->ivf = -1;
		// free memory used for index
		@@ -640,11 +632,6 @@ int quicktime_template_parser( camogm_state *state,
		iFileLen = strlen(iFile);
		lseek(ofd, 0, SEEK_SET);

		// === debug ===
		struct timeval m_len = state->audio.m_len; // duration of movie
		fprintf(debug_file, "frameno: %d, duration: %ld:%06ld, audio_samples: %ld\n", state->frameno, m_len.tv_sec, m_len.tv_usec, state->audio.audio_samples);
		// === ebd of debug ===

		audio_timescale = state->audio.audio_rate;
		audio_rate = audio_timescale; // QuickTime defines sample rate as unsigned 16.16 fixed-point number
		audio_rate <<= 16;