Skip to content

E
elphel-apps-camogm
Commit 313f638e authored by Mikhail Karpenko's avatar Mikhail Karpenko
Browse files
Options

Process audio between video frames

parent b50eb743
Show whitespace changes
Inline Side-by-side
Showing with 292 additions and 330 deletions
src/camogm.c
View file @ 313f638e
......@@ -168,6 +168,7 @@ static void camogm_set_dummy_read(camogm_state *state, int d);
static void camogm_set_audio_state(camogm_state *state, char *args);
static void camogm_set_audio_volume(camogm_state *state, char *args);
static void camogm_set_audio_format(camogm_state *state, char *args);
static void camogm_select_sync_port(camogm_state *state);
void put_uint16(void *buf, u_int16_t val)
{
......@@ -356,7 +357,11 @@ int camogm_start(camogm_state *state, bool restart)
int * ifp_this = (int*)&(state->this_frame_params[port]);
if (state->kml_enable) camogm_init_kml(); // do nothing
audio_init(&state->audio, restart);
if (state->frames_skip > 0) {
state->audio.audio_enable = 0;
D0(fprintf(debug_file, "Audio recording is not supported in time lapse mode, audio disabled\n"));
}
audio_init_hw(&state->audio, restart);
if (state->format != state->set_format) {
state->format = state->set_format;
switch (state->format) {
......@@ -501,8 +506,12 @@ int camogm_start(camogm_state *state, bool restart)
}
}
// init audio stuff
camogm_select_sync_port(state);
state->audio.frame_period_us = state->frame_period[state->audio.sync_port];
audio_init_sw(&state->audio, restart, state->frames_per_chunk);
// here we are ready to initialize Ogm (or other) file
audio_start(&state->audio);
switch (state->format) {
case CAMOGM_FORMAT_NONE: rslt = 0; break;
case CAMOGM_FORMAT_OGM: rslt = camogm_start_ogm(state); break;
......@@ -540,12 +549,6 @@ int sendImageFrame(camogm_state *state)
int fp;
int port = state->port_num;
// === debug code ===
struct timeval tv;
gettimeofday(&tv, NULL);
fprintf(debug_file, "start time %ld:%06ld\n", tv.tv_sec, tv.tv_usec);
// === end of debug ===
// This is probably needed only for Quicktime (not to exceed already allocated frame index)
if (!state->rawdev_op && (state->frameno >= (state->max_frames))) {
D3(fprintf(debug_file, "sendImageFrame:1: state->frameno(0x%x) >= state->max_frames(0x%x)\n", state->frameno, state->max_frames));
......@@ -630,11 +633,6 @@ int sendImageFrame(camogm_state *state)
return -CAMOGM_FRAME_NOT_READY; // the required frame is not ready
}
// === debug code ===
gettimeofday(&tv, NULL);
fprintf(debug_file, " time %ld:%06ld ", tv.tv_sec, tv.tv_usec);
// === end of debug ===
D3(fprintf(debug_file, "_4_"));
if (state->exif) {
D3(fprintf(debug_file, "_5_"));
......@@ -697,10 +695,10 @@ int sendImageFrame(camogm_state *state)
state->audio.ts_video.tv_usec = state->this_frame_params[port].timestamp_usec;
int sync_ok = 1;
// synchronize audio and video before recording has started, this need to be done only once
if (state->audio.ctx_a.begin_of_stream_with_audio) {
if (state->audio.begin_of_stream_with_audio) {
D6(fprintf(debug_file, "\n"));
if (state->audio.ctx_a.audio_trigger) {
state->audio.ctx_a.audio_trigger = 0;
if (state->audio.audio_trigger) {
state->audio.audio_trigger = 0;
// calculate how many audio frames we need to skip to synch with the next video frame
state->audio.ts_video_start = state->audio.ts_audio;
struct timeval tv = state->audio.ts_video; // next frame right after audio started
......@@ -708,18 +706,19 @@ int sendImageFrame(camogm_state *state)
tv.tv_usec += state->frame_period[port];
time_normalize(&tv);
}
struct timeval skip_audio_time = time_sub(&tv, &state->audio.ts_audio); // audio time we need to skip to the next frame
struct timeval skip_audio_time; // audio time we need to skip to the next frame
timersub(&tv, &state->audio.ts_audio, &skip_audio_time);
unsigned long long skip_audio_us = time_to_us(&skip_audio_time);
double s = state->audio.audio_rate;
s /= 1000.0;
s *= skip_audio_us;
s /= 1000.0;
state->audio.ctx_a.audio_skip_samples = (long) s;
state->audio.audio_skip_samples = (long) s;
state->audio.ctx_a.time_start = tv;
D6(fprintf(debug_file , "audio started at: %ld:%06ld; we need to record it from: %ld:%06ld; audio_to_skip_us == %lld; "
"audio samples to skip == %lld\n",
D6(fprintf(debug_file , "audio started at: %ld:%06ld; we need to record it from: %ld:%06ld; skip_audio_us == %lld; "
"audio samples to skip == %lu\n",
state->audio.ts_audio.tv_sec, state->audio.ts_audio.tv_usec, tv.tv_sec, tv.tv_usec, skip_audio_us,
state->audio.ctx_a.audio_skip_samples));
state->audio.audio_skip_samples));
}
D6(fprintf(debug_file, "audio (start): %ld:%06ld; video (current): %ld:%06ld; frame period is: %d us\n",
state->audio.ts_audio.tv_sec, state->audio.ts_audio.tv_usec, state->audio.ts_video.tv_sec, state->audio.ts_video.tv_usec,
......@@ -729,15 +728,11 @@ int sendImageFrame(camogm_state *state)
sync_ok = 0;
} else {
D6(fprintf(debug_file, "save video frame with time: %ld:%06ld\n", state->audio.ts_video.tv_sec, state->audio.ts_video.tv_usec));
state->audio.ctx_a.begin_of_stream_with_audio = 0;
state->audio.begin_of_stream_with_audio = 0;
state->chunk_frame_cntr = state->frames_per_chunk;
}
}
// === debug code ===
gettimeofday(&tv, NULL);
fprintf(debug_file, " time %ld:%06ld ", tv.tv_sec, tv.tv_usec);
// === end of debug ===
if (sync_ok) {
switch (state->format) {
case CAMOGM_FORMAT_NONE: rslt = 0; break;
......@@ -751,11 +746,6 @@ int sendImageFrame(camogm_state *state)
rslt = 0;
}
// === debug code ===
gettimeofday(&tv, NULL);
fprintf(debug_file, " time %ld:%06ld ", tv.tv_sec, tv.tv_usec);
// === end of debug ===
if (rslt) {
D3(fprintf(debug_file, "sendImageFrame:12: camogm_frame_***() returned %d\n", rslt));
return rslt;
......@@ -1444,14 +1434,7 @@ char * getLineFromPipe(FILE* npipe)
if (!cmdbufp) cmdbuf[cmdbufp] = 0; //null-terminate first access (probably not needed for the static buffer
nlp = strpbrk(cmdbuf, ";\n");
if (!nlp) { //no complete string, try to read more
// === debug code (around fread) ===
struct timeval tv1, tv2;
gettimeofday(&tv1, NULL);
fl = fread(&cmdbuf[cmdbufp], 1, sizeof(cmdbuf) - cmdbufp - 1, npipe);
gettimeofday(&tv2, NULL);
fprintf(debug_file, "pipe read time: start %ld:%06ld, end %ld:%06ld\n", tv1.tv_sec, tv1.tv_usec, tv2.tv_sec, tv2.tv_usec);
// === end of debug ===
cmdbuf[cmdbufp + fl] = 0;
// is there any complete string in a buffer after reading?
nlp = strpbrk(&cmdbuf[cmdbufp], ";\n"); // there were no new lines before cmdbufp
......@@ -1761,57 +1744,30 @@ int listener_loop(camogm_state *state)
} else if (state->prog_state == STATE_RUNNING) { // no commands in queue, started
switch ((rslt = -sendImageFrame(state))) {
case 0: {
// === debug ===
double fps = 1000000 / state->frame_period[state->port_num];
double avg_rate = 0;
double ratio = 0;
struct timeval tv;
int samples;
if (state->frameno != 0) {
avg_rate = ((double)(state->audio.audio_samples + state->audio.avail_samples) / (double)state->frameno) * fps;
ratio = (double)state->audio.audio_rate / avg_rate;
}
samples = ((double)state->frameno / fps) * state->audio.audio_rate;
fprintf(debug_file, "frames recorded: %d, average sampling rate: %f, ratio: %f, expected sample count: %d\n",
state->frameno, avg_rate, ratio, samples);
gettimeofday(&tv, NULL);
fprintf(debug_file, "system time %ld:%06ld\n", tv.tv_sec, tv.tv_usec);
// === end of debug ===
// skip audio processing while sync video frame is not found
if (state->format == CAMOGM_FORMAT_MOV && !state->audio.ctx_a.begin_of_stream_with_audio) {
// skip audio processing while not in sync with video
if (state->format == CAMOGM_FORMAT_MOV &&
state->audio.audio_enable &&
!state->audio.begin_of_stream_with_audio) {
state->chunk_frame_cntr--;
if (state->chunk_frame_cntr == 0) {
state->audio.save_data = true;
state->chunk_frame_cntr = state->frames_per_chunk;
} else {
D6(fprintf(debug_file, "not recording audio samples after this frame\n"));
}
audio_process(&state->audio);
// === debug code ===
float fps = 1000000 / state->frame_period[state->port_num];
int samples;
samples = ((float)state->frameno / fps) * state->audio.audio_rate;
float r = (float)state->audio.audio_samples / (float)samples;
fprintf(debug_file, "(recorded samples / expected samples) = %f\n", r);
long calc_diff = samples - state->audio.calc_frames;
fprintf(debug_file, "calc_frames_diff = %ld\n", calc_diff);
// === end of debug ===
state->audio.frame_period = state->frame_period[state->port_num];
}
}
break; // frame sent OK, nothing to do (TODO: check file length/duration)
case CAMOGM_FRAME_NOT_READY: // just wait for the frame to appear at the current pointer
// we'll wait for a frame, not to waste resources. But if the compressor is stopped this program will not respond to any commands
// TODO - add another wait with (short) timeout?
if (state->audio.audio_enable == 0 || state->audio.sleep_period_us == 0) {
fp0 = lseek(state->fd_circ[curr_port], 0, SEEK_CUR);
if (fp0 < 0) {
D0(fprintf(debug_file, "%s:line %d got broken frame (%d) before waiting for ready\n", __FILE__, __LINE__, fp0));
rslt = CAMOGM_FRAME_BROKEN;
} else {
// === debug code (around lseek) ===
struct timeval tv1, tv2;
gettimeofday(&tv1, NULL);
fp1 = lseek(state->fd_circ[curr_port], LSEEK_CIRC_WAIT, SEEK_END);
gettimeofday(&tv2, NULL);
fprintf(debug_file,"time in sleep: start %ld:%06ld, end %ld:%06ld\n", tv1.tv_sec, tv1.tv_usec, tv2.tv_sec, tv2.tv_usec);
// === end of debug ===
if (fp1 < 0) {
D0(fprintf(debug_file, "%s:line %d got broken frame (%d) while waiting for ready. Before that fp0=0x%x\n", __FILE__, __LINE__, fp1, fp0));
rslt = CAMOGM_FRAME_BROKEN;
......@@ -1819,6 +1775,13 @@ int listener_loop(camogm_state *state)
break;
}
}
} else {
// frame period is too long for sleep in lseek, use shorter sleep periods and process audio buffer in between
audio_process(&state->audio);
D6(fprintf(debug_file, "usleep(%u)\n", state->audio.sleep_period_us));
usleep(state->audio.sleep_period_us);
break;
}
// no break
case CAMOGM_FRAME_CHANGED: // frame parameters have changed
case CAMOGM_FRAME_NEXTFILE: // next file needed (need to switch to a new file (time/size exceeded limit)
......@@ -2094,12 +2057,6 @@ unsigned int select_port(camogm_state *state)
D6(fprintf(debug_file, "Selecting sensor port, buffer free size: "));
for (int i = 0; i < SENSOR_PORTS; i++) {
// === debug code ===
struct timeval tv;
gettimeofday(&tv, NULL);
fprintf(debug_file, " time: %ld:%06ld ", tv.tv_sec, tv.tv_usec);
// === end of debug ===
if (is_chn_active(state, i)) {
file_pos = lseek(state->fd_circ[i], 0, SEEK_CUR);
if (file_pos != -1) {
......@@ -2403,3 +2360,17 @@ static void get_fpga_time_w(const struct audio *audio, struct timeval *tv)
port = state->port_num;
*tv = get_fpga_time(state->fd_fparmsall[port], port);
}
/**
* @brief Select sensor port to which audio will be synchronized.
*
* This function selects the first active sensor port and sets it as synchronization port for
* audio stream. Audio data is recorded when new video frame from this port becomes available.
* @param[in,out] state a pointer to a structure containing current state
*/
static void camogm_select_sync_port(camogm_state *state)
{
for (int i = 0; i < SENSOR_PORTS; i++)
if (is_chn_active(state, i))
state->audio.sync_port = i;
}
src/camogm.h
View file @ 313f638e
......@@ -223,6 +223,7 @@ typedef struct {
int max_frames;
int set_max_frames;
int frames_per_chunk; ///< QuickTime, the number of samples (images or audio samples) in a chunk
unsigned int chunk_frame_cntr; ///< QuickTime, number of video frames recorded in current chunk
int set_frames_per_chunk; ///< QuickTime, index for fast forward (sample-to-chunk atom)
int frameno; ///< total image frame counter, does not include audio samples
unsigned long *frame_lengths; ///< QuickTime; pointer to an array of frame lengths wich includes both image frames
......
src/camogm_audio.c
View file @ 313f638e
......@@ -29,25 +29,33 @@
// for debug only
#include <math.h>
enum {
AUDIO_NONE,
AUDIO_PROCESS,
AUDIO_FINISH,
AUDIO_LAST_CHUNK
};
static void audio_deinit(struct audio *audio);
static bool skip_audio(struct audio *audio, snd_pcm_uframes_t frames);
static long frames_to_bytes(const struct audio *audio, long frames);
static void record_buffer(struct audio *audio, int opt);
/**
* Initialize audio interface.
* Initialize HW part of audio interface.
* Asserts:
* audio->get_fpga_time pointer to callback function is not set
* @param audio pointer to a structure containing audio parameters and buffers
* @param restart flag indicating that full restart is requested
* @return None
*/
void audio_init(struct audio *audio, bool restart)
void audio_init_hw(struct audio *audio, bool restart)
{
// apply settings in case they have changed
audio->audio_enable = audio->set_audio_enable;
audio->audio_rate = audio->set_audio_rate;
audio->audio_channels = audio->set_audio_channels;
audio->audio_volume = audio->set_audio_volume;
audio->ctx_a.sample_time = SAMPLE_TIME;
if(audio->audio_enable == 0) {
return;
......@@ -64,20 +72,9 @@ void audio_init(struct audio *audio, bool restart)
snd_pcm_status_t *status; // allocated on stack, do not free
snd_timestamp_t audio_ts;
audio->audio_format = SND_PCM_FORMAT_S16_LE;
audio->ctx_a.sbuffer_len = audio->audio_rate * audio->ctx_a.sample_time;
audio->ctx_a.sbuffer_len /= 1000;
audio->ctx_a.sbuffer_len -= audio->ctx_a.sbuffer_len % 2;
audio->ctx_a.audio_format = SND_PCM_FORMAT_S16_LE;
// 'while' loop here just to break initialization sequence after an error
while (true) {
size_t buff_size = audio->ctx_a.sbuffer_len * audio->audio_channels * (snd_pcm_format_physical_width(audio->audio_format) / 8);
audio->ctx_a.sbuffer = malloc(buff_size);
if (audio->ctx_a.sbuffer == NULL) {
D0(fprintf(debug_file, "error: can not allocate buffer for audio samples: %s\n", strerror(errno)));
break;
}
D6(fprintf(debug_file, "audio sbuffer_len == %ld\n", audio->ctx_a.sbuffer_len));
if ((err = snd_pcm_open(&audio->ctx_a.capture_hnd, audio->dev_name, SND_PCM_STREAM_CAPTURE, 0)) < 0)
break;
snd_pcm_hw_params_alloca(&hw_params);
......@@ -85,7 +82,7 @@ void audio_init(struct audio *audio, bool restart)
break;
if ((err = snd_pcm_hw_params_set_access(audio->ctx_a.capture_hnd, hw_params, SND_PCM_ACCESS_RW_INTERLEAVED)) < 0)
break;
if ((err = snd_pcm_hw_params_set_format(audio->ctx_a.capture_hnd, hw_params, audio->audio_format)) < 0)
if ((err = snd_pcm_hw_params_set_format(audio->ctx_a.capture_hnd, hw_params, audio->ctx_a.audio_format)) < 0)
break;
if ((err = snd_pcm_hw_params_set_rate_near(audio->ctx_a.capture_hnd, hw_params, &t, 0)) < 0)
......@@ -117,18 +114,19 @@ void audio_init(struct audio *audio, bool restart)
break;
}
if (init_ok) {
char tmp_buff[32];
snd_pcm_prepare(audio->ctx_a.capture_hnd);
snd_pcm_reset(audio->ctx_a.capture_hnd);
audio_set_volume(audio->audio_volume);
// read some samples to force the driver to start time stamping
snd_pcm_readi(audio->ctx_a.capture_hnd, audio->ctx_a.sbuffer, 8);
// read some frames to force the driver to start reporting correct number of available frames
snd_pcm_readi(audio->ctx_a.capture_hnd, tmp_buff, 8);
snd_pcm_status_alloca(&status);
snd_pcm_status(audio->ctx_a.capture_hnd, status);
snd_pcm_status_get_tstamp(status, &audio_ts);
audio->ctx_a.begin_of_stream_with_audio = 1;
audio->ctx_a.audio_trigger = 1;
audio->ctx_a.audio_skip_samples = 0;
audio->begin_of_stream_with_audio = 1;
audio->audio_trigger = 1;
audio->audio_skip_samples = 0;
struct timeval fpga_tv, sys_tv;
assert(audio->get_fpga_time);
......@@ -136,19 +134,11 @@ void audio_init(struct audio *audio, bool restart)
gettimeofday(&sys_tv, NULL);
struct timeval d; // system and FPGA time difference
d = time_sub(&sys_tv, &fpga_tv);
audio->sys_fpga_timediff = d;
timersub(&sys_tv, &fpga_tv, &d);
struct timeval tv;
tv.tv_sec = audio_ts.tv_sec;
tv.tv_usec = audio_ts.tv_usec;
audio->ts_audio = time_sub(&tv, &d);
audio->sf_timediff = tv;
// === debug code ===
snd_pcm_uframes_t val;
snd_pcm_hw_params_get_buffer_size_max(hw_params, &val);
fprintf(debug_file, "ALSA buffer size: %lu\n", val);
// === end of debug ===
timersub(&tv, &d, &audio->ts_audio);
D4(fprintf(debug_file, "audio_init OK, system time = %ld:%06ld, FPGA time = %ld:%06ld, audio start time = %ld:%06ld, audio_ts = %ld:%06ld\n",
sys_tv.tv_sec, sys_tv.tv_usec, fpga_tv.tv_sec, fpga_tv.tv_usec, audio->ts_audio.tv_sec, audio->ts_audio.tv_usec,
......@@ -161,30 +151,89 @@ void audio_init(struct audio *audio, bool restart)
}
}
void audio_start(struct audio *audio)
/**
* Initialize SW part of audio interface. This function assumes that HW part has already been initialized and
* all values required to calculate audio buffer size are properly set. Audio buffer is allocated here.
* Asserts:
* audio buffer is not initialized when full restart is requested
* @param audio pointer to a structure containing audio parameters and buffers
* @param restart flag indicating that full restart is requested
* @param frames number of frames in video chunk; audio chunk is recorder after video chunk thus we need
* this value to calculate buffer size
* @return None
*/
void audio_init_sw(struct audio *audio, bool restart, int frames)
{
audio->audio_frameno = 0;
audio->audio_samples = 0;
audio->ctx_a.rem_samples = 0;
audio->ctx_a.time_last.tv_sec = 0;
audio->ctx_a.time_last.tv_usec = 0;
audio->ctx_a.sample_time = SAMPLE_TIME;
audio->ctx_a.sbuffer_pos = 0;
if (audio->audio_enable == 0)
return;
if (restart) {
size_t buff_size;
size_t max_buff_frames, def_buff_frames; // maximum and default buffer sizes, in frames
float v_chunk_time; // duration of one video chunk, in seconds
assert(audio->ctx_a.sbuffer == NULL);
/* decide if camogm can sleep using lseek into video buffer: if audio HW buffer size is less than
* video frame period then we need to process audio stream between video frames and must use shorter
* sleep periods to prevent buffer overflow, and we are good if audio HW buffer is bigger than
* video frame period.
*/
if (audio->frame_period_us < (BUFFER_TIME * 1000)) {
audio->sleep_period_us = 0;
} else {
audio->sleep_period_us = (BUFFER_TIME * 1000) / 2;
}
/* maximum buffer size takes into account video frame period and the number of video frames in one chunk;
* if calculated maximum buffer size is less than BUFFER_TIME then BUFFER_TIME is used as audio buffer size,
*/
v_chunk_time = (float)(audio->frame_period_us * frames) / 1000000.0f;
max_buff_frames = v_chunk_time * audio->audio_rate;
max_buff_frames -= max_buff_frames % 2;
def_buff_frames = audio->audio_rate * audio->ctx_a.sample_time;
def_buff_frames /= 1000;
def_buff_frames -= def_buff_frames % 2;
if (max_buff_frames > def_buff_frames) {
audio->ctx_a.sbuffer_len = max_buff_frames;
// round buffer size up to the nearest multiple of SAMPLE_TIME to simplify things
audio->ctx_a.sbuffer_len += (max_buff_frames % def_buff_frames);
} else {
audio->ctx_a.sbuffer_len = def_buff_frames;
}
audio->ctx_a.read_frames = def_buff_frames;
buff_size = audio->ctx_a.sbuffer_len * audio->audio_channels * (snd_pcm_format_physical_width(audio->ctx_a.audio_format) / 8);
audio->ctx_a.sbuffer = malloc(buff_size);
if (audio->ctx_a.sbuffer == NULL) {
audio->set_audio_enable = 0;
audio->audio_enable = 0;
snd_pcm_close(audio->ctx_a.capture_hnd);
D0(fprintf(debug_file, "error: can not allocate %u bytes for audio buffer: %s\n", buff_size, strerror(errno)));
}
D6(fprintf(debug_file, "allocated audio buffer for %ld frames, read granularity is %ld frames\n",
audio->ctx_a.sbuffer_len, audio->ctx_a.read_frames));
}
}
/**
* Process audio stream.
* Asserts:
* audio->write_samples pointer to a function is not set
* number of audio frames remaining for recording is positive value
* number of audio frames remaining for recording is negative value
* @param audio pointer to a structure containing audio parameters and buffers
* @return None
*/
void audio_process(struct audio *audio)
{
snd_pcm_sframes_t slen;
int counter = 0;
void *_buf;
long _buf_len;
struct timeval tv_sys;
snd_timestamp_t ts;
snd_pcm_status_t *status; // allocated on stack, do not free
......@@ -193,141 +242,68 @@ void audio_process(struct audio *audio)
assert(audio->write_samples);
// first of all, save all we have in the buffer by the moment
if (audio->save_data && audio->ctx_a.sbuffer_pos > audio->ctx_a.sample_time)
record_buffer(audio, AUDIO_PROCESS);
snd_pcm_status_alloca(&status);
for (;;) {
long avail = 0;
int to_push_flag = 0;
int to_push_flag = AUDIO_NONE;
counter++;
gettimeofday(&tv_sys, NULL);
tv_sys.tv_usec += audio->ctx_a.sample_time;
time_normalize(&tv_sys);
snd_pcm_status(audio->ctx_a.capture_hnd, status);
snd_pcm_status_get_tstamp(status, &ts);
avail = snd_pcm_status_get_avail(status);
// === debug code ====
int sbf; // samples before recent frame
int samples, total;
struct timeval av_tdiff, ts_corrected;
total = audio->audio_samples + audio->skip_samples;
fprintf(debug_file, "recent frame tstamp: %ld:%06ld\n", audio->ts_video.tv_sec, audio->ts_video.tv_usec);
fprintf(debug_file, "available samples: %ld, recorded samples (+skipped): %ld (%d)\n",
avail, audio->audio_samples, total);
ts_corrected = time_sub(&ts, &audio->sys_fpga_timediff);
fprintf(debug_file, "tstamp: %ld:%06ld, corrected tstamp: %ld:%06ld\n", ts.tv_sec, ts.tv_usec, ts_corrected.tv_sec, ts_corrected.tv_usec);
av_tdiff = time_sub(&ts_corrected, &audio->ts_video);
samples = (int)floor(((double)av_tdiff.tv_sec + (double)av_tdiff.tv_usec / 1000000) * audio->audio_rate);
fprintf(debug_file, "time diff since last frame: %ld:%06ld, # of samples since last frame: %d\n", av_tdiff.tv_sec, av_tdiff.tv_usec, samples);
if (samples > avail) {
// some samples have already been recorded
samples -= avail;
sbf = audio->audio_samples - samples;
} else {
sbf = audio->audio_samples + (avail - samples);
}
fprintf(debug_file, "samples before recent frame: %d\n", sbf);
if (avail == 0) {
snd_pcm_state_t s = snd_pcm_status_get_state(status);
fprintf(debug_file, "stream state: %d\n", s);
}
audio->avail_samples = avail;
// === end of debug ===
D6(fprintf(debug_file, "\navailable audio frames: %ld\n", avail));
assert(audio->ctx_a.rem_samples >= 0);
snd_pcm_uframes_t to_read = audio->ctx_a.sbuffer_len; // length in audio frames
if (avail >= audio->ctx_a.sbuffer_len && audio->ctx_a.rem_samples == 0) {
snd_pcm_uframes_t to_read = audio->ctx_a.read_frames; // length in audio frames
if (avail >= audio->ctx_a.read_frames && audio->ctx_a.rem_samples == 0) {
if (skip_audio(audio, avail))
continue;
to_push_flag = 1;
to_push_flag = AUDIO_PROCESS;
}
if (audio->ctx_a.rem_samples > 0) {
if (audio->ctx_a.rem_samples > audio->ctx_a.sbuffer_len) {
if (avail >= audio->ctx_a.sbuffer_len) {
to_read = audio->ctx_a.sbuffer_len;
audio->ctx_a.rem_samples -= audio->ctx_a.sbuffer_len;
to_push_flag = 2;
if (audio->ctx_a.rem_samples > audio->ctx_a.read_frames) {
if (avail >= audio->ctx_a.read_frames) {
to_read = audio->ctx_a.read_frames;
audio->ctx_a.rem_samples -= audio->ctx_a.read_frames;
to_push_flag = AUDIO_FINISH;
}
} else {
if (avail >= audio->ctx_a.rem_samples) {
to_read = audio->ctx_a.rem_samples;
audio->ctx_a.rem_samples = 0;
to_push_flag = 2;
to_push_flag = AUDIO_LAST_CHUNK;
}
}
}
if (to_push_flag) {
slen = snd_pcm_readi(audio->ctx_a.capture_hnd, audio->ctx_a.sbuffer, to_read);
if (slen > 0) {
int flag = 1;
long offset = 0;
// // check the length of the movie and sound track, proceed only if audio and video already in sync
// if (to_push_flag == 1 && audio->ctx_a.begin_of_stream_with_audio) {
// struct timeval sl = audio->ctx_a.time_last;
// sl.tv_usec += audio->ctx_a.sample_time;
// time_normalize(&sl);
// struct timeval m_end;
// m_end = audio->ts_video;
// m_end.tv_usec += audio->frame_period / 2;
// time_normalize(&m_end);
// struct timeval m_len;
// m_len = time_sub(&m_end, &audio->ctx_a.time_start);
// if (time_comp(&sl, &m_len) > 0) {
// D4(fprintf(debug_file, "Sound chunk is too early, skip it\n"));
// break;
// }
// }
// we need to skip some samples in a new session, but if we just switch the frames then
// we need to split new samples in the buffer into two parts - for the previous file,
// and the next one...
// so we can just save in the first file new data, and in the next use "skip_samples" field
if (audio->ctx_a.audio_skip_samples != 0) {
D5(fprintf(debug_file, "skip_samples = %lld, available samples = %ld\n", audio->ctx_a.audio_skip_samples, slen));
if (audio->ctx_a.audio_skip_samples >= slen) {
audio->ctx_a.audio_skip_samples -= slen;
flag = 0;
} else {
offset = audio->ctx_a.audio_skip_samples;
audio->ctx_a.audio_skip_samples = 0;
}
}
if (flag) {
long samples = slen - offset;
_buf = (void *)audio->ctx_a.sbuffer;
_buf = (void *)((char *) _buf + offset * audio->audio_channels * (snd_pcm_format_physical_width(audio->audio_format) / 8));
_buf_len = samples * audio->audio_channels * (snd_pcm_format_physical_width(audio->audio_format) / 8);
audio->write_samples(audio, _buf, _buf_len, samples);
float tr = 1.0 / audio->audio_rate;
float l = tr * audio->audio_samples;
unsigned long s = (unsigned long) l;
l -= s;
l *= 1000000;
unsigned long us = (unsigned long) l;
audio->ctx_a.time_last.tv_sec = s;
audio->ctx_a.time_last.tv_usec = us;
D6(fprintf(debug_file, "%d: sound time %lu:%06lu, at %ld:%06ld; samples: %ld\n",
counter, s, us, tv_sys.tv_sec, tv_sys.tv_usec, samples));
assert((to_read + audio->ctx_a.sbuffer_pos) <= audio->ctx_a.sbuffer_len);
char *buff_ptr = audio->ctx_a.sbuffer + frames_to_bytes(audio, audio->ctx_a.sbuffer_pos);
slen = snd_pcm_readi(audio->ctx_a.capture_hnd, buff_ptr, to_read);
if (slen > 0) {
audio->ctx_a.sbuffer_pos += slen;
if (audio->save_data || (to_push_flag == AUDIO_FINISH) || (to_push_flag == AUDIO_LAST_CHUNK)) {
record_buffer(audio, to_push_flag);
}
} else {
// TODO: recovery below does not work as expected, snd_pcm_status_get_avail() always returns 0 after buffer overflow; need to be fixed
if (slen == -EPIPE || slen == -ESTRPIPE) {
int err;
fprintf(debug_file, "snd_pcm_readi returned error: %ld\n", (long)slen);
D0(fprintf(debug_file, "snd_pcm_readi returned error: %ld\n", (long)slen));
err = snd_pcm_recover(audio->ctx_a.capture_hnd, slen, 0);
snd_pcm_reset(audio->ctx_a.capture_hnd);
// snd_pcm_drain(audio->ctx_a.capture_hnd);
// err = snd_pcm_prepare(audio->ctx_a.capture_hnd);
if (err != 0) {
D0(fprintf(debug_file, "error: ALSA could not recover audio buffer, error code: %s\n", snd_strerror(err)));
// TODO: restart audio interface
break;
} else {
fprintf(debug_file, "audio error recover complete, trying to restart the stream\n");
// snd_pcm_drain(audio->ctx_a.capture_hnd);
// err = snd_pcm_prepare(audio->ctx_a.capture_hnd);
// fprintf(debug_file, "snd_pcm_prepare returned %d\n", err);
D0(fprintf(debug_file, "audio error recover complete, trying to restart the stream\n"));
}
}
}
......@@ -336,6 +312,7 @@ void audio_process(struct audio *audio)
break;
}
}
audio->save_data = false;
}
/**
......@@ -348,21 +325,26 @@ void audio_process(struct audio *audio)
*/
void audio_finish(struct audio *audio, bool reset)
{
long to_finish_us;
float period_us;
struct timeval m_end, m_len, av_diff, frame_period;
D6(fprintf(debug_file, "movie start at: %ld:%06ld\n", audio->ctx_a.time_start.tv_sec, audio->ctx_a.time_start.tv_usec));
m_end = audio->ts_video;
D6(fprintf(debug_file, "movie end at: %ld:%06ld\n", m_end.tv_sec, m_end.tv_usec));
m_len = time_sub(&m_end, &audio->ctx_a.time_start);
timersub(&m_end, &audio->ctx_a.time_start, &m_len);
D6(fprintf(debug_file, "movie length: %ld:%06ld\n", m_len.tv_sec, m_len.tv_usec));
audio->m_len = m_len;
audio->ctx_a.time_start = m_end;
assert(audio->get_fpga_time);
// first of all, save all we have in the buffer by the moment
if (audio->ctx_a.sbuffer_pos > audio->ctx_a.sample_time)
record_buffer(audio, AUDIO_PROCESS);
// calculate how many samples we need to save now for the end
struct timeval fpga_tv, sys_tv, audio_last;
struct timeval fpga_tv, sys_tv;
audio->get_fpga_time((const struct audio *)audio, &fpga_tv);
gettimeofday(&sys_tv, NULL);
......@@ -371,29 +353,26 @@ void audio_finish(struct audio *audio, bool reset)
D6(fprintf(debug_file, " FPGA time == %ld:%06ld\n", fpga_tv.tv_sec, fpga_tv.tv_usec));
D6(fprintf(debug_file, "AUDIO sys time == %ld:%06ld\n", audio->ctx_a.time_last.tv_sec, audio->ctx_a.time_last.tv_usec););
// audio_last = audio->ctx_a.time_last;
// if (m_len.tv_sec > audio_last.tv_sec) {
// m_len.tv_sec--;
// m_len.tv_usec += 1000000;
// }
// m_len.tv_sec -= audio_last.tv_sec;
// m_len.tv_usec -= audio_last.tv_usec;
// time_normalize(&m_len);
// long to_finish_us = time_to_us(&m_len);
av_diff = time_sub(&m_len, &audio->ctx_a.time_last);
frame_period = us_to_time(audio->frame_period);
av_diff = time_add(&av_diff, &frame_period); // plus duration of the last video frame
long to_finish_us = time_to_us(&av_diff);
float period_us = (1.0 / audio->audio_rate) * 1000000;
// D6(fprintf(debug_file, "... and now we need to save audio for this time: %ld:%06ld - i.e. %06ld usecs\n", m_len.tv_sec, m_len.tv_usec, to_finish_us));
frame_period = us_to_time(audio->frame_period_us);
if (timercmp(&m_len, &audio->ctx_a.time_last, >)) {
timersub(&m_len, &audio->ctx_a.time_last, &av_diff);
fprintf(debug_file, "av_diff: %ld:%06ld\n", av_diff.tv_sec, av_diff.tv_usec);
timeradd(&av_diff, &frame_period, &av_diff); // plus duration of the last video frame
to_finish_us = time_to_us(&av_diff);
D6(fprintf(debug_file, "... and now we need to save audio for this time: %ld:%06ld - i.e. %06ld usecs\n", av_diff.tv_sec, av_diff.tv_usec, to_finish_us));
} else {
// audio is ahead of video, we do not need to save any additional audio frames
timersub(&audio->ctx_a.time_last, &m_len, &av_diff);
D6(fprintf(debug_file, "audio/video difference: -%ld:%06ld\n", av_diff.tv_sec, av_diff.tv_usec));
to_finish_us = 0;
}
period_us = (1.0 / audio->audio_rate) * 1000000;
if (to_finish_us > period_us) {
double s = audio->audio_rate;
s /= 1000.0;
s *= to_finish_us;
s /= 1000.0;
audio->ctx_a.rem_samples = (long) s;
// from the state->tv_video_start to ctx_a.time_last (with FPGA time recalculation)
do {
fprintf(debug_file, "process remaining %ld samples\n", audio->ctx_a.rem_samples);
audio_process(audio);
......@@ -463,6 +442,7 @@ static void audio_deinit(struct audio *audio)
snd_pcm_close(audio->ctx_a.capture_hnd);
free(audio->ctx_a.sbuffer);
audio->ctx_a.sbuffer = NULL;
audio->ctx_a.sbuffer_pos = 0;
gettimeofday(&tv, NULL);
D4(fprintf(debug_file, "audio deinitialized at %ld:%06ld\n", tv.tv_sec, tv.tv_usec));
......@@ -479,14 +459,14 @@ static bool skip_audio(struct audio *audio, snd_pcm_uframes_t frames)
bool ret_val = false;
snd_pcm_uframes_t skip;
if (audio->ctx_a.audio_skip_samples != 0) {
D5(fprintf(debug_file, "skip_samples = %lld, available samples = %ld\n", audio->ctx_a.audio_skip_samples, frames));
if (audio->ctx_a.audio_skip_samples >= frames) {
audio->ctx_a.audio_skip_samples -= frames;
if (audio->audio_skip_samples != 0) {
D5(fprintf(debug_file, "skip_samples = %lu, available samples = %ld\n", audio->audio_skip_samples, frames));
if (audio->audio_skip_samples >= frames) {
audio->audio_skip_samples -= frames;
skip = frames;
} else {
skip = audio->ctx_a.audio_skip_samples;
audio->ctx_a.audio_skip_samples = 0;
skip = audio->audio_skip_samples;
audio->audio_skip_samples = 0;
}
snd_pcm_forward(audio->ctx_a.capture_hnd, skip);
ret_val = true;
......@@ -494,3 +474,57 @@ static bool skip_audio(struct audio *audio, snd_pcm_uframes_t frames)
return ret_val;
}
/**
* Convert the number of audio frames given to the number of bytes.
* @param audio pointer to a structure containing audio parameters and buffers
* @return number of bytes in audio frames
*/
static long frames_to_bytes(const struct audio *audio, long frames)
{
return frames * audio->audio_channels * (snd_pcm_format_physical_width(audio->ctx_a.audio_format) / 8);
}
/**
* Record data from audio buffer to storage media. Audio frames are recorded in SAMPLE_TIME chunks and this function
* assumes that the buffer contains at least this amount of data.
* @param audio pointer to a structure containing audio parameters and buffers
*/
static void record_buffer(struct audio *audio, int opt)
{
void *_buf;
long _buf_len;
long frames;
long rem_frames;
_buf = audio->ctx_a.sbuffer;
rem_frames = audio->ctx_a.sbuffer_pos;
while (rem_frames >= audio->ctx_a.read_frames || opt == AUDIO_LAST_CHUNK) {
frames = audio->ctx_a.read_frames;
if (opt == AUDIO_LAST_CHUNK)
frames = rem_frames;
_buf_len = frames_to_bytes(audio, frames);
audio->write_samples(audio, _buf, _buf_len, frames);
_buf += _buf_len;
rem_frames -= frames;
float tr = 1.0 / audio->audio_rate;
float l = tr * audio->audio_samples;
unsigned long s = (unsigned long) l;
l -= s;
l *= 1000000;
unsigned long us = (unsigned long) l;
audio->ctx_a.time_last.tv_sec = s;
audio->ctx_a.time_last.tv_usec = us;
D6(fprintf(debug_file, "sound time %lu:%06lu, recorded frames: %ld, frames: %ld, remaining frames: %ld\n",
s, us, audio->audio_samples, frames, rem_frames));
opt = AUDIO_NONE;
}
if (rem_frames > 0) {
// move remaining data to the beginning of the buffer and record it on next iteration
_buf_len = frames_to_bytes(audio, rem_frames);
memcpy(audio->ctx_a.sbuffer, _buf, _buf_len);
D6(fprintf(debug_file, "copy remaining %ld bytes to the beginning of audio buffer\n", _buf_len));
}
audio->ctx_a.sbuffer_pos = rem_frames;
}
src/camogm_audio.h
View file @ 313f638e
......@@ -25,8 +25,8 @@
#include <sys/time.h>
#include <alsa/asoundlib.h>
#define SAMPLE_RATE 44100
#define SAMPLE_CHANNELS 2
#define SAMPLE_RATE 44100 ///< default sampling rate
#define SAMPLE_CHANNELS 2 ///< default number of audio channels
#define SAMPLE_TIME 200 ///< restrict ALSA to have this period, in milliseconds
#define BUFFER_TIME 1000 ///< approximate ALSA buffer duration, in milliseconds
#define DEFAULT_SND_DEVICE "plughw:0,0"
......@@ -36,27 +36,34 @@
#define AUDIO_RATE_MAX 44100
#define DEFAULT_AUDIO_VOLUME 0xffff
/**
* @brief Audio recording context related to stream management.
* Members of this structure should not be used outside audio module.
*/
struct context_audio {
char *sbuffer; ///< buffer for audio samples
long sbuffer_len; ///< the length of samples buffer in samples
long sbuffer_len; ///< total length of audio buffer, in audio frames
long sbuffer_pos; ///< pointer to current write position in audio buffer, in frames
long read_frames; ///< read granularity, in frames
long sample_time; ///< duration of one chunk of audio data, in ms
struct timeval time_start; ///< start time, set only when stream starts and updated with each new file
struct timeval time_last; ///< calculated time of last audio sample (this value is not taken from ALSA)
long rem_samples; ///< remaining samples
int begin_of_stream_with_audio; ///<
int audio_trigger; ///< indicates the beginning of audio recording to make some initial set ups
long long audio_skip_samples; ///<
snd_pcm_format_t audio_format; ///< format of audio samples as defined in 'enum snd_pcm_format_t'
snd_pcm_t *capture_hnd; ///< ALSA PCM handle
};
/**
* @brief Various parameters related to audio recording.
*/
struct audio {
int audio_enable; ///< flag indicating that audio is enabled
int audio_rate; ///< sample rate, in Hz
int audio_channels; ///< number of channels
int audio_volume; ///< volume set in range [0..0xFFFF]
int sync_port; ///< synch audio stream to this sensor port
int set_audio_enable; ///< temporary storage for new value
int set_audio_rate; ///< temporary storage for new value
......@@ -74,27 +81,24 @@ struct audio {
struct timeval ts_audio; ///< time stamp when audio stream started
struct timeval ts_video; ///< time stamp of each new frame
struct timeval ts_video_start; ///< time stamp of starting video frame
int frame_period; ///< video frame period, in microseconds
snd_pcm_format_t audio_format; ///< format of audio samples as defined in 'enum snd_pcm_format_t'
int frame_period_us; ///< video frame period measured for #sync_port, in microseconds
unsigned long audio_skip_samples; ///< skip this number audio frames to sync to video
int begin_of_stream_with_audio; ///< flag indicating that A/V sync is in progress
int audio_trigger; ///< indicates the beginning of audio recording to make some initial set ups
bool save_data; ///< flag indicating that audio data should be recorded, otherwise audio frames should be
///< stored in buffer for delayed recording
unsigned int sleep_period_us; ///< sleep period between frames while processing audio stream, in microseconds
void (*get_fpga_time)(const struct audio *audio, struct timeval *tv);//< callback function which can get FPGA time
int (*write_samples)(struct audio *audio, void *buff, long len, long slen); ///< callback function which actually write data to file, this must be set
///< in the camogm_init_* function when appropriate format is selected
// === debug ===
struct timeval sf_timediff; // system to fpga time difference at the beginning of the stream
struct timeval m_len;
struct timeval sys_fpga_timediff;
int avail_samples;
long calc_frames; // calculated number of frames by current video frame
struct timeval prev_ts_video;
long long skip_samples;
// === end of debug ===
};
void audio_init(struct audio *audio, bool restart);
void audio_start(struct audio *audio);
void audio_init_hw(struct audio *audio, bool restart);
void audio_init_sw(struct audio *audio, bool restart, int frames);
void audio_process(struct audio *audio);
void audio_finish(struct audio *audio, bool reset);
void audio_set_volume(int nvolume);
unsigned long audio_get_hw_buffer_max(void);
#endif /* _CAMOGM_AUDIO_H */
src/camogm_mov.c
View file @ 313f638e
......@@ -26,11 +26,7 @@
#include <sys/types.h>
#include <assert.h>
// for debug only
#include <math.h>
#include "camogm_mov.h"
#include "thelper.h"
/** @brief QuickTime header length (w/o index tables) enough to accommodate static data */
#define QUICKTIME_MIN_HEADER 0x300
......@@ -228,9 +224,9 @@ int camogm_frame_mov(camogm_state *state)
/**
* Write audio samples to file.
* @param[in] buff pointer to buffer containing audio samples
* @param[in] buff pointer to buffer containing audio frames
* @param[in] len the size of buffer, in bytes
* @param[in] slen the number of audio samples in buffer
* @param[in] slen the number of audio frames in buffer
* @return 0 if data was recorded successfully and negative error code otherwise
*/
static int camogm_audio_mov(struct audio *audio, void *buff, long len, long slen)
......@@ -240,7 +236,7 @@ static int camogm_audio_mov(struct audio *audio, void *buff, long len, long slen
ssize_t wr_len;
camogm_state *state = container_of(audio, camogm_state, audio);
D6(fprintf(debug_file, "write audio sample, len = %ld, slen = %ld\n", len, slen));
D6(fprintf(debug_file, "write audio chunk, len = %ld, slen = %ld\n", len, slen));
wr_len = write(state->ivf, buff, len);
if (wr_len < len) {
......@@ -289,19 +285,15 @@ int camogm_end_mov(camogm_state *state)
q_template, // string containing header template
state->ivf, // output file descriptor (opened)
state->width, // width in pixels
state->height,
state->height, // height in pixels
state->frameno, // the number of image frames
state->frame_period[port] / (1000000 / timescale),
state->frames_per_chunk,
0, // frame size - will look in the table
(int)((float)timescale / (state->timescale)),
// state->frame_lengths, // array of frame lengths to build an index
NULL, // array of frame lengths to build an index
state->frame_data_start
);
// === debug code ===
fprintf(debug_file, "total # of video frames: %d, total # of audio samples: %ld\n", state->frameno, state->audio.audio_samples);
// === end of debug ===
close(state->ivf);
state->ivf = -1;
// free memory used for index
......@@ -640,11 +632,6 @@ int quicktime_template_parser( camogm_state *state,
iFileLen = strlen(iFile);
lseek(ofd, 0, SEEK_SET);
// === debug ===
struct timeval m_len = state->audio.m_len; // duration of movie
fprintf(debug_file, "frameno: %d, duration: %ld:%06ld, audio_samples: %ld\n", state->frameno, m_len.tv_sec, m_len.tv_usec, state->audio.audio_samples);
// === ebd of debug ===
audio_timescale = state->audio.audio_rate;
audio_rate = audio_timescale; // QuickTime defines sample rate as unsigned 16.16 fixed-point number
audio_rate <<= 16;
......
src/thelper.c
View file @ 313f638e
......@@ -51,36 +51,3 @@ int time_comp(struct timeval *t1, struct timeval *t2)
}
return -1;
}
/**
* Subtract one time value from another and return the difference
* @param tv1 time value to subtract from
* @param tv2 time value to be subtracted
* @return tv1 - tv2
*/
struct timeval time_sub(const struct timeval *tv1, const struct timeval *tv2)
{
struct timeval ret_val = *tv1;
ret_val.tv_sec -= 1;
ret_val.tv_usec += 1000000;
ret_val.tv_sec -= tv2->tv_sec;
ret_val.tv_usec -= tv2->tv_usec;
time_normalize(&ret_val);
return ret_val;
}
/**
* Add one time value to another and return the sum
*/
struct timeval time_add(const struct timeval *tv1, const struct timeval *tv2)
{
struct timeval ret_val = *tv1;
ret_val.tv_sec += tv2->tv_sec;
ret_val.tv_usec += tv2->tv_usec;
time_normalize(&ret_val);
return ret_val;
}
src/thelper.h
View file @ 313f638e
......@@ -26,8 +26,6 @@
void time_normalize(struct timeval *tv);
int time_comp(struct timeval *t1, struct timeval *t2);
struct timeval time_sub(const struct timeval *tv1, const struct timeval *tv2);
struct timeval time_add(const struct timeval *tv1, const struct timeval *tv2);
/**
* Convert time represented by timeval structure to time in microseconds
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment