Commit 07e65a43 authored by Mikhail Karpenko's avatar Mikhail Karpenko
Browse files

Use delta between FPGA time and ALSA time for SR timestamps 

ALSA reports monotonic timestamps regardles of the time stamps type set
by snd_pcm_params_set_tstamp_type(). Calculate delta between FPGA time
and ALSA time when audio stream starts and use this delta to correct
time stamps in SR packets.
parent 58b954f2

src/audio.cpp

+91 −111
Original line number Diff line number Diff line
@@ -52,8 +52,9 @@ 
	#define D2(s_port, a)

#endif



#define SAMPLE_TIME	20	                                        // in milliseconds

#define BUFFER_TIME	1000                                        // in milliseconds

#define SAMPLE_TIME	              20	                        //< restrict ALSA to have this period, in milliseconds

#define BUFFER_TIME	              1000                          //< approximate buffer duration for ALSA, in milliseconds

#define LEN                       1200                          //< the size of data buffer for RTP packet, in bytes



using namespace std;
@@ -92,11 +93,13 @@ Audio::Audio(int port, bool enable, Parameters *pars, int sample_rate, int chann 
	D(sensor_port, cerr << "sbuffer_len == " << sbuffer_len << endl);

	_ptype = 97;

	sbuffer = NULL;

	packet_buffer = NULL;



	if (enable) {

		// open ALSA for capturing

		int err;

		sbuffer = new short[sbuffer_len * 2 * _channels];

		packet_buffer = new unsigned char[LEN + 20];

		bool init_ok = false;

		while (true) {

			if ((err = snd_pcm_open(&capture_handle, "default", SND_PCM_STREAM_CAPTURE, 0)) < 0)
@@ -130,6 +133,12 @@ Audio::Audio(int port, bool enable, Parameters *pars, int sample_rate, int chann 
				break;

			if ((err = snd_pcm_sw_params_set_tstamp_mode(capture_handle, sw_params, SND_PCM_TSTAMP_ENABLE)) < 0)

				break;

			/* SND_PCM_TSTAMP_TYPE_GETTIMEOFDAY for some reason does not produce time stamps equal to system time,

			 * so stick with monotonic time stamps and apply the difference between FPGA time and ALSA time stamps to

			 * the current samples during transmission

			 */

			if ((err = snd_pcm_sw_params_set_tstamp_type(capture_handle, sw_params, SND_PCM_TSTAMP_TYPE_MONOTONIC)) < 0)

				break;

			if ((err = snd_pcm_sw_params(capture_handle, sw_params)) < 0)

				break;

			init_ok = true;
@@ -157,12 +166,13 @@ Audio::~Audio(void) { 
	}

	if (sbuffer != NULL)

		delete[] sbuffer;

	if (packet_buffer != NULL)

		delete[] packet_buffer;

}



void Audio::set_capture_volume(int nvolume) {

	if (_present == false)

		return;

//	int err;

	snd_mixer_t *mixer;

	snd_mixer_elem_t *elem;

	snd_mixer_selem_id_t *sid;
@@ -210,6 +220,18 @@ void Audio::set_capture_volume(int nvolume) { 
	snd_mixer_close(mixer);

}



/**

 * @brief Start audio stream if it was enabled

 * The time delta between FPGA time and time stamps reported by ALSA is calculated before stream is started.

 * This delta is applied to time stamps received from ALSA. Previously, this delta was calculated as time difference

 * between FPGA time and system time reported by \e getsystimeofday(), but as for now ALSA reports monotonic

 * time stamps regardless of the time stamp type set by \e snd_pcm_sw_params_set_tstamp_type(). See git commit

 * 58b954f239b695b7deda7a33657841d2c64476ae (or earlier) for previous implementation.

 * @param   ip   ip/port pair for socket

 * @param   port ip/port pair for socket

 * @param   ttl  ttl value for socket

 * @return  None

 */

void Audio::Start(string ip, long port, int ttl) {

	if (!_present)

		return;
@@ -220,63 +242,9 @@ void Audio::Start(string ip, long port, int ttl) { 
	snd_pcm_prepare(capture_handle);

	snd_pcm_reset(capture_handle);



	// get FPGA/sys time delta

//	Parameters *params = Parameters::instance();

	unsigned long write_data[6];

	write_data[0] = FRAMEPARS_GETFPGATIME;

	write_data[1] = 0;

	params->write(write_data, sizeof(unsigned long) * 2);

	struct timeval tv_1;

	tv_1.tv_sec = params->getGPValue(G_SECONDS);

	tv_1.tv_usec = params->getGPValue(G_MICROSECONDS);



	struct timeval tv_sys;

	gettimeofday(&tv_sys, NULL);



	params->write(write_data, sizeof(unsigned long) * 2);

	struct timeval tv_2;

	tv_2.tv_sec = params->getGPValue(G_SECONDS);

	tv_2.tv_usec = params->getGPValue(G_MICROSECONDS);



	unsigned long delta = tv_2.tv_sec - tv_1.tv_sec;

	delta *= 1000000;

	delta += tv_2.tv_usec;

	delta -= tv_1.tv_usec;

	delta /= 2;

	struct timeval tv_fpga;

	tv_fpga.tv_sec = tv_1.tv_sec;

	tv_fpga.tv_usec = tv_1.tv_usec + delta;

	tv_fpga.tv_sec += tv_fpga.tv_usec / 1000000;

	tv_fpga.tv_usec = tv_fpga.tv_usec % 1000000;



	bool fpga_gt = false;

	if (tv_fpga.tv_sec > tv_sys.tv_sec)

		fpga_gt = true;

	if (tv_fpga.tv_sec == tv_sys.tv_sec)

		if (tv_fpga.tv_usec > tv_sys.tv_usec)

			fpga_gt = true;

	struct timeval *tv_b = &tv_sys;

	struct timeval *tv_s = &tv_fpga;

	if (fpga_gt) {

		tv_b = &tv_fpga;

		tv_s = &tv_sys;

	}

	delta_fpga_sys = 0;

	delta_fpga_sys = tv_b->tv_sec - tv_s->tv_sec;

	delta_fpga_sys *= 1000000;

	delta_fpga_sys += tv_b->tv_usec;

	delta_fpga_sys -= tv_s->tv_usec;

	if (fpga_gt)

		delta_fpga_sys = -delta_fpga_sys;



	D2(sensor_port, cerr << "first time = " << tv_1.tv_sec << ":" << tv_1.tv_usec

			<< ", second time = " << tv_2.tv_sec << ":" << tv_2.tv_usec << endl);

	D2(sensor_port, cerr << "FPGA time = " << tv_fpga.tv_sec << ":" << tv_fpga.tv_usec

			<< ", system time = " << tv_sys.tv_sec << ":" << tv_sys.tv_usec << endl);

	D2(sensor_port, cerr << "time delta = " << delta_fpga_sys << endl << endl);

//fprintf(stderr, "first time == %d:%06d; second time == %d:%06d\n", tv_1.tv_sec, tv_1.tv_usec, tv_2.tv_sec, tv_2.tv_usec);

//fprintf(stderr, "FPGA time == %d:%06d; system time == %d:%06d\n", tv_fpga.tv_sec, tv_fpga.tv_usec, tv_sys.tv_sec, tv_sys.tv_usec);

//fprintf(stderr, "times delta == %06d\n\n", delta_fpga_sys);

	// get FPGA/ALSA time delta

	delta_fpga_alsa = get_delta_fpga_alsa();

	D2(sensor_port, cerr << "FPGA/ALSA time delta = " << delta_fpga_alsa << " us" << endl);



	RTP_Stream::Start(ip, port, ttl);

}
@@ -314,44 +282,11 @@ long Audio::process(void) { 
				// correct A time to V time for RTCP sync packets

				f_tv.tv_sec -= 1;

				f_tv.tv_usec += 1000000;

				f_tv.tv_sec -= delta_fpga_sys / 1000000;

				f_tv.tv_usec -= delta_fpga_sys % 1000000;

				f_tv.tv_sec -= delta_fpga_alsa / 1000000;

				f_tv.tv_usec -= delta_fpga_alsa % 1000000;

				f_tv.tv_sec += f_tv.tv_usec / 1000000;

				f_tv.tv_usec = f_tv.tv_usec % 1000000;

/*

				if(is_first) {

					struct timeval tv;

					gettimeofday(&tv, NULL);

					is_first = false;

					fprintf(stderr, "AUDIO first with time: %d:%06d at: %d:%06d\n", f_tv.tv_sec, f_tv.tv_usec, tv.tv_sec, tv.tv_usec);

					sec = f_tv.tv_sec;

				} else {

					if(sec != f_tv.tv_sec) {

						

						sec = f_tv.tv_sec;

					}

				}

*/

			}

/*

			snd_pcm_status(capture_handle, status);

			snd_pcm_status_get_tstamp(status, &ts);

			avail = (unsigned long)snd_pcm_status_get_avail(status);

			struct timeval tv_real;

			gettimeofday(&tv_real, NULL);

			struct timeval t_delta = {0, 0};

			if(ts_old.tv_sec != 0 && ts_old.tv_usec != 0) {

				long td = ts.tv_sec - ts_old.tv_sec;

				td *= 1000000;

				td += ts.tv_usec;

				td -= ts_old.tv_usec;

				t_delta.tv_sec = td / 1000000;

				t_delta.tv_usec = td % 1000000;

			}

fprintf(stderr, "status timestamp == %d:%06d at %d:%06d; delta == %d:%06d, slen == %d; avail == %d\n", ts.tv_sec, ts.tv_usec, tv_real.tv_sec, tv_real.tv_usec, t_delta.tv_sec, t_delta.tv_usec, slen, avail);

			ts_old = ts;

*/

			// --==--

			ret += slen;

			process_send(slen);

			// check again that buffer is not empty
@@ -376,18 +311,12 @@ fprintf(stderr, "status timestamp == %d:%06d at %d:%06d; delta == %d:%06d, slen 
}



long Audio::process_send(long sample_len) {

	static unsigned short packet_num = 0;

	unsigned short pnum;



	void *m = (void *) sbuffer;



	int i;

	long offset = 0;

#define LEN 1200

	static unsigned char *d = NULL;

	if (d == NULL) {

		d = (unsigned char *) malloc(LEN + 20);

	}

	int to_send = sample_len * 2 * _channels;

	int size = to_send;

	long count = 0;
@@ -409,27 +338,78 @@ long Audio::process_send(long sample_len) { 
		timestamp_rtcp += ts_delta;

		if (timestamp_rtcp > 0xFFFFFFFF)

			timestamp_rtcp &= 0xFFFFFFFF;

		//

		packet_num++;

		pnum = htons(packet_num);

		count += i;

		// fill RTP header

		d[0] = 0x80;

		packet_buffer[0] = 0x80;

		if (count >= size)

			d[1] = _ptype + 0x80;

			packet_buffer[1] = _ptype + 0x80;

		else

			d[1] = _ptype;

		memcpy((void *) &d[2], (void *) &pnum, 2);

		memcpy((void *) &d[4], (void *) &ts, 4);

		memcpy((void *) &d[8], (void *) &SSRC, 4);

			packet_buffer[1] = _ptype;

		memcpy((void *) &packet_buffer[2], (void *) &pnum, 2);

		memcpy((void *) &packet_buffer[4], (void *) &ts, 4);

		memcpy((void *) &packet_buffer[8], (void *) &SSRC, 4);

		// fill data

		memcpy((void *) &d[12], (void *) ((char *) m + offset), i);

		memcpy((void *) &packet_buffer[12], (void *) ((char *) m + offset), i);

		offset += i;

		// send packet

		rtp_packets++;

		rtp_octets += i; // data + MJPEG header

		rtp_socket->send((void *) d, i + 12);

		rtp_octets += i;                                        // total amount of payload data transmitted

		rtp_socket->send((void *)packet_buffer, i + 12);

	}



	return sample_len;

}



/**

 * @brief Calculate the difference between FPGA time and time stamps reported by ALSA.

 * This function gets two time stamps from FPGA and one time stamp from ALSA driver between the first two,

 * then calculates mean value of the two FPGA samples and finds the delta between the mean value and

 * ALSA time stamp. This value is applied to each time stamp in sound data packet to compensate the difference.

 * @return   The time delta in microseconds

 */

long long Audio::get_delta_fpga_alsa(void)

{

	snd_pcm_status_t *pcm_status;

	snd_timestamp_t snd_ts;

	struct timeval snd_tv;

	struct timeval tv_1, tv_2;

	struct timeval delta_tv;

	unsigned long delta_us;

	long long ret_val;

	bool fpga_greater = false;



	snd_pcm_status_alloca(&pcm_status);

	tv_1 = params->get_fpga_time();

	snd_pcm_status(capture_handle, pcm_status);

	snd_pcm_status_get_tstamp(pcm_status, &snd_ts);

	tv_2 = params->get_fpga_time();

	snd_tv.tv_sec = snd_ts.tv_sec;

	snd_tv.tv_usec = snd_ts.tv_usec;

	delta_us = time_delta_us(tv_2, tv_1);

	delta_us /= 2;

	tv_1 = time_plus_us(tv_1, delta_us);



	if (tv_1.tv_sec > snd_tv.tv_sec)

		fpga_greater = true;

	else if (tv_1.tv_sec == snd_tv.tv_sec)

		if (tv_1.tv_usec > snd_tv.tv_usec)

			fpga_greater = true;

	struct timeval *tv_g = &snd_tv;                             // greater time value

	struct timeval *tv_s = &tv_1;                               // smaller time value

	if (fpga_greater) {

		tv_g = &tv_1;

		tv_s = &snd_tv;

	}

	delta_tv = time_minus(*tv_g, *tv_s);

	ret_val = (long long)delta_tv.tv_sec * 1000000 + (long long)delta_tv.tv_usec;

	if (fpga_greater) {

		ret_val = -ret_val;

	}



	D2(sensor_port, cerr << "ALSA time stamp = " << snd_ts.tv_sec << ":" << snd_ts.tv_usec << endl);

	D2(sensor_port, cerr << "delta_tv = " << delta_tv.tv_sec << ":" << delta_tv.tv_usec << endl);



	return ret_val;

}

src/audio.h

+8 −4
Original line number Diff line number Diff line
@@ -52,10 +52,10 @@ public: 
	void Stop(void);

protected:

//	int fd;

	snd_pcm_t *capture_handle;                                  // PCM handle, returned from snd_pcm_open

	short *sbuffer;

	snd_pcm_t *capture_handle;                                  //< PCM handle, returned from snd_pcm_open

	short *sbuffer;                                             //< buffer for sound data

	long sbuffer_len;

	bool _present;                                              // flag indicating that audio interface has been initialized

	bool _present;                                              //< flag indicating that audio interface has been initialized

	long _sample_rate;

	long _channels;

	long _volume;
@@ -66,9 +66,13 @@ protected: 
	void set_capture_volume(int volume);



	uint64_t timestamp_rtcp;

	long delta_fpga_sys;	                                    // A/V clocks delta for RTCP

	long delta_fpga_sys;	                                    //< A/V clocks delta for RTCP

//	bool is_first;

//	bool is_first2;

	long long delta_fpga_alsa;                                  //< time delta between FPGA time and time reported by ALSA, in microseconds

	unsigned char *packet_buffer;                               //< buffer for RTP packet data (header plus PCM samples)

private:

	long long get_delta_fpga_alsa(void);                        //< return time delta between FPGA time and time stamps reported by ALSA, in microseconds

};



#endif // _AUDIO__H_

src/helper.h

+19 −4
Original line number Diff line number Diff line 
/**

 * @file FILENAME

 * @brief BRIEF DESCRIPTION

 * @copyright Copyright (C) YEAR Elphel Inc.

 * @file helper.h

 * @brief Various time helper functions to work with \e timeval structures

 * @copyright Copyright (C) 2017 Elphel Inc.

 * @author AUTHOR <EMAIL>

 *

 * @par License:
@@ -54,4 +54,19 @@ inline struct timeval time_minus(struct timeval tv_1, const struct timeval &tv_2 
	return tv_1;

}



#endif
 
/**

 * @brief Add microseconds to a value represented by \e timeval structure

 * @param   tv   time to which microseconds should be added

 * @param   us   the value of microseconds

 * @return  The sum of two values

 */

inline struct timeval time_plus_us(struct timeval &tv, unsigned long us)

{

	tv.tv_usec += us;

	tv.tv_sec += tv.tv_usec / 1000000;

	tv.tv_usec = tv.tv_usec % 1000000;



	return tv;

}



#endif /* __H_HELPER__ */

src/parameters.cpp

+16 −0
Original line number Diff line number Diff line
@@ -119,3 +119,19 @@ bool Parameters::daemon_enabled(void) { 
void Parameters::setPValue(unsigned long *val_array, int count) {

	this->write(val_array, sizeof(unsigned long) * count);

}



/**

 * @brief Get current FPGA time

 * @return   Time value in \e timeval structure

 */

struct timeval Parameters::get_fpga_time(void)

{

	struct timeval tv;

	unsigned long write_data[] = {FRAMEPARS_GETFPGATIME, 0};



	write(write_data, sizeof(unsigned long) * 2);

	tv.tv_sec = getGPValue(G_SECONDS);

	tv.tv_usec = getGPValue(G_MICROSECONDS);



	return tv;

}

src/parameters.h

+1 −0
Original line number Diff line number Diff line
@@ -48,6 +48,7 @@ public: 
	bool daemon_enabled(void);

	void setPValue(unsigned long *val_array, int count);

	inline int get_port_num() const {return sensor_port;}

	struct timeval get_fpga_time(void);

protected:

//	static Parameters *_parameters;