Fixing corrupted images after restart (091f52ba) · Commits · Elphel / elphel-apps-camogm

src/camogm.c

+9 −5

src/camogm.h

+2 −1

Original line number	Original line	Diff line number	Diff line
	@@ -31,6 +31,7 @@
	#include <malloc.h> // debugging		#include <malloc.h> // debugging

	#define NUM_WRITER_THREADS 1+0 // 2		#define NUM_WRITER_THREADS 1+0 // 2

	#define NUM_NEED_EMPTY (NUM_WRITER_THREADS + 1) // run main thread to prepage more pages if >= empty		#define NUM_NEED_EMPTY (NUM_WRITER_THREADS + 1) // run main thread to prepage more pages if >= empty
	#define NUM_NEED_NEMPTY (NUM_WRITER_THREADS + 0) // wake up main thread if there are at least this number of empty pages		#define NUM_NEED_NEMPTY (NUM_WRITER_THREADS + 0) // wake up main thread if there are at least this number of empty pages
	#define NUM_FREE_MANY (NUM_WRITER_THREADS + 1) // Set many if get_num_empty() < NUM_FREE_MANY		#define NUM_FREE_MANY (NUM_WRITER_THREADS + 1) // Set many if get_num_empty() < NUM_FREE_MANY
	@@ -112,7 +113,7 @@
	#define FILE_CHUNKS_NUM 8		#define FILE_CHUNKS_NUM 8
	/** @brief Number of chunk pages to overlap preparation with SSD raw write */		/** @brief Number of chunk pages to overlap preparation with SSD raw write */
	#define FILE_CHUNKS_PAGES 4 // 2		#define FILE_CHUNKS_PAGES 4 // 2
	#define FILE_CHUNKS_TOTAL ((FILE_CHUNKS_NUM) * (FILE_CHUNKS_PAGES))		#define FILE_CHUNKS_TOTAL ((FILE_CHUNKS_NUM) * (FILE_CHUNKS_PAGES)) // never used
	#define JPEG_MARKER_LEN 2 ///< The size in bytes of JPEG marker		#define JPEG_MARKER_LEN 2 ///< The size in bytes of JPEG marker
	#define JPEG_TRAILER_LEN 2 ///< The size in bytes of JPEG trailer		#define JPEG_TRAILER_LEN 2 ///< The size in bytes of JPEG trailer
	#define CIRCBUF_ALIGNMENT_SIZE 32 ///< Align of CIRCBUF entries		#define CIRCBUF_ALIGNMENT_SIZE 32 ///< Align of CIRCBUF entries

src/camogm_align.c

+14 −14

src/camogm_jpeg.c

+13 −0

Original line number	Original line	Diff line number	Diff line
	@@ -700,6 +700,7 @@ int camogm_start_jpeg(camogm_state *state)
	params->stat_update = get_fpga_time64(state->fd_fparmsall[0], 0);		params->stat_update = get_fpga_time64(state->fd_fparmsall[0], 0);
	D1(fprintf(debug_file, "Started %d writer threads\n", NUM_WRITER_THREADS));		D1(fprintf(debug_file, "Started %d writer threads\n", NUM_WRITER_THREADS));
	// check if immediately exited because of open error? if (params->exit_write_threads) - now and while waiting for free buffer ?		// check if immediately exited because of open error? if (params->exit_write_threads) - now and while waiting for free buffer ?
			reset_segments (state, true, params->chunk_page_prep); // reset carry buffer (no end of the previous non-existing frame)
	}		}
	return 0;		return 0;
	}		}
	@@ -956,6 +957,7 @@ int camogm_end_jpeg(camogm_state *state)
	struct writer_params *params = &state->writer_params;		struct writer_params *params = &state->writer_params;
	int seg_page = params->chunk_page_prep;		int seg_page = params->chunk_page_prep;
	uint64_t disk_size = (params->lba_end - params->lba_start) * PHY_BLOCK_SIZE;		uint64_t disk_size = (params->lba_end - params->lba_start) * PHY_BLOCK_SIZE;
			struct iovec glue_vec_tmp; // temprary save pointer for swapping
	if (state->rawdev_op) {		if (state->rawdev_op) {
	bytes = params->glue_carry_vec.iov_len; // is not modified by writer threads		bytes = params->glue_carry_vec.iov_len; // is not modified by writer threads
	if (bytes > 0) {		if (bytes > 0) {
	@@ -1023,8 +1025,19 @@ int camogm_end_jpeg(camogm_state *state)
	#endif		#endif
	pthread_mutex_unlock(&params->writer_mutex);		pthread_mutex_unlock(&params->writer_mutex);
	// prepare last segment, increment segment page		// prepare last segment, increment segment page
			/*
			// How it was - pointer params->data_segments[seg_page][SEGMENT_GLUE].iov_base was lost.
	params->data_segments[seg_page][SEGMENT_GLUE].iov_base = params->glue_carry_vec.iov_base;		params->data_segments[seg_page][SEGMENT_GLUE].iov_base = params->glue_carry_vec.iov_base;
	params->data_segments[seg_page][SEGMENT_GLUE].iov_len = params->glue_carry_vec.iov_len;		params->data_segments[seg_page][SEGMENT_GLUE].iov_len = params->glue_carry_vec.iov_len;
			*/

			glue_vec_tmp.iov_base = params->glue_carry_vec.iov_base; // Save previous params->glue_carry_vec to local glue_vec
			glue_vec_tmp.iov_len = params->glue_carry_vec.iov_len;
			params->glue_carry_vec.iov_base = params->data_segments[seg_page][SEGMENT_GLUE].iov_base; // point params->glue_carry_vec.iov_base to now unneeded buffer (not to get it lost)
			params->glue_carry_vec.iov_len = 0; // params->data_segments[seg_page][SEGMENT_GLUE].iov_len; // reset length to 0
			params->data_segments[seg_page][SEGMENT_GLUE].iov_base = glue_vec_tmp.iov_base; // Set current SEGMENT_GLUE from previous params->glue_carry_vec
			params->data_segments[seg_page][SEGMENT_GLUE].iov_len = glue_vec_tmp.iov_len; // icluding length

	params->data_segments[seg_page][SEGMENT_FIRST].iov_len = 0; // check iov_len first before using iov_base		params->data_segments[seg_page][SEGMENT_FIRST].iov_len = 0; // check iov_len first before using iov_base
	params->data_segments[seg_page][SEGMENT_SECOND].iov_len = 0; // check iov_len first before using iov_base		params->data_segments[seg_page][SEGMENT_SECOND].iov_len = 0; // check iov_len first before using iov_base
	params->segment_pos[seg_page] = params->next_segment_pos;		params->segment_pos[seg_page] = params->next_segment_pos;

Original line number	Original line	Diff line number	Diff line
	@@ -416,7 +416,7 @@ int camogm_start(camogm_state *state, bool restart)
	// Check/set circbuf read pointer		// Check/set circbuf read pointer
	D3(fprintf(debug_file, "1: state->cirbuf_rp=0x%x\n", state->cirbuf_rp[chn]));		D3(fprintf(debug_file, "1: state->cirbuf_rp=0x%x\n", state->cirbuf_rp[chn]));
	D3(fprintf(debug_file, "1a: compressed frame number = %li\n", lseek(state->fd_circ[chn], LSEEK_CIRC_GETFRAME, SEEK_END)));		D3(fprintf(debug_file, "1a: compressed frame number = %li\n", lseek(state->fd_circ[chn], LSEEK_CIRC_GETFRAME, SEEK_END)));
	D3(fprintf(debug_file, "1a: compressed frame number = %li\n", lseek(state->fd_circ[chn], LSEEK_CIRC_GETFRAME, SEEK_END)));		// D3(fprintf(debug_file, "1a: compressed frame number = %li\n", lseek(state->fd_circ[chn], LSEEK_CIRC_GETFRAME, SEEK_END)));

	if ( (restart) \|\| // restart - always flush buffer (will loose frames, but not continue broken frames)		if ( (restart) \|\| // restart - always flush buffer (will loose frames, but not continue broken frames)
	(state->cirbuf_rp[chn] < 0) \|\|		(state->cirbuf_rp[chn] < 0) \|\|
	@@ -638,8 +638,7 @@ int camogm_start(camogm_state *state, bool restart)
	pthread_mutex_lock(&state->mutex);		pthread_mutex_lock(&state->mutex);
	state->prog_state = STATE_RUNNING;		state->prog_state = STATE_RUNNING;
	pthread_mutex_unlock(&state->mutex);		pthread_mutex_unlock(&state->mutex);
	D1(fprintf(debug_file, "Started OK\n"));		D1(fprintf(debug_file, "Started OK @ %07d\n",get_fpga_usec(state->fd_fparmsall[0], 0))); // D6(fprintf(debug_file, " ==> %d @ %07d\n",rslt,get_fpga_usec(state->fd_fparmsall[0], 0)));

	return 0;		return 0;
	}		}

	@@ -820,7 +819,7 @@ int sendImageFrame(camogm_state *state)
	state->packetchunks[state->chunk_index ].bytes = state->head_size[port] - 2;		state->packetchunks[state->chunk_index ].bytes = state->head_size[port] - 2;
	state->packetchunks[state->chunk_index++].chunk = &(state->jpegHeader[port][2]); // [3] rest of jpeg header		state->packetchunks[state->chunk_index++].chunk = &(state->jpegHeader[port][2]); // [3] rest of jpeg header
	}		}
	} else {		} else { // Tiff always has Exif Data
	D4(fprintf(debug_file, "_8_"));		D4(fprintf(debug_file, "_8_"));
	state->packetchunks[state->chunk_index ].bytes = state->head_size[port];		state->packetchunks[state->chunk_index ].bytes = state->head_size[port];
	state->packetchunks[state->chunk_index++].chunk = state->jpegHeader[port]; // [1] all of jpeg marker		state->packetchunks[state->chunk_index++].chunk = state->jpegHeader[port]; // [1] all of jpeg marker
	@@ -1970,7 +1969,10 @@ int listener_loop(camogm_state *state)
	if ((curr_port >= 0) && (curr_port < SENSOR_PORTS)) {		if ((curr_port >= 0) && (curr_port < SENSOR_PORTS)) {
	state->error_stat[curr_port][rslt]++;		state->error_stat[curr_port][rslt]++;
	} else {		} else {
	D0(fprintf(debug_file, "%s:line %d - * Tried to write statistics for negative port %d, ERR= %d !!! *\n", __FILE__, __LINE__, curr_port, rslt));		for (i = 0; i < SENSOR_PORTS; i++){
			state->error_stat[i][rslt]++;
			}
			// D0(fprintf(debug_file, "%s:line %d - * Tried to write statistics for negative port %d, ERR= %d !!! *\n", __FILE__, __LINE__, curr_port, rslt));
	}		}
	}		}

	@@ -1985,9 +1987,11 @@ int listener_loop(camogm_state *state)
	__FILE__, __LINE__, rslt, state->writer_params.next_segment_pos, get_fpga_usec(state->fd_fparmsall[0], 0)));		__FILE__, __LINE__, rslt, state->writer_params.next_segment_pos, get_fpga_usec(state->fd_fparmsall[0], 0)));
	D2(syslog (LOG_INFO, "%s:%d: Error (code = %d), params->next_segment_pos=%llu @ %07d", \		D2(syslog (LOG_INFO, "%s:%d: Error (code = %d), params->next_segment_pos=%llu @ %07d", \
	__FILE__, __LINE__, rslt, state->writer_params.next_segment_pos, get_fpga_usec(state->fd_fparmsall[0], 0)));		__FILE__, __LINE__, rslt, state->writer_params.next_segment_pos, get_fpga_usec(state->fd_fparmsall[0], 0)));

Original line number	Original line	Diff line number	Diff line
	@@ -122,17 +122,17 @@ size_t remap_vectors(camogm_state state) // , struct iovec chunks)
	int i, data_index, data_pre, num_glue_pages, trim_segment, circbuff_offs;		int i, data_index, data_pre, num_glue_pages, trim_segment, circbuff_offs;
	int gap_size;		int gap_size;
	unsigned char * aligned_data;		unsigned char * aligned_data;
	struct iovec glue_vec;		struct iovec glue_vec_tmp; // temprary save pointer for swapping
	struct iovec * glue_vec_p; // = &glue_vec;		struct iovec * glue_vec_p;
	glue_vec.iov_base = params->glue_carry_vec.iov_base;		glue_vec_tmp.iov_base = params->glue_carry_vec.iov_base; // Save previous params->glue_carry_vec to local glue_vec
	glue_vec.iov_len = params->glue_carry_vec.iov_len;		glue_vec_tmp.iov_len = params->glue_carry_vec.iov_len;
	D7(fprintf(debug_file, "_13c01_:remap_vectors @ %07d\n",get_fpga_usec(state->fd_fparmsall[0], 0)));		D7(fprintf(debug_file, "_13c01_:remap_vectors @ %07d\n",get_fpga_usec(state->fd_fparmsall[0], 0)));
	params->glue_carry_vec.iov_base = params->data_segments[seg_page][SEGMENT_GLUE].iov_base;		params->glue_carry_vec.iov_base = params->data_segments[seg_page][SEGMENT_GLUE].iov_base; // point params->glue_carry_vec.iov_base to now unneeded buffer (not to get it lost)
	params->glue_carry_vec.iov_len = 0; // params->data_segments[seg_page][SEGMENT_GLUE].iov_len;		params->glue_carry_vec.iov_len = 0; // params->data_segments[seg_page][SEGMENT_GLUE].iov_len; // reset length to 0
	params->data_segments[seg_page][SEGMENT_GLUE].iov_base = glue_vec.iov_base;		params->data_segments[seg_page][SEGMENT_GLUE].iov_base = glue_vec_tmp.iov_base; // Set current SEGMENT_GLUE from previous params->glue_carry_vec
	params->data_segments[seg_page][SEGMENT_GLUE].iov_len = glue_vec.iov_len;		params->data_segments[seg_page][SEGMENT_GLUE].iov_len = glue_vec_tmp.iov_len; // icluding length
	params->data_segments[seg_page][SEGMENT_FIRST].iov_len = 0; // check iov_len first before using iov_base		params->data_segments[seg_page][SEGMENT_FIRST].iov_len = 0; // check iov_len first before using iov_base // reset SEGMENT_FIRST to zero/null
	params->data_segments[seg_page][SEGMENT_SECOND].iov_len = 0; // check iov_len first before using iov_base		params->data_segments[seg_page][SEGMENT_SECOND].iov_len = 0; // check iov_len first before using iov_base //
	// calculate total header size		// calculate total header size
	head_bytes = 0;		head_bytes = 0;
	for (i = 0; i < state->chunk_data_index; i++){		for (i = 0; i < state->chunk_data_index; i++){
	@@ -150,9 +150,9 @@ size_t remap_vectors(camogm_state state) // , struct iovec chunks)
	glue_vec_p = &params->data_segments[seg_page][SEGMENT_GLUE];		glue_vec_p = &params->data_segments[seg_page][SEGMENT_GLUE];
	D7(fprintf(debug_file, "_13c03_:remap_vectors, circbuff_offs=0x%x aligned_data=%p data_pre=0x%x @ %07d\n", circbuff_offs, aligned_data, data_pre, get_fpga_usec(state->fd_fparmsall[0], 0)));		D7(fprintf(debug_file, "_13c03_:remap_vectors, circbuff_offs=0x%x aligned_data=%p data_pre=0x%x @ %07d\n", circbuff_offs, aligned_data, data_pre, get_fpga_usec(state->fd_fparmsall[0], 0)));
	D7(fprintf(debug_file, "_13c04_:remap_vectors, state->packetchunks[state->chunk_data_index].bytes=%ld @ %07d\n", state->packetchunks[state->chunk_data_index].bytes, get_fpga_usec(state->fd_fparmsall[0], 0)));		D7(fprintf(debug_file, "_13c04_:remap_vectors, state->packetchunks[state->chunk_data_index].bytes=%ld @ %07d\n", state->packetchunks[state->chunk_data_index].bytes, get_fpga_usec(state->fd_fparmsall[0], 0)));
	if (state->packetchunks[state->chunk_data_index].bytes < data_pre) {		if (state->packetchunks[state->chunk_data_index].bytes < data_pre) { // unlikely - short frame
	// Too short frame data - special treatment with zero data, moving some to carry, and possibly outputing some from the GLUE		// Too short frame data - special treatment with zero data, moving some to carry, and possibly outputing some from the GLUE
	// there is definitely no second data - otherwise the first segment would end at page boundary		// there is definitely no second data - otherwise the first segment would end at page boundary as the buffer itself is page-aligned
	// copy everything to params->data_segments[seg_page][SEGMENT_GLUE], trim to pages, pass residual to		// copy everything to params->data_segments[seg_page][SEGMENT_GLUE], trim to pages, pass residual to
	// params->glue_carry_vec		// params->glue_carry_vec
	// copy header data, no gap		// copy header data, no gap
	@@ -169,7 +169,7 @@ size_t remap_vectors(camogm_state state) // , struct iovec chunks)
	}		}
	} else {		} else {
	head_bytes += data_pre;		head_bytes += data_pre;
	tail_bytes = glue_vec_p->iov_len;		tail_bytes = glue_vec_p->iov_len; //length of previous params->glue_carry_vec
	num_glue_pages = PAGE_PHYS_NUMBER(tail_bytes + head_bytes);		num_glue_pages = PAGE_PHYS_NUMBER(tail_bytes + head_bytes);
	gap_size = num_glue_pages * PAGE_PHYS - tail_bytes - head_bytes;		gap_size = num_glue_pages * PAGE_PHYS - tail_bytes - head_bytes;
	D7(fprintf(debug_file, "_13c06_:remap_vectors, head_bytes=%ld, tail_bytes=%ld, num_glue_pages=%d, gap_size=%d @ %07d\n", head_bytes, tail_bytes, num_glue_pages, gap_size, get_fpga_usec(state->fd_fparmsall[0], 0)));		D7(fprintf(debug_file, "_13c06_:remap_vectors, head_bytes=%ld, tail_bytes=%ld, num_glue_pages=%d, gap_size=%d @ %07d\n", head_bytes, tail_bytes, num_glue_pages, gap_size, get_fpga_usec(state->fd_fparmsall[0], 0)));
	@@ -298,7 +298,7 @@ int init_align_buffers(camogm_state *state)
	params->data_segments[seg_page][SEGMENT_GLUE].iov_len = 0;		params->data_segments[seg_page][SEGMENT_GLUE].iov_len = 0;
	}		}
	params->glue_carry_vec.iov_base = params->glue_buffs[SEGMENTS_PAGES];		params->glue_carry_vec.iov_base = params->glue_buffs[SEGMENTS_PAGES];

Admin message