camogm_read.c 40.4 KB
Newer Older
1
/** @file camogm_read.c
2
 * @brief Provides reading data written to raw device storage and transmitting the data over a socket.
3 4
 * @copyright  Copyright (C) 2016 Elphel, Inc.
 *
5
 * @par <b>License</b>
6 7 8 9 10 11 12 13 14 15 16 17
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

18 19 20 21 22 23
/**
 * @addtogroup SPECIAL_INCLUDES Special includes
 * These defines are needed to use lseek64, strptime and usleep and should be set before includes
 * @{
 */
/** Needed for lseek64 */
24
#define _LARGEFILE64_SOURCE
25
/** Needed for strptime and usleep */
26
#define _XOPEN_SOURCE
27
/** Needed for usleep */
28
#define _XOPEN_SOURCE_EXTENDED
29
/** @} */
30 31 32 33 34 35 36 37 38

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <fcntl.h>
#include <unistd.h>
#include <string.h>
#include <linux/limits.h>
#include <netinet/in.h>
39 40 41
#include <errno.h>
#include <time.h>
#include <ctype.h>
42
#include <math.h>
43
#include <asm/byteorder.h>
44
#include <sys/statvfs.h>
45 46
#include <sys/mman.h>
#include <sys/socket.h>
47
#include <sys/stat.h>
48

49
#include "camogm_read.h"
50
#include "index_list.h"
51

52 53
/** @brief Offset in Exif where TIFF header starts */
#define TIFF_HDR_OFFSET           12
54
/** @brief The date and time format of Exif field */
55
#define EXIF_DATE_TIME_FORMAT     "%Y:%m:%d %H:%M:%S"
56 57
/** @brief The format string used for file parameters reporting. Time and port number are extracted from Exif */
#define INDEX_FORMAT_STR          "port_number=%d;unix_time=%ld;usec_time=%06u;offset=0x%010llx;file_size=%u\n"
58
/** @brief The delimiters used to separate several commands in one command string sent over socket */
59
#define CMD_DELIMITER             "/?"
60
/** @brief The length of a buffer for command string */
61
#define CMD_BUFF_LEN              1024
62
/** @brief The length of a buffer for string formatting */
63
#define SMALL_BUFF_LEN            32
64
/** @brief 64 bit mask to align offsets to 4 kb page boundary */
65
#define PAGE_BOUNDARY_MASK        0xffffffffffffe000
66 67 68 69 70
/** @brief The size of read buffer in bytes. The data will be read from disk in blocks of this size */
#define PHY_BLK_SZ                4096
/** @brief Include or exclude file start and stop markers from resulting file. This must be set to 1 for JPEG files */
#define INCLUDE_MARKERS           1
/** @brief File starting marker on a raw device. It corresponds to SOI JPEG marker */
71
static unsigned char elphelst[] = {0xff, 0xd8};
72
/** @brief File ending marker on a raw device. It corresponds to EOI JPEG marker */
73
static unsigned char elphelen[] = {0xff, 0xd9};
74 75 76 77 78 79 80 81 82
static const struct iovec elphel_st = {
		.iov_base = elphelst,
		.iov_len = sizeof(elphelst)
};
static const struct iovec elphel_en = {
		.iov_base = elphelen,
		.iov_len = sizeof(elphelen)
};

83 84 85
/** @brief X Macro for commands. Add new commands to @e COMMAND_TABLE
 * @{
 */
86 87 88 89 90 91 92 93
#define COMMAND_TABLE \
	X(CMD_BUILD_INDEX, "build_index") \
	X(CMD_GET_INDEX, "get_index") \
	X(CMD_READ_DISK, "read_disk") \
	X(CMD_READ_FILE, "read_file") \
	X(CMD_READ_ALL_FILES, "read_all_files") \
	X(CMD_STATUS, "status")

94
/** @enum socket_commands */
95 96 97 98 99 100 101 102 103 104 105
#define X(a, b) a,
enum socket_commands {
	COMMAND_TABLE
};
#undef X

#define X(a, b) b,
static const char *cmd_list[] = {
	COMMAND_TABLE
};
#undef X
106
/** @} */
107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124

/**
 * @enum match_result
 * @brief The result codes for file markers search function
 * @var match_result::MATCH_FOUND
 * File marker was found in the current data buffer
 * @var match_result::MATCH_NOT_FOUND
 * File marker was not found in the current data buffer
 * @var match_result::MATCH_PARTIAL
 * Only a fraction of a marker was found in the end of the current data buffer
 */
enum match_result {
	MATCH_FOUND         = -1,
	MATCH_NOT_FOUND     = -2,
	MATCH_PARTIAL       = -3
};

/**
125 126 127 128 129 130
 * @enum search_state
 * @brief The state of the program during file search in raw device buffer
 * @var search_state::SEARCH_SKIP
 * Skip data in read buffer
 * @var search_state::SEARCH_FILE_DATA
 * The data in read buffer is a file data with known start offset
131
 */
132 133 134 135
enum search_state {
	SEARCH_SKIP,
	SEARCH_FILE_DATA
};
136

137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198
/**
 * @brief Exif data format table.
 *
 * This table is used to convert Exif data format to its byte length and then calculate the length
 * of a record in Image File Directory. The data format value 0 does not exist thus the first
 * element of the array is 0 too which gives zero length field. Exif data format can one of the
 * following:
 * Value | Format            | Bytes/Components
 * ------|-------------------|-----------------
 *  1    | unsigned byte     | 1
 *  2    | ascii string      | 1
 *  3    | unsigned short    | 2
 *  4    | unsigned long     | 4
 *  5    | unsigned rational | 8
 *  6    | signed byte       | 1
 *  7    | undefined         | 1
 *  8    | signed short      | 2
 *  9    | signed long       | 4
 *  10   | signed rational   | 8
 *  11   | signed float      | 4
 *  12   | double float      | 8
 */
const unsigned char exif_data_fmt[] = {
		0, 1, 1, 2, 4, 8, 1,
		1, 2, 4, 8, 4, 8
};

/**
 * @struct ifd_entry
 * @brief Represents a single Image File Directory record
 * @var ifd_entry::tag
 * Exif tag number
 * @var ifd_entry::format
 * Data format in the record
 * @var ifd_entry::len
 * The number of components in the record
 * @var ifd_entry::offset
 * The data value or offset to the data
 */
struct ifd_entry {
	unsigned short tag;
	unsigned short format;
	unsigned long len;
	unsigned long offset;
};

/**
 * @struct tiff_hdr
 * @brief TIFF header structure
 * @var tiff_hdr::byte_order
 * Motorola (MM) or Inter (II) byte order
 * @var tiff_hdr::mark
 * Tag mark
 * @var tiff_hdr::offset
 * Offset to first IFD
 */
struct tiff_hdr {
	unsigned short byte_order;
	unsigned short mark;
	unsigned long offset;
};

199 200 201 202 203 204 205 206 207 208 209 210 211
/**
 * @struct crb_ptrs
 * @brief A set of vectors pointing to a file marker crossing read buffer boundary.
 * @var crb_ptrs::first_buff
 * Points to the end of first read buffer where a file marker starts
 * @var crb_ptrs::second_buff
 * Points to the start of second read buffer where a file marker ends
 */
struct crb_ptrs {
	struct iovec        first_buff;
	struct iovec        second_buff;
};

212 213 214 215 216 217 218 219 220 221 222 223 224
/**
 * @struct exit_state
 * @brief Container for the resources which should be freed before exit
 * @var exit_state::state
 * Pointer to #camogm_state structure containing current program state. This structure holds
 * file descriptors and memory mapped regions which should be closed and unmapped accordingly
 * @var exit_state::idir
 * Pointer to disk index directory. The nodes of this directory are dynamically allocated and must be freed
 * @var exit_state::sockfd_const
 * Socket descriptor
 * @var exit_state::sockfd_temp
 * Socket descriptor
 */
225 226
struct exit_state {
	camogm_state *state;
227 228 229
	struct disk_idir *idir;
	int *sockfd_const;
	int *sockfd_temp;
230 231
};

232
static inline void exit_thread(void *arg);
233
static void build_index(camogm_state *state, struct disk_idir *idir);
234

235 236 237 238 239
/**
 * @brief Debug function, prints the content of disk index directory
 * @param[in]   idir   pointer to disk index directory to be printed
 * @return      none
 */
240 241 242 243 244
void dump_index_dir(const struct disk_idir *idir)
{
	struct disk_index *ind = idir->head;

	while (ind != NULL) {
245
		fprintf(debug_file, INDEX_FORMAT_STR,
246 247 248 249 250
				ind->port, ind->rawtime, ind->usec, ind->f_offset, ind->f_size);
		ind = ind->next;
	}
}

251 252 253 254 255 256 257 258 259
/**
 * @brief Find pattern in a data buffer
 *
 * This function searches for the first occurrence of pattern in a data buffer and returns a pointer to
 * the position of this pattern in the buffer.
 * @param[in]   buff_ptr  pointer to an array of char values where the pattern should be found
 * @param[in]   buff_sz   size of the data array
 * @param[in]   pattern   pointer to an array of char values containing pattern
 * @param[in]   pt_sz     size of the pattern array
260
 * @param[in]   add_pattern include or exclude marker from resulting buffer offset
261 262
 * @return      the index in data buffer where pattern matches or error code from #match_result if it was not found
 */
263 264
static int find_marker(const unsigned char * restrict buff_ptr, ssize_t buff_sz, const unsigned char * restrict pattern, ssize_t pt_sz,
		int add_pattern)
265 266 267 268 269 270 271 272 273 274 275 276 277 278
{
	int ret = MATCH_NOT_FOUND;
	int j = 0;
	int i;

	for (i = 0; i < buff_sz; i++) {
		if (buff_ptr[i] != pattern[j]) {
			// current symbol in data buffer and first symbol of pattern does not match
			j = 0;
		} else if (buff_ptr[i] == pattern[j] && j < pt_sz - 1) {
			// pattern symbol match
			j++;
		} else if (buff_ptr[i] == pattern[j] && j == pt_sz - 1) {
			// last pattern symbol match
279 280 281 282
			if (add_pattern)
				ret = i;
			else
				ret = i - j;
283 284 285 286 287 288 289 290 291 292 293
			j = 0;
			break;
		}
	}
	if (j > 0) {
		// partial match found in the end of data buffer, we need more data for further comparison
		ret = MATCH_PARTIAL;
	}
	return ret;
}

294 295 296 297 298
/**
 * @brief Converts byte order for all fields in #ifd_entry structure
 * @param[in,out] ifd   a pointer to a structure which should be converted
 * @return        None
 */
299
static void ifd_byte_order(struct ifd_entry *ifd)
300 301 302 303 304 305 306 307 308 309 310 311
{
	ifd->tag = __be16_to_cpu(ifd->tag);
	ifd->format = __be16_to_cpu(ifd->format);
	ifd->len = __be32_to_cpu(ifd->len);
	ifd->offset = __be32_to_cpu(ifd->offset);
	if (exif_data_fmt[ifd->format] == 2) {
		ifd->offset = (ifd->offset >> 16) & 0xffff;
	}
}

/**
 * @brief Convert byte order for all fields in #tiff_hdr structure
312
 * @param[in,out] hdr   a pointer to a structure which should be converted
313 314
 * @return        None
 */
315
static void hdr_byte_order(struct tiff_hdr *hdr)
316 317 318 319 320 321 322
{
	hdr->byte_order = __be16_to_cpu(hdr->byte_order);
	hdr->mark = __be16_to_cpu(hdr->mark);
	hdr->offset = __be32_to_cpu(hdr->offset);
}

/**
323
 * @brief Read a string from Exif for a given record.
324
 * @param[in]   state   a pointer to a structure containing current state
325 326
 * @param[in]   tag     Exif image file directory record containing string offset
 * @param[out]  buff    buffer for the string to be read
327 328
 * @return      The number of bytes placed to the read buffer
 */
329
static size_t exif_get_text(camogm_state *state, struct ifd_entry *tag, char *buff)
330
{
331 332 333
	size_t bytes = 0;
	size_t str_len = tag->len * exif_data_fmt[tag->format];
	uint64_t curr_pos = state->rawdev.file_start + TIFF_HDR_OFFSET + tag->offset;
334 335

	lseek64(state->rawdev.rawdev_fd, curr_pos, SEEK_SET);
336
	bytes = read(state->rawdev.rawdev_fd, buff, str_len);
337

338
	return bytes;
339 340 341
}

/**
342 343 344 345 346
 * @brief Read Exif data from the file starting from #rawdev_buffer::file_start offset and
 * create a new node in disk index directory
 * @param[in]      state   a pointer to a structure containing current state
 * @param[in,out]  idir    pointer to disk index directory
 * @return      0 if new node was successfully added and -1 otherwise
347
 */
348
static int save_index(camogm_state *state, struct disk_idir *idir)
349
{
350
	int ret = 0;
351 352 353 354 355 356 357
	int process = 2;
	uint32_t data32;
	uint16_t num_entries = 0;
	uint64_t curr_pos;
	uint64_t subifd_offset = 0;
	struct tiff_hdr hdr;
	struct ifd_entry ifd;
358 359 360 361
	struct ifd_entry ifd_page_num = {0};
	struct ifd_entry ifd_date_time = {0};
	struct ifd_entry ifd_subsec = {0};
	struct disk_index *node = NULL;
362
	unsigned char read_buff[TIFF_HDR_OFFSET] = {0};
363
	char str_buff[SMALL_BUFF_LEN] = {0};
364 365
	uint64_t save_pos = lseek64(state->rawdev.rawdev_fd, 0, SEEK_CUR);

366
	if (idir == NULL)
367 368
		return -1;

369 370 371
	if (create_node(&node) != 0)
		return -1;

372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413
	curr_pos = state->rawdev.file_start;
	lseek64(state->rawdev.rawdev_fd, curr_pos, SEEK_SET);
	if (read(state->rawdev.rawdev_fd, read_buff, sizeof(read_buff)) <= 0) {
		lseek64(state->rawdev.rawdev_fd, save_pos, SEEK_SET);
		return -1;
	}
	if (read_buff[2] == 0xff && read_buff[3] == 0xe1) {
		// get IFD0 offset from TIFF header
		read(state->rawdev.rawdev_fd, &hdr, sizeof(struct tiff_hdr));
		hdr_byte_order(&hdr);
		curr_pos = state->rawdev.file_start + TIFF_HDR_OFFSET + hdr.offset;
		lseek64(state->rawdev.rawdev_fd, curr_pos, SEEK_SET);
		// process IFD0 and SubIFD fields
		do {
			read(state->rawdev.rawdev_fd, &num_entries, sizeof(num_entries));
			num_entries = __be16_to_cpu(num_entries);
			for (int i = 0; i < num_entries; i++) {
				read(state->rawdev.rawdev_fd, &ifd, sizeof(struct ifd_entry));
				ifd_byte_order(&ifd);
				switch (ifd.tag) {
				case Exif_Image_PageNumber:
					ifd_page_num = ifd;
					break;
				case Exif_Photo_DateTimeOriginal & 0xffff:
					ifd_date_time = ifd;
					break;
				case Exif_Image_ExifTag:
					subifd_offset = ifd.offset;
					break;
				case Exif_Photo_SubSecTimeOriginal & 0xffff:
					ifd_subsec = ifd;
					break;
				}
			}
			// ensure that IFD0 finished correctly (0x00000000 in the end), set file pointer to SubIFD and
			// process remaining fields
			read(state->rawdev.rawdev_fd, &data32, sizeof(data32));
			process -= (subifd_offset == 0 || data32 != 0) ? 2 : 1;
			curr_pos = state->rawdev.file_start + TIFF_HDR_OFFSET + subifd_offset;
			lseek64(state->rawdev.rawdev_fd, curr_pos, SEEK_SET);
		} while (process > 0);

414 415
		// fill disk index node with Exif data and add it to disk index directory
		node->f_offset = state->rawdev.file_start;
416
		if (ifd_page_num.len != 0) {
417
			node->port = (uint32_t)ifd_page_num.offset;
418 419
		}
		if (ifd_date_time.len != 0) {
420 421 422 423
			struct tm tm;
			exif_get_text(state, &ifd_date_time, str_buff);
			strptime(str_buff, EXIF_DATE_TIME_FORMAT, &tm);
			node->rawtime = mktime(&tm);
424 425
		}
		if (ifd_subsec.len != 0) {
426 427
			exif_get_text(state, &ifd_subsec, str_buff);
			node->usec = strtoul(str_buff, NULL, 10);
428
		}
429 430 431 432
		if (node->rawtime != -1)
			add_node(idir, node);
		else
			ret = -1;
433 434 435
	}

	lseek64(state->rawdev.rawdev_fd, save_pos, SEEK_SET);
436 437 438
	return ret;
}

439 440 441 442 443 444 445 446
/**
 * @brief Calculate the size of current file and update the value in disk index directory
 * @param[in,out]   idir       pointer to disk index directory
 * @param[in]       pos_stop   the offset of the last byte of the current file
 * @return          0 if the file size was successfully updated and -1 otherwise
 * @note @e pos_stop points to the last byte of the file marker thus the size is incremented
 * by 1
 */
447 448 449 450
int stop_index(struct disk_idir *idir, uint64_t pos_stop)
{
	int ret = 0;
	if (idir->tail != NULL) {
451
		idir->tail->f_size = pos_stop - idir->tail->f_offset + 1;
452 453 454 455
	} else {
		ret = -1;
	}
	return ret;
456 457
}

458 459 460 461 462 463 464 465 466 467
/**
 * @brief Detect cases when file marker crosses read buffer boundary
 * @param[in]   from   pointer to current read buffer which can hold the beginning of
 * file marker
 * @param[in]   to     pointer to a buffer containing next chunk of data which can hold
 * the end of file marker
 * @param[in]   marker pointer to a buffer holding file marker to be detect
 * @param[in]   crbp   pointer to a structure which will store two cross border pointers
 * @return      a constant of #match_result type
 */
468
static int check_edge_case(const struct iovec *from, const struct iovec *to, const struct iovec *marker, struct crb_ptrs *crbp)
469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
{
	unsigned char *start_ptr = from->iov_base + from->iov_len - marker->iov_len;
	unsigned char *end_ptr = from->iov_base + from->iov_len;
	unsigned char *marker_ptr = marker->iov_base;
	unsigned int bytes_processed = 0;
	int match = 0;

	// search for the first part of marker in the end of *from* array
	while (start_ptr <= end_ptr) {
		if (*start_ptr == *marker_ptr && !match) {
			crbp->first_buff.iov_base = start_ptr;
			crbp->first_buff.iov_len = end_ptr - start_ptr;
			match = 1;
		}
		if (*start_ptr == *marker_ptr && match) {
			marker_ptr++;
			bytes_processed++;
		} else {
			break;
		}
		start_ptr++;
	}
	if (start_ptr != end_ptr) {
		// match not found in the end of *from* array
		return MATCH_NOT_FOUND;
	}

	// search for the second part of marker in the beginning of *to* array
	start_ptr = to->iov_base;
	end_ptr = to->iov_base + (marker->iov_len - bytes_processed);
	while (start_ptr <= end_ptr) {
		if (*start_ptr++ != *marker_ptr++)
			break;
	}
	if (start_ptr != end_ptr) {
		// match not found in the beginning of *to* array
		return MATCH_NOT_FOUND;
	}
	crbp->second_buff.iov_base = to->iov_base;
	crbp->second_buff.iov_len = marker->iov_len - bytes_processed;

	return MATCH_FOUND;
}

/**
514 515 516 517 518
 * @brief Send mmaped buffer over opened socket
 * @param[in]   sockfd    opened socket descriptor
 * @param[in]   buff      pointer to memory mapped buffer
 * @param[in]   sz        the size of @e buff
 * @return      none
519
 */
520
void send_buffer(int sockfd, unsigned char *buff, size_t sz)
521
{
522
	size_t bytes_left = sz;
523
	ssize_t bytes_written = 0;
524 525 526 527
	size_t offset = 0;

	while (bytes_left > 0) {
		bytes_written = write(sockfd, &buff[offset], bytes_left);
528 529 530 531
		if (bytes_written < 0) {
			perror(__func__);
			return;
		}
532 533 534 535 536
		bytes_left -= bytes_written;
		offset += bytes_written;
	}
}

537 538 539 540 541 542 543 544
/**
 * @brief Map a piece of raw device buffer to memory
 * @param[in,out]   rawdev   pointer to #rawdev_buffer structure containing
 * the current state of raw device buffer
 * @param[in]       range    pointer to #range structure holding the offsets of
 * mmaped region
 * @return          0 in case the region was mmaped successfully and -1 in case of an error
 */
545
int mmap_disk(rawdev_buffer *rawdev, const struct range *range)
546 547
{
	int ret = 0;
548
	size_t mmap_size = range->to - range->from;
549 550 551 552 553

	rawdev->rawdev_fd = open(rawdev->rawdev_path, O_RDONLY);
	if (rawdev->rawdev_fd < 0) {
		return -1;
	}
554
	rawdev->disk_mmap = mmap(0, mmap_size, PROT_READ, MAP_SHARED, rawdev->rawdev_fd, range->from);
555
	if (rawdev->disk_mmap == MAP_FAILED) {
556
		rawdev->disk_mmap = NULL;
557 558 559
		close(rawdev->rawdev_fd);
		return -1;
	}
560 561
	rawdev->mmap_offset = range->from;
	rawdev->mmap_current_size = mmap_size;
562 563 564 565

	return ret;
}

566 567 568 569 570 571
/**
 * @brief Unmap currently mapped raw device buffer
 * @param[in,out]   rawdev   pointer to #rawdev_buffer structure containing
 * the current state of raw device buffer
 * @return          0 in case the region was unmapped successfully and -1 in case of an error
 */
572
int munmap_disk(rawdev_buffer *rawdev)
573
{
574 575
	if (rawdev->disk_mmap == NULL)
		return 0;
576

577
	if (munmap(rawdev->disk_mmap, rawdev->mmap_current_size) != 0)
578 579 580
		return -1;
	if (close(rawdev->rawdev_fd) != 0)
		return -1;
581 582
	rawdev->mmap_offset = 0;
	rawdev->disk_mmap = NULL;
583

584 585 586
	return 0;
}

587 588 589 590 591 592 593 594
/**
 * @brief Check if the file pointed by disk index node is in the memory mapped region
 * @param[in]   range   pointer to #range structure holding the offsets of
 * memory mapped region
 * @param[in]   indx    pointer to the disk index node which should be checked for
 * presence in currently mapped region
 * @return      @b true if the file is the region and @b false otherwise
 */
595 596 597 598 599 600 601 602
bool is_in_range(struct range *range, struct disk_index *indx)
{
	if (indx->f_offset >= range->from &&
			indx->f_offset <= range->to &&
			(indx->f_offset + indx->f_size) <= range->to)
		return true;
	else
		return false;
603 604
}

605 606 607
/**
 * @brief Prepare socket for communication
 * @param[out]   socket_fd   pointer to socket descriptor
608
 * @param[in]    port_num    socket port number
609 610
 * @return       none
 */
611
void prep_socket(int *socket_fd, uint16_t port_num)
612 613 614 615 616 617
{
	int opt = 1;
	struct sockaddr_in sock;

	memset((char *)&sock, 0, sizeof(struct sockaddr_in));
	sock.sin_family = AF_INET;
618
	sock.sin_port = htons(port_num);
619 620 621 622 623 624 625
	*socket_fd = socket(AF_INET, SOCK_STREAM, 0);
	setsockopt(*socket_fd, SOL_SOCKET, SO_REUSEADDR, (char *)&opt, sizeof(opt));
	bind(*socket_fd, (struct sockaddr *) &sock, sizeof(struct sockaddr_in));
	listen(*socket_fd, 10);
}

/**
626 627 628 629 630
 * @brief Parse command line
 * @param[in]   cmd   pointer to command line buffer, this pointer is
 * updated to point to current command
 * @return Positive command number from #socket_commands, -1 if command not recognized and
 * -2 to indicate that the command buffer has been fully processed
631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647
 */
int parse_command(char **cmd)
{
	size_t cmd_len;
	int cmd_indx = -1;
	char *char_ptr;

	D6(fprintf(debug_file, "Parsing command line: %s\n", *cmd));
	char_ptr = strpbrk(*cmd, CMD_DELIMITER);
	if (char_ptr != NULL) {
		char_ptr[0] = '\0';
		char_ptr++;
		for (int i = 0; i < sizeof(cmd_list) / sizeof(cmd_list[0]); i++) {
			cmd_len = strlen(cmd_list[i]);
			if (strncmp(char_ptr, cmd_list[i], cmd_len) == 0) {
				cmd_indx = i;
				break;
648 649
			}
		}
650 651 652 653 654 655 656 657 658 659 660 661
		*cmd = char_ptr;
	} else {
		cmd_indx = -2;
	}

	return cmd_indx;
}

/**
 * @brief Break HTTP GET string after the command part as we do not need that part. The function
 * finds the first space character after the command part starts and replaces it with null.
 * @param[in,out]   cmd   pointer to HTTP GET string
662 663
 * @param[in]       cmd_len the length of the command in buffer
 * @return          none
664 665 666 667 668 669 670 671 672 673 674 675 676 677 678
 */
void trim_command(char *cmd, ssize_t cmd_len)
{
	char *ptr_start, *ptr_end;

	if (cmd_len >= 0 && cmd_len < CMD_BUFF_LEN)
		cmd[cmd_len] = '\0';
	ptr_start = strpbrk(cmd, CMD_DELIMITER);
	if (ptr_start) {
		ptr_end = strchr(ptr_start, ' ');
		if (ptr_end)
			ptr_end[0] = '\0';
	}
}

679 680 681 682 683 684 685 686
/**
 * @brief Send a file crossing raw device buffer boundary.
 * Such a file is split into two parts, the one in the end of the buffer and the other
 * in the beginning, and should be sent in two steps
 * @param[in]   rawdev   pointer to #rawdev_buffer structure containing
 * @param[in]   indx     pointer to the disk index node
 * @param[in]   sockfd   opened socket descriptor
 */
687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714
void send_split_file(rawdev_buffer *rawdev, struct disk_index *indx, int sockfd)
{
	ssize_t rcntr = 0;
	ssize_t scntr = 0;
	size_t head_sz = rawdev->end_pos - indx->f_offset;
	size_t tail_sz = indx->f_size - head_sz;
	uint64_t curr_pos = lseek64(rawdev->rawdev_fd, 0, SEEK_CUR);
	unsigned char *buff = malloc(indx->f_size);

	if (buff == NULL)
		return;

	lseek64(rawdev->rawdev_fd, indx->f_offset, SEEK_SET);
	while (rcntr < head_sz && rcntr != -1)
		rcntr += read(rawdev->rawdev_fd, &buff[rcntr], head_sz - rcntr);
	rcntr = 0;
	lseek64(rawdev->rawdev_fd, rawdev->start_pos, SEEK_SET);
	while (rcntr < tail_sz && rcntr != -1)
		rcntr += read(rawdev->rawdev_fd, &buff[head_sz + rcntr], tail_sz - rcntr);

	while (scntr < indx->f_size && scntr != -1)
		scntr += write(sockfd, &buff[scntr], head_sz - scntr);

	fprintf(debug_file, "%s: %d bytes sent\n", __func__, scntr);

	lseek64(rawdev->rawdev_fd, curr_pos, SEEK_SET);
}

715 716 717 718 719 720
/**
 * @brief Send the number of files (or disk chunks) found over socket connection
 * @param[in]   sockfd   opened socket descriptor
 * @param[in]   num      the number to be sent
 * @return      none
 */
721 722 723 724 725 726 727 728 729 730
void send_fnum(int sockfd, size_t num)
{
	char buff[SMALL_BUFF_LEN] = {0};
	int len;

	len = snprintf(buff, SMALL_BUFF_LEN - 1, "Number of files: %d\n", num);
	buff[len] = '\0';
	write(sockfd, buff, len);
}

731 732 733 734 735 736
/**
 * @brief Read file parameters from a string and fill in disk index node structure
 * @param[in]    cmd   pointer to a string with file parameters
 * @param[out]   indx  pointer to disk index node structure
 * @return       the number of parameters extracted from the string or -1 in case of an error
 */
737 738 739 740 741 742 743 744 745
int get_indx_args(char *cmd, struct disk_index *indx)
{
	char *cmd_start = strchr(cmd, ':');

	if (cmd_start == NULL)
		return -1;
	return sscanf(++cmd_start, INDEX_FORMAT_STR, &indx->port, &indx->rawtime, &indx->usec, &indx->f_offset, &indx->f_size);
}

746
/**
747
 * @brief Raw device buffer reading function.
748
 *
749 750 751 752 753 754 755
 * This function is started in a separated thread right after the application has started. It opens a
 * communication socket, waits for commands sent over the socket and process them.
 * @param[in, out]   arg   pointer to #camogm_state structure
 * @return           none
 * @warning The main processing loop of the function is enclosed in @e pthread_cleanup_push and @e pthread_cleanup_pop
 * calls. The effect of use of normal @b return or @b break to prematurely leave this loop is undefined.
 * @todo print unrecognized command to debug output file
756 757 758 759
 */
void *reader(void *arg)
{
	int sockfd, fd;
760
	int disk_chunks;
761 762 763
	int cmd;
	char cmd_buff[CMD_BUFF_LEN] = {0};
	char *cmd_ptr;
764
	char send_buff[CMD_BUFF_LEN] = {0};
765
	bool transfer;
766
	ssize_t cmd_len;
767 768
	size_t mm_file_start, mm_file_size;
	size_t file_cntr;
769 770
	camogm_state *state = (camogm_state *)arg;
	rawdev_buffer *rawdev = &state->rawdev;
771 772 773 774 775 776 777 778
	struct stat stat_buff;
	struct range mmap_range;
	struct disk_index *disk_indx, *cross_boundary_indx;
	struct disk_idir index_dir = {
			.head = NULL,
			.tail = NULL,
			.size = 0
	};
779 780
	struct exit_state exit_state = {
			.state = state,
781 782
			.idir = &index_dir,
			.sockfd_const = &sockfd,
783
			.sockfd_temp = &fd
784 785
	};

786
	prep_socket(&sockfd, state->sock_port);
787
	pthread_cleanup_push(exit_thread, &exit_state);
788 789 790 791
	while (true) {
		fd = accept(sockfd, NULL, 0);
		if (fd == -1)
			continue;
792 793 794 795 796 797 798 799 800
		if (state->prog_state == STATE_STOPPED && state->rawdev_op) {
			pthread_mutex_lock(&state->mutex);
			state->prog_state = STATE_READING;
			pthread_mutex_unlock(&state->mutex);
		} else {
			close(fd);
			D0(fprintf(debug_file, "Can not change state of the program, check settings\n"));
			continue;
		}
801 802 803
		cmd_len = read(fd, cmd_buff, sizeof(cmd_buff) - 1);
		cmd_ptr = cmd_buff;
		trim_command(cmd_ptr, cmd_len);
804
		while ((cmd = parse_command(&cmd_ptr)) != -2 && state->rawdev.thread_state != STATE_CANCEL) {
805
			if (cmd >= 0)
806
				D6(fprintf(debug_file, "Got command '%s', number %d\n", cmd_list[cmd], cmd));
807 808
			switch (cmd) {
			case CMD_BUILD_INDEX:
809
				// scan raw device buffer and create disk index directory
810 811 812 813 814
				if (index_dir.size != 0) {
					delete_idir(&index_dir);
				}
				build_index(state, &index_dir);
				D3(fprintf(debug_file, "%d files read from %s\n", index_dir.size, state->rawdev.rawdev_path));
815 816
				break;
			case CMD_GET_INDEX:
817 818 819 820 821
				// send the content of disk index directory over socket
				if (index_dir.size > 0) {
					int len;
					disk_indx = index_dir.head;
					while (disk_indx != NULL) {
822
						len = snprintf(send_buff, CMD_BUFF_LEN - 1, INDEX_FORMAT_STR,
823 824 825 826 827 828 829 830 831
								disk_indx->port, disk_indx->rawtime, disk_indx->usec, disk_indx->f_offset, disk_indx->f_size);
						send_buff[len] = '\0';
						write(fd, send_buff, len);
						disk_indx = disk_indx->next;
					}
				} else {
					D0(fprintf(debug_file, "Index directory does not contain any files. Try to rebuild index "
							"directory with 'build_index' command\n"));
				}
832 833
				break;
			case CMD_READ_DISK:
834
				// mmap raw device buffer in MMAP_CHUNK_SIZE chunks and send them over socket
835
				mmap_range.from = rawdev->start_pos & PAGE_BOUNDARY_MASK;
836
				mmap_range.to = mmap_range.from + rawdev->mmap_default_size;
837 838 839
				disk_chunks = (size_t)ceil((double)(rawdev->end_pos - rawdev->start_pos) / (double)rawdev->mmap_default_size);
				transfer = true;
				mm_file_start = rawdev->start_pos;
840
				mm_file_size = rawdev->mmap_default_size - rawdev->start_pos;
841
				send_fnum(fd, disk_chunks);
842
				close(fd);
843
				while (disk_chunks > 0 && transfer && state->rawdev.thread_state != STATE_CANCEL) {
844 845 846 847 848 849 850 851 852 853 854 855 856
					fd = accept(sockfd, NULL, 0);
					if (mmap_disk(rawdev, &mmap_range) == 0) {
						send_buffer(fd, &rawdev->disk_mmap[mm_file_start], mm_file_size);
					} else {
						transfer = false;
						D0(fprintf(debug_file, "Unable to map disk to memory region:"
								"disk region start = 0x%llx, disk region end = 0x%llx\n", mmap_range.from, mmap_range.to));
					}
					if (munmap_disk(rawdev) != 0) {
						transfer = false;
						D0(fprintf(debug_file, "Unable to unmap memory region\n"));
					}
					mm_file_start = 0;
857
					mm_file_size = rawdev->mmap_default_size;
858
					disk_chunks--;
859
					mmap_range.from = mmap_range.to;
860
					mmap_range.to = mmap_range.from + rawdev->mmap_default_size;
861
					if (mmap_range.to > rawdev->end_pos) {
862
						mmap_range.to = rawdev->end_pos;
863 864
						mm_file_size = mmap_range.to - mmap_range.from;
					}
865 866
					close(fd);
				}
867 868
				break;
			case CMD_READ_FILE:
869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886
				if (index_dir.size > 0) {
					struct disk_index indx;
					if (get_indx_args(cmd_ptr, &indx) > 0 &&
							(disk_indx = find_by_offset(&index_dir, indx.f_offset)) != NULL){
						mmap_range.from = indx.f_offset & PAGE_BOUNDARY_MASK;
						mmap_range.to = indx.f_offset + indx.f_size;
						mm_file_start = indx.f_offset - mmap_range.from;
						if (mmap_disk(rawdev, &mmap_range) == 0) {
							send_buffer(fd, &rawdev->disk_mmap[mm_file_start], indx.f_size);
							if (munmap_disk(rawdev) != 0) {
								D0(fprintf(debug_file, "Unable to unmap memory region\n"));
							}
						} else {
							D0(fprintf(debug_file, "Unable to map disk to memory region:"
									"disk region start = 0x%llx, disk region end = 0x%llx\n", mmap_range.from, mmap_range.to));
						}
					}
				}
887 888
				break;
			case CMD_READ_ALL_FILES:
889 890
				// read files from raw device buffer and send them over socket; the disk index directory
				// should be built beforehand
891
				if (index_dir.size > 0) {
892 893
					send_fnum(fd, index_dir.size);
					close(fd);
894 895 896 897 898 899
					mmap_range.from = rawdev->start_pos;
					mmap_range.to = rawdev->start_pos + rawdev->mmap_default_size;
					disk_indx = index_dir.head;
					cross_boundary_indx = NULL;
					file_cntr = 0;
					transfer = true;
900
					while (file_cntr < index_dir.size && disk_indx != NULL && state->rawdev.thread_state != STATE_CANCEL) {
901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939
						if (is_in_range(&mmap_range, disk_indx) && rawdev->disk_mmap != NULL) {
							fd = accept(sockfd, NULL, 0);
							mm_file_start = disk_indx->f_offset - rawdev->mmap_offset;
							send_buffer(fd, &rawdev->disk_mmap[mm_file_start], disk_indx->f_size);
							close(fd);
							disk_indx = disk_indx->next;
							file_cntr++;
						} else {
							if (munmap_disk(rawdev) == 0) {
								mmap_range.from = disk_indx->f_offset & PAGE_BOUNDARY_MASK;
								mmap_range.to = mmap_range.from + rawdev->mmap_default_size;
								if (mmap_range.to > rawdev->end_pos) {
									mmap_range.to = rawdev->end_pos;
								}
								if (disk_indx->f_offset + disk_indx->f_size <= mmap_range.to) {
									if (mmap_disk(rawdev, &mmap_range) < 0) {
										disk_indx = NULL;
										D0(fprintf(debug_file, "Unable to map disk to memory region:"
												" disk region start = 0x%llx, disk region end = 0x%llx\n", mmap_range.from, mmap_range.to));
									}
								} else {
									cross_boundary_indx = disk_indx;
									disk_indx = NULL;
								}
							} else {
								disk_indx = NULL;
								D0(fprintf(debug_file, "Unable to unmap memory region\n"));
							}
						}
					}
					munmap_disk(rawdev);
					if (cross_boundary_indx != NULL) {
						send_split_file(rawdev, cross_boundary_indx, fd);
						close(fd);
					}
				} else {
					D0(fprintf(debug_file, "Index directory does not contain any files. Try to rebuild index "
							"directory with 'build_index' command\n"));
				}
940 941 942 943 944 945
				break;
			case CMD_STATUS:
				break;
			default:
				D0(fprintf(debug_file, "Unrecognized command is skipped\n"));
			}
946
		}
947 948
		if (fstat(fd, &stat_buff) != EBADF)
			close(fd);
949 950 951
		pthread_mutex_lock(&state->mutex);
		state->prog_state = STATE_STOPPED;
		pthread_mutex_unlock(&state->mutex);
952
		usleep(COMMAND_LOOP_DELAY);
953
	}
954
	pthread_cleanup_pop(0);
955 956

	return (void *) 0;
957 958
}

959 960 961 962 963 964 965
/**
 * @brief Clean up after the reading thread is closed. This function is thread-cancellation handler and it is
 * passed to @e pthread_cleanup_push() function.
 * @param[in]   arg   pointer to #exit_state structure containing resources that
 * should be closed
 * @return      none
 */
966
static inline void exit_thread(void *arg)
967 968
{
	struct exit_state *s = (struct exit_state *)arg;
969
	struct stat buff;
970

971 972 973
	if (fstat(s->state->rawdev.rawdev_fd, &buff) != EBADF) {
		if (s->state->rawdev.disk_mmap != NULL)
			munmap(s->state->rawdev.disk_mmap, s->state->rawdev.mmap_current_size);
974 975 976
		close(s->state->rawdev.rawdev_fd);
		s->state->rawdev.rawdev_fd = -1;
	}
977 978 979 980 981 982
	if (s->idir->size != 0)
		delete_idir(s->idir);
	if (fstat(*s->sockfd_const, &buff) != EBADF)
		close(*s->sockfd_const);
	if (fstat(*s->sockfd_temp, &buff) != EBADF)
		close(*s->sockfd_temp);
983 984
}

985
/**
986 987
 * @brief Extract the position and parameters of JPEG files in raw device buffer and
 * build disk index directory for further file extraction.
988 989
 *
 * Data from raw device is read to a buffer in #PHY_BLK_SZ blocks. The buffer is
990 991
 * then analyzed for JPEG markers stored in #elphelst and #elphelen arrays. The offsets and
 * sizes of the files found in the buffer are recorded to disk index directory.
992
 * @param[in]   state   a pointer to a structure containing current state
993 994 995 996
 * @param[out]  idir    a pointer to disk index directory. This directory will contain
 * offset of the files found in the raw device buffer.
 * @return      none
 * @todo reorder decision tree
997
 */
998
void build_index(camogm_state *state, struct disk_idir *idir)
999 1000 1001 1002
{
	const int include_markers = INCLUDE_MARKERS;
	int process;
	int zero_cross;
1003
	int err;
1004 1005
	int pos_start, pos_stop;
	int buff_processed;
1006 1007
	int search_state;
	int idir_result;
1008
	ssize_t rd;
1009 1010
	unsigned char buff[PHY_BLK_SZ];
	unsigned char next_buff[PHY_BLK_SZ];
1011
	unsigned char *active_buff = buff;
1012 1013 1014
	unsigned char *save_from = NULL;
	unsigned char *save_to = NULL;
	uint64_t dev_curr_pos = 0;
1015
	uint64_t include_st_marker, include_en_marker;
1016
	size_t add_stm_len, add_enm_len;
1017

1018 1019 1020 1021
	state->rawdev.rawdev_fd = open(state->rawdev.rawdev_path, O_RDONLY);
	if (state->rawdev.rawdev_fd < 0) {
		D0(perror(__func__));
		D0(fprintf(debug_file, "Error opening raw device %s\n", state->rawdev.rawdev_path));
1022
		return;
1023 1024 1025
	}

	if (include_markers) {
1026 1027 1028 1029
		include_st_marker = 0;
		add_stm_len = elphel_st.iov_len;
		include_en_marker = 1;
		add_enm_len = 0;
1030 1031
	} else {
		include_st_marker = 0;
1032 1033 1034
		add_stm_len = 0;
		include_en_marker = 1;
		add_enm_len = elphel_en.iov_len;
1035
	}
1036

1037 1038
	process = 1;
	zero_cross = 0;
1039 1040
	search_state = SEARCH_SKIP;
	idir_result = 0;
1041
	while (process && state->rawdev.thread_state != STATE_CANCEL) {
1042 1043 1044
		rd = read(state->rawdev.rawdev_fd, buff, sizeof(buff));
		err = errno;
		if ((rd > 0) && (dev_curr_pos + rd > state->rawdev.end_pos)) {
1045
			// read pointer jumped over the raw storage buffer end, truncate excessive data
Mikhail Karpenko's avatar
Mikhail Karpenko committed
1046
			D3(fprintf(debug_file, "End of raw storage buffer is reached, will start from the beginning\n"));
1047
			rd = state->rawdev.end_pos - dev_curr_pos;
1048
			zero_cross = 1;
1049 1050
			lseek64(state->rawdev.rawdev_fd, state->rawdev.start_pos, SEEK_SET);
			dev_curr_pos = state->rawdev.start_pos;
1051 1052 1053 1054 1055 1056
			if (rd == 0) {
				continue;
			}
		} else if (rd < 0) {
			// read error or read pointer exceeded raw storage capacity, close files and terminate
			process = 0;
Mikhail Karpenko's avatar
Mikhail Karpenko committed
1057
			D0(fprintf(debug_file, "Raw device read was unsuccessful: %s\n", strerror(err)));
1058 1059
		} else if (rd == 0) {
			// end of device file reached
Mikhail Karpenko's avatar
Mikhail Karpenko committed
1060
			D3(fprintf(debug_file, "End of raw storage device file is reached, will start from the beginning\n"));
1061
			zero_cross = 1;
1062 1063
			lseek64(state->rawdev.rawdev_fd, state->rawdev.start_pos, SEEK_SET);
			dev_curr_pos = state->rawdev.start_pos;
1064 1065 1066 1067
		}
		if (process) {
			save_from = buff;
			save_to = buff + rd;
1068
			active_buff = buff;
1069 1070 1071
			buff_processed = 0;
			do {
				// process data in read buffer
1072 1073 1074 1075 1076 1077
				pos_start = find_marker(save_from, save_to - save_from, elphel_st.iov_base, elphel_st.iov_len, include_st_marker);
				pos_stop = find_marker(save_from, save_to - save_from, elphel_en.iov_base, elphel_en.iov_len, include_en_marker);

				if (pos_start == MATCH_NOT_FOUND && pos_stop == MATCH_NOT_FOUND) {
					// normal condition, search in progress
					buff_processed = 1;
1078
//					D6(fprintf(debug_file, "State 'skip data'\n"));
1079 1080 1081 1082
				} else if (pos_start >= 0 && pos_stop == MATCH_NOT_FOUND && search_state == SEARCH_SKIP) {
					// normal condition, new file found
					search_state = SEARCH_FILE_DATA;
					state->rawdev.file_start = dev_curr_pos + pos_start + (save_from - active_buff);
1083
					idir_result = save_index(state, idir);
1084
					buff_processed = 1;
Mikhail Karpenko's avatar
Mikhail Karpenko committed
1085
					D6(fprintf(debug_file, "New file found. File start position: %llu\n", state->rawdev.file_start));
1086 1087 1088
					D6(fprintf(debug_file, "State 'starting file'\n"));
				} else if (pos_start >= 0 && pos_stop == MATCH_NOT_FOUND && search_state == SEARCH_FILE_DATA) {
					// error condition (normally should not happen), discard current index and start a new one
1089
					buff_processed = 1;
1090
					remove_node(idir, idir->tail);
1091 1092
					if (zero_cross == 0) {
						state->rawdev.file_start = dev_curr_pos + pos_start + (save_from - active_buff);
1093
						idir_result = save_index(state, idir);
1094 1095 1096 1097
					} else {
						process = 0;
					}
					D6(fprintf(debug_file, "State 'abnormal start marker, remove current disk index from directory and skip data'\n"));
1098
				} else if (pos_start == MATCH_NOT_FOUND && pos_stop >= 0 &&
1099 1100 1101 1102
						search_state == SEARCH_FILE_DATA) {
					// normal condition, save current file size to index directory
					uint64_t disk_pos = dev_curr_pos + pos_stop + (save_from - active_buff);
					search_state = SEARCH_SKIP;
1103
					idir_result = stop_index(idir, disk_pos);
1104 1105 1106
					buff_processed = 1;
					if (zero_cross)
						process = 0;
Mikhail Karpenko's avatar
Mikhail Karpenko committed
1107
					D6(fprintf(debug_file, "State 'finishing file'\n"));
1108 1109 1110 1111 1112 1113 1114
				} else if (pos_start == MATCH_NOT_FOUND && pos_stop >= 0 && search_state == SEARCH_SKIP) {
					// error condition (normally should not happen), drop current read buffer and do nothing
					buff_processed = 1;
					D6(fprintf(debug_file, "State 'abnormal stop marker, skip data'\n"));
				} else if (pos_start >= 0 && pos_stop >= 0 && pos_start > pos_stop && search_state == SEARCH_FILE_DATA) {
					// normal condition, start marker following stop marker found - this indicates a new file
					uint64_t disk_pos = dev_curr_pos + pos_stop + (save_from - active_buff);
1115
					idir_result = stop_index(idir, disk_pos);
1116 1117
					if (zero_cross == 0) {
						state->rawdev.file_start = dev_curr_pos + pos_start + (save_from - active_buff);
1118
						idir_result = save_index(state, idir);
1119 1120 1121 1122
						save_from = save_from + pos_start + add_stm_len;
						// @todo: replace with pointer to current buffer
						save_to = buff + rd;
					} else {
1123 1124 1125
						buff_processed = 1;
						process = 0;
					}
1126 1127 1128
					D6(fprintf(debug_file, "State 'stop current file and start new file'\n"));
				} else if (pos_start == MATCH_PARTIAL && search_state == SEARCH_SKIP) {
					// partial start marker found in the end of read buffer, get next chunk of data and try to find marker there
1129 1130 1131 1132 1133 1134 1135 1136 1137
					enum match_result result;
					struct crb_ptrs field_markers;
					struct iovec next_chunk = {
							.iov_base = next_buff,
					};
					struct iovec curr_chunk = {
							.iov_base = save_from,
							.iov_len = save_to - save_from
					};
1138
					ssize_t next_rd = read(state->rawdev.rawdev_fd, next_buff, sizeof(next_buff));
1139 1140
					next_chunk.iov_len = next_rd;
					result = check_edge_case(&curr_chunk, &next_chunk, &elphel_st, &field_markers);
1141 1142 1143
					if (result == MATCH_FOUND) {
						search_state = SEARCH_FILE_DATA;
						state->rawdev.file_start = dev_curr_pos + pos_start + (save_from - active_buff);
1144
						idir_result = save_index(state, idir);
Mikhail Karpenko's avatar
Mikhail Karpenko committed
1145
						D6(fprintf(debug_file, "File start position: %llu\n", state->rawdev.file_start));
1146 1147 1148 1149 1150 1151
						save_from = next_chunk.iov_base + field_markers.second_buff.iov_len;
						save_to = next_chunk.iov_base + next_chunk.iov_len;
					} else {
						save_from = next_chunk.iov_base;
						save_to = next_chunk.iov_base + next_chunk.iov_len;
					}
1152 1153
					dev_curr_pos += next_rd;
					active_buff = next_buff;
Mikhail Karpenko's avatar
Mikhail Karpenko committed
1154
					D6(fprintf(debug_file, "State 'check elphel_st cross boundary'; result = %d\n", result));
1155
				} else if (pos_stop == MATCH_PARTIAL && search_state == SEARCH_FILE_DATA) {
1156 1157 1158 1159 1160 1161 1162 1163 1164 1165
					// partial end marker found in the end of read buffer - get next chunk of data and try to find marker there
					enum match_result result;
					struct crb_ptrs field_markers;
					struct iovec next_chunk = {
							.iov_base = next_buff,
					};
					struct iovec curr_chunk = {
							.iov_base = save_from,
							.iov_len = save_to - save_from
					};
1166
					ssize_t next_rd = read(state->rawdev.rawdev_fd, next_buff, sizeof(next_buff));
1167 1168 1169
					next_chunk.iov_len = next_rd;
					result = check_edge_case(&curr_chunk, &next_chunk, &elphel_en, &field_markers);
					if (result == MATCH_FOUND) {
1170 1171
						search_state = SEARCH_SKIP;
						uint64_t disk_pos = dev_curr_pos + pos_stop + (save_from - active_buff);
1172
						idir_result = stop_index(idir, disk_pos);
1173 1174 1175 1176 1177 1178
						save_from = next_chunk.iov_base + field_markers.second_buff.iov_len;
						save_to = next_chunk.iov_base + next_chunk.iov_len;
					} else {
						save_from = next_chunk.iov_base;
						save_to = next_chunk.iov_base + next_chunk.iov_len;
					}
1179 1180
					dev_curr_pos += next_rd;
					active_buff = next_buff;
Mikhail Karpenko's avatar
Mikhail Karpenko committed
1181
					D6(fprintf(debug_file, "State 'check elphel_en' cross boundary:; result = %d\n", result));
1182 1183
				} else {
					// no markers found and new file has not bee started yet - skip data
1184 1185
					D6(fprintf(debug_file, "Undefined state: pos_start = %d, pos_stop = %d, search_state = %d\n",
							pos_start, pos_stop, search_state));
1186 1187 1188 1189
					buff_processed = 1;
					if (zero_cross)
						process = 0;
				}
1190 1191
			} while (buff_processed == 0 && idir_result == 0);
			if (idir_result != 0) {
1192 1193
				process = 0;
			}
1194
			dev_curr_pos += rd;
1195
			state->rawdev.curr_pos_r = dev_curr_pos;
1196 1197 1198
		}
	}

1199
	if (close(state->rawdev.rawdev_fd) != 0) {
1200 1201
		perror("Unable to close raw device: ");
	}
1202
	state->rawdev.rawdev_fd = -1;
1203
}