camogm_read.c 50.3 KB
Newer Older
1
/** @file camogm_read.c
2
 * @brief Provides reading data written to raw device storage and transmitting the data over a socket.
3 4
 * @copyright  Copyright (C) 2016 Elphel, Inc.
 *
5
 * @par <b>License</b>
6 7 8 9 10 11 12 13 14 15 16 17
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

18 19 20 21 22 23
/**
 * @addtogroup SPECIAL_INCLUDES Special includes
 * These defines are needed to use lseek64, strptime and usleep and should be set before includes
 * @{
 */
/** Needed for lseek64 */
24
#define _LARGEFILE64_SOURCE
25
/** Needed for strptime and usleep */
26
#define _XOPEN_SOURCE
27
/** Needed for usleep */
28
#define _XOPEN_SOURCE_EXTENDED
29
/** @} */
30 31 32 33 34 35 36 37 38

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <fcntl.h>
#include <unistd.h>
#include <string.h>
#include <linux/limits.h>
#include <netinet/in.h>
39 40 41
#include <errno.h>
#include <time.h>
#include <ctype.h>
42
#include <math.h>
43
#include <asm/byteorder.h>
44
#include <sys/statvfs.h>
45 46
#include <sys/mman.h>
#include <sys/socket.h>
47
#include <sys/stat.h>
48

49
#include "camogm_read.h"
50
#include "index_list.h"
51

52 53
/** @brief Offset in Exif where TIFF header starts */
#define TIFF_HDR_OFFSET           12
54
/** @brief The date and time format of Exif field */
55
#define EXIF_DATE_TIME_FORMAT     "%Y:%m:%d %H:%M:%S"
56 57
/** @brief The date and time format of 'find_file' command */
#define EXIF_TIMESTAMP_FORMAT     "%04d:%02d:%02d_%02d:%02d:%02d"
58 59
/** @brief The format string used for file parameters reporting. Time and port number are extracted from Exif */
#define INDEX_FORMAT_STR          "port_number=%d;unix_time=%ld;usec_time=%06u;offset=0x%010llx;file_size=%u\n"
60
/** @brief The delimiters used to separate several commands in one command string sent over socket */
61
#define CMD_DELIMITER             "/?"
62
/** @brief The length of a buffer for command string */
63
#define CMD_BUFF_LEN              1024
64
/** @brief 64 bit mask to align offsets to 4 kb page boundary */
65
#define PAGE_BOUNDARY_MASK        0xffffffffffffe000
66 67 68 69
/** @brief The size of read buffer in bytes. The data will be read from disk in blocks of this size */
#define PHY_BLK_SZ                4096
/** @brief Include or exclude file start and stop markers from resulting file. This must be set to 1 for JPEG files */
#define INCLUDE_MARKERS           1
70 71 72
/** @brief The size of file search window. This window is memory mapped. */
#define SEARCH_SIZE_WINDOW        ((uint64_t)4 * (uint64_t)1048576)
/** @brief Time window (in seconds) used for disk index search. Index within this window is considered a candidate */
73
#define SEARCH_TIME_WINDOW        60
74
/** @brief File starting marker on a raw device. It corresponds to SOI JPEG marker */
75
static unsigned char elphelst[] = {0xff, 0xd8};
76
/** @brief File ending marker on a raw device. It corresponds to EOI JPEG marker */
77
static unsigned char elphelen[] = {0xff, 0xd9};
78 79 80 81 82 83 84 85 86
static const struct iovec elphel_st = {
		.iov_base = elphelst,
		.iov_len = sizeof(elphelst)
};
static const struct iovec elphel_en = {
		.iov_base = elphelen,
		.iov_len = sizeof(elphelen)
};

87 88 89
/** @brief X Macro for commands. Add new commands to @e COMMAND_TABLE
 * @{
 */
90 91 92 93 94
#define COMMAND_TABLE \
	X(CMD_BUILD_INDEX, "build_index") \
	X(CMD_GET_INDEX, "get_index") \
	X(CMD_READ_DISK, "read_disk") \
	X(CMD_READ_FILE, "read_file") \
95
	X(CMD_FIND_FILE, "find_file") \
96 97
	X(CMD_NEXT_FILE, "next_file") \
	X(CMD_PREV_FILE, "prev_file") \
98 99 100
	X(CMD_READ_ALL_FILES, "read_all_files") \
	X(CMD_STATUS, "status")

101
/** @enum socket_commands */
102 103 104 105 106 107 108 109 110 111 112
#define X(a, b) a,
enum socket_commands {
	COMMAND_TABLE
};
#undef X

#define X(a, b) b,
static const char *cmd_list[] = {
	COMMAND_TABLE
};
#undef X
113
/** @} */
114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131

/**
 * @enum match_result
 * @brief The result codes for file markers search function
 * @var match_result::MATCH_FOUND
 * File marker was found in the current data buffer
 * @var match_result::MATCH_NOT_FOUND
 * File marker was not found in the current data buffer
 * @var match_result::MATCH_PARTIAL
 * Only a fraction of a marker was found in the end of the current data buffer
 */
enum match_result {
	MATCH_FOUND         = -1,
	MATCH_NOT_FOUND     = -2,
	MATCH_PARTIAL       = -3
};

/**
132 133 134 135 136 137
 * @enum search_state
 * @brief The state of the program during file search in raw device buffer
 * @var search_state::SEARCH_SKIP
 * Skip data in read buffer
 * @var search_state::SEARCH_FILE_DATA
 * The data in read buffer is a file data with known start offset
138
 */
139 140 141 142
enum search_state {
	SEARCH_SKIP,
	SEARCH_FILE_DATA
};
143

144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
/**
 * @brief Exif data format table.
 *
 * This table is used to convert Exif data format to its byte length and then calculate the length
 * of a record in Image File Directory. The data format value 0 does not exist thus the first
 * element of the array is 0 too which gives zero length field. Exif data format can one of the
 * following:
 * Value | Format            | Bytes/Components
 * ------|-------------------|-----------------
 *  1    | unsigned byte     | 1
 *  2    | ascii string      | 1
 *  3    | unsigned short    | 2
 *  4    | unsigned long     | 4
 *  5    | unsigned rational | 8
 *  6    | signed byte       | 1
 *  7    | undefined         | 1
 *  8    | signed short      | 2
 *  9    | signed long       | 4
 *  10   | signed rational   | 8
 *  11   | signed float      | 4
 *  12   | double float      | 8
 */
const unsigned char exif_data_fmt[] = {
		0, 1, 1, 2, 4, 8, 1,
		1, 2, 4, 8, 4, 8
};

/**
 * @struct ifd_entry
 * @brief Represents a single Image File Directory record
 * @var ifd_entry::tag
 * Exif tag number
 * @var ifd_entry::format
 * Data format in the record
 * @var ifd_entry::len
 * The number of components in the record
 * @var ifd_entry::offset
 * The data value or offset to the data
 */
struct ifd_entry {
	unsigned short tag;
	unsigned short format;
	unsigned long len;
	unsigned long offset;
};

/**
 * @struct tiff_hdr
 * @brief TIFF header structure
 * @var tiff_hdr::byte_order
 * Motorola (MM) or Inter (II) byte order
 * @var tiff_hdr::mark
 * Tag mark
 * @var tiff_hdr::offset
 * Offset to first IFD
 */
struct tiff_hdr {
	unsigned short byte_order;
	unsigned short mark;
	unsigned long offset;
};

206 207 208 209 210 211 212 213 214 215 216 217 218
/**
 * @struct crb_ptrs
 * @brief A set of vectors pointing to a file marker crossing read buffer boundary.
 * @var crb_ptrs::first_buff
 * Points to the end of first read buffer where a file marker starts
 * @var crb_ptrs::second_buff
 * Points to the start of second read buffer where a file marker ends
 */
struct crb_ptrs {
	struct iovec        first_buff;
	struct iovec        second_buff;
};

219 220 221 222 223 224 225 226 227 228 229 230 231
/**
 * @struct exit_state
 * @brief Container for the resources which should be freed before exit
 * @var exit_state::state
 * Pointer to #camogm_state structure containing current program state. This structure holds
 * file descriptors and memory mapped regions which should be closed and unmapped accordingly
 * @var exit_state::idir
 * Pointer to disk index directory. The nodes of this directory are dynamically allocated and must be freed
 * @var exit_state::sockfd_const
 * Socket descriptor
 * @var exit_state::sockfd_temp
 * Socket descriptor
 */
232 233
struct exit_state {
	camogm_state *state;
234
	struct disk_idir *idir;
235
	struct disk_idir *sparse_idir;
236 237
	int *sockfd_const;
	int *sockfd_temp;
238 239
};

240
static inline void exit_thread(void *arg);
241
static void build_index(camogm_state *state, struct disk_idir *idir);
242 243
static int mmap_disk(rawdev_buffer *rawdev, const struct range *range);
static int munmap_disk(rawdev_buffer *rawdev);
244

245 246 247
/**
 * @brief Debug function, prints the content of disk index directory
 * @param[in]   idir   pointer to disk index directory to be printed
248
 * @return      None
249
 */
250 251 252 253 254
void dump_index_dir(const struct disk_idir *idir)
{
	struct disk_index *ind = idir->head;

	while (ind != NULL) {
255
		fprintf(debug_file, INDEX_FORMAT_STR,
256
				ind->port, ind->rawtime, ind->usec, ind->f_offset, ind->f_size);
257
		fprintf(debug_file, "\tCurrent pointer: 0x%p, Prev pointer: 0x%p, next pointer: 0x%p\n", ind, ind->prev, ind->next);
258 259 260 261
		ind = ind->next;
	}
}

262 263 264 265 266 267 268 269 270
/**
 * @brief Find pattern in a data buffer
 *
 * This function searches for the first occurrence of pattern in a data buffer and returns a pointer to
 * the position of this pattern in the buffer.
 * @param[in]   buff_ptr  pointer to an array of char values where the pattern should be found
 * @param[in]   buff_sz   size of the data array
 * @param[in]   pattern   pointer to an array of char values containing pattern
 * @param[in]   pt_sz     size of the pattern array
271
 * @param[in]   add_pattern include or exclude marker from resulting buffer offset
272
 * @return      The index in data buffer where pattern matches or error code from #match_result if it was not found
273
 */
274 275
static int find_marker(const unsigned char * restrict buff_ptr, ssize_t buff_sz, const unsigned char * restrict pattern, ssize_t pt_sz,
		int add_pattern)
276 277 278 279 280 281 282 283 284 285 286 287 288 289
{
	int ret = MATCH_NOT_FOUND;
	int j = 0;
	int i;

	for (i = 0; i < buff_sz; i++) {
		if (buff_ptr[i] != pattern[j]) {
			// current symbol in data buffer and first symbol of pattern does not match
			j = 0;
		} else if (buff_ptr[i] == pattern[j] && j < pt_sz - 1) {
			// pattern symbol match
			j++;
		} else if (buff_ptr[i] == pattern[j] && j == pt_sz - 1) {
			// last pattern symbol match
290 291 292 293
			if (add_pattern)
				ret = i;
			else
				ret = i - j;
294 295 296 297 298 299 300 301 302 303 304
			j = 0;
			break;
		}
	}
	if (j > 0) {
		// partial match found in the end of data buffer, we need more data for further comparison
		ret = MATCH_PARTIAL;
	}
	return ret;
}

305 306 307 308 309 310 311 312 313 314 315 316
/**
 * @brief Find pattern in a data buffer in reverse order
 *
 * This function searches for the first occurrence of pattern in a data buffer and returns a pointer to
 * the position of this pattern in the buffer.
 * @param[in]   buff_ptr  pointer to an array of char values where the pattern should be found
 * @param[in]   buff_sz   size of the data array
 * @param[in]   pattern   pointer to an array of char values containing pattern
 * @param[in]   pt_sz     size of the pattern array
 * @param[in]   add_pattern include or exclude marker from resulting buffer offset
 * @return      The index in data buffer where pattern matches or error code from #match_result if it was not found
 */
317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346
static int find_marker_backward(const unsigned char * restrict buff_ptr, ssize_t buff_sz, const unsigned char * restrict pattern, ssize_t pt_sz,
		int add_pattern)
{
	int ret = MATCH_NOT_FOUND;
	int j = 0;

	for (int i = buff_sz - 1; i > 0; i--) {
		if (buff_ptr[i] != pattern[j]) {
			// current symbol in data buffer and last symbol of pattern does not match
			j = pt_sz - 1;
		} else if (buff_ptr[i] == pattern[j] && j > 0) {
			// pattern symbol match
			j--;
		} else if (buff_ptr[i] == pattern[j] && j == 0) {
			// first pattern symbol match
			if (add_pattern)
				ret = i - j;
			else
				ret = i;
			j = 0;
			break;
		}
	}
	if (j > 0) {
		// partial match found in the end of data buffer, we need more data for further comparison
		ret = MATCH_PARTIAL;
	}
	return ret;
}

347 348 349 350 351
/**
 * @brief Converts byte order for all fields in #ifd_entry structure
 * @param[in,out] ifd   a pointer to a structure which should be converted
 * @return        None
 */
352
static void ifd_byte_order(struct ifd_entry *ifd)
353 354 355 356 357 358 359 360 361 362 363 364
{
	ifd->tag = __be16_to_cpu(ifd->tag);
	ifd->format = __be16_to_cpu(ifd->format);
	ifd->len = __be32_to_cpu(ifd->len);
	ifd->offset = __be32_to_cpu(ifd->offset);
	if (exif_data_fmt[ifd->format] == 2) {
		ifd->offset = (ifd->offset >> 16) & 0xffff;
	}
}

/**
 * @brief Convert byte order for all fields in #tiff_hdr structure
365
 * @param[in,out] hdr   a pointer to a structure which should be converted
366 367
 * @return        None
 */
368
static void hdr_byte_order(struct tiff_hdr *hdr)
369 370 371 372 373 374 375
{
	hdr->byte_order = __be16_to_cpu(hdr->byte_order);
	hdr->mark = __be16_to_cpu(hdr->mark);
	hdr->offset = __be32_to_cpu(hdr->offset);
}

/**
376
 * @brief Read a string from Exif for a given record.
377 378 379 380
 * @param[in,out]   rawdev   pointer to #rawdev_buffer structure containing
 * the current state of raw device buffer
 * @param[in]       tag     Exif image file directory record containing string offset
 * @param[out]      buff    buffer for the string to be read
381
 * @return          The number of bytes placed to the read buffer
382
 */
383
static size_t exif_get_text(rawdev_buffer *rawdev, struct ifd_entry *tag, char *buff)
384
{
385 386
	size_t bytes = 0;
	size_t str_len = tag->len * exif_data_fmt[tag->format];
387
	uint64_t curr_pos = rawdev->file_start + TIFF_HDR_OFFSET + tag->offset;
388

389 390
	lseek64(rawdev->rawdev_fd, curr_pos, SEEK_SET);
	bytes = read(rawdev->rawdev_fd, buff, str_len);
391

392
	return bytes;
393 394 395
}

/**
396
 * @brief Read Exif data from the file starting from #rawdev_buffer::file_start offset and
397 398 399 400
 * create a new node corresponding to the offset
 * @param[in,out]   rawdev   pointer to #rawdev_buffer structure containing
 * the current state of raw device buffer
 * @param[out]      indx     pointer to new disk index node
401
 * @return          0 if new node was successfully created and -1 otherwise
402
 */
403
static int read_index(rawdev_buffer *rawdev, struct disk_index **indx)
404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465
{
	int ret = 0;
	int process = 2;
	uint32_t data32;
	uint16_t num_entries = 0;
	uint64_t curr_pos;
	uint64_t subifd_offset = 0;
	struct tiff_hdr hdr;
	struct ifd_entry ifd;
	struct ifd_entry ifd_page_num = {0};
	struct ifd_entry ifd_date_time = {0};
	struct ifd_entry ifd_subsec = {0};
	struct disk_index *node = NULL;
	unsigned char read_buff[TIFF_HDR_OFFSET] = {0};
	char str_buff[SMALL_BUFF_LEN] = {0};
	uint64_t save_pos = lseek64(rawdev->rawdev_fd, 0, SEEK_CUR);

	if (indx == NULL)
		return -1;

	curr_pos = rawdev->file_start;
	lseek64(rawdev->rawdev_fd, curr_pos, SEEK_SET);
	if (read(rawdev->rawdev_fd, read_buff, sizeof(read_buff)) <= 0) {
		lseek64(rawdev->rawdev_fd, save_pos, SEEK_SET);
		return -1;
	}
	if (read_buff[2] == 0xff && read_buff[3] == 0xe1) {
		// get IFD0 offset from TIFF header
		read(rawdev->rawdev_fd, &hdr, sizeof(struct tiff_hdr));
		hdr_byte_order(&hdr);
		curr_pos = rawdev->file_start + TIFF_HDR_OFFSET + hdr.offset;
		lseek64(rawdev->rawdev_fd, curr_pos, SEEK_SET);
		// process IFD0 and SubIFD fields
		do {
			read(rawdev->rawdev_fd, &num_entries, sizeof(num_entries));
			num_entries = __be16_to_cpu(num_entries);
			for (int i = 0; i < num_entries; i++) {
				read(rawdev->rawdev_fd, &ifd, sizeof(struct ifd_entry));
				ifd_byte_order(&ifd);
				switch (ifd.tag) {
				case Exif_Image_PageNumber:
					ifd_page_num = ifd;
					break;
				case Exif_Photo_DateTimeOriginal & 0xffff:
					ifd_date_time = ifd;
					break;
				case Exif_Image_ExifTag:
					subifd_offset = ifd.offset;
					break;
				case Exif_Photo_SubSecTimeOriginal & 0xffff:
					ifd_subsec = ifd;
					break;
				}
			}
			// ensure that IFD0 finished correctly (0x00000000 in the end), set file pointer to SubIFD and
			// process remaining fields
			read(rawdev->rawdev_fd, &data32, sizeof(data32));
			process -= (subifd_offset == 0 || data32 != 0) ? 2 : 1;
			curr_pos = rawdev->file_start + TIFF_HDR_OFFSET + subifd_offset;
			lseek64(rawdev->rawdev_fd, curr_pos, SEEK_SET);
		} while (process > 0);

466 467 468 469 470 471 472
		// create and fill new disk index node with Exif data
		if (create_node(&node) == 0) {
			node->f_offset = rawdev->file_start;
			if (ifd_page_num.len != 0) {
				node->port = (uint32_t)ifd_page_num.offset;
			}
			if (ifd_date_time.len != 0) {
473
				struct tm tm = {0};
474 475 476 477 478 479 480 481 482 483 484 485 486 487 488
				exif_get_text(rawdev, &ifd_date_time, str_buff);
				strptime(str_buff, EXIF_DATE_TIME_FORMAT, &tm);
				node->rawtime = mktime(&tm);
			}
			if (ifd_subsec.len != 0) {
				exif_get_text(rawdev, &ifd_subsec, str_buff);
				node->usec = strtoul(str_buff, NULL, 10);
			}
			if (node->rawtime != -1) {
				*indx = node;
			} else {
				free(node);
				ret = -1;
			}
		} else {
489
			ret = -1;
490 491 492
		}
	} else {
		ret = -1;
493 494 495 496 497
	}

	lseek64(rawdev->rawdev_fd, save_pos, SEEK_SET);
	return ret;
}
498 499
/**
 * @brief Calculate the size of current file and update the value in disk index directory
500
 * @param[in,out]   indx       pointer to disk index node which size should be calculated
501 502 503 504 505
 * @param[in]       pos_stop   the offset of the last byte of the current file
 * @return          0 if the file size was successfully updated and -1 otherwise
 * @note @e pos_stop points to the last byte of the file marker thus the size is incremented
 * by 1
 */
506
static int stop_index(struct disk_index *indx, uint64_t pos_stop)
507 508
{
	int ret = 0;
509 510 511

	if (indx != NULL) {
		indx->f_size = pos_stop - indx->f_offset + 1;
512 513 514
	} else {
		ret = -1;
	}
515

516
	return ret;
517 518
}

519 520 521 522 523 524 525 526
/**
 * @brief Detect cases when file marker crosses read buffer boundary
 * @param[in]   from   pointer to current read buffer which can hold the beginning of
 * file marker
 * @param[in]   to     pointer to a buffer containing next chunk of data which can hold
 * the end of file marker
 * @param[in]   marker pointer to a buffer holding file marker to be detect
 * @param[in]   crbp   pointer to a structure which will store two cross border pointers
527
 * @return      A constant of #match_result type
528
 */
529
static int check_edge_case(const struct iovec *from, const struct iovec *to, const struct iovec *marker, struct crb_ptrs *crbp)
530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574
{
	unsigned char *start_ptr = from->iov_base + from->iov_len - marker->iov_len;
	unsigned char *end_ptr = from->iov_base + from->iov_len;
	unsigned char *marker_ptr = marker->iov_base;
	unsigned int bytes_processed = 0;
	int match = 0;

	// search for the first part of marker in the end of *from* array
	while (start_ptr <= end_ptr) {
		if (*start_ptr == *marker_ptr && !match) {
			crbp->first_buff.iov_base = start_ptr;
			crbp->first_buff.iov_len = end_ptr - start_ptr;
			match = 1;
		}
		if (*start_ptr == *marker_ptr && match) {
			marker_ptr++;
			bytes_processed++;
		} else {
			break;
		}
		start_ptr++;
	}
	if (start_ptr != end_ptr) {
		// match not found in the end of *from* array
		return MATCH_NOT_FOUND;
	}

	// search for the second part of marker in the beginning of *to* array
	start_ptr = to->iov_base;
	end_ptr = to->iov_base + (marker->iov_len - bytes_processed);
	while (start_ptr <= end_ptr) {
		if (*start_ptr++ != *marker_ptr++)
			break;
	}
	if (start_ptr != end_ptr) {
		// match not found in the beginning of *to* array
		return MATCH_NOT_FOUND;
	}
	crbp->second_buff.iov_base = to->iov_base;
	crbp->second_buff.iov_len = marker->iov_len - bytes_processed;

	return MATCH_FOUND;
}

/**
575 576 577 578
 * @brief Send mmaped buffer over opened socket
 * @param[in]   sockfd    opened socket descriptor
 * @param[in]   buff      pointer to memory mapped buffer
 * @param[in]   sz        the size of @e buff
579
 * @return      None
580
 */
581
static void send_buffer(int sockfd, unsigned char *buff, size_t sz)
582
{
583
	size_t bytes_left = sz;
584
	ssize_t bytes_written = 0;
585 586 587 588
	size_t offset = 0;

	while (bytes_left > 0) {
		bytes_written = write(sockfd, &buff[offset], bytes_left);
589 590 591 592
		if (bytes_written < 0) {
			perror(__func__);
			return;
		}
593 594 595 596 597
		bytes_left -= bytes_written;
		offset += bytes_written;
	}
}

598 599 600 601 602 603
/**
 * @brief Send file pointed by disk index node over opened socket
 * @param[in,out]   rawdev   pointer to #rawdev_buffer structure containing
 * the current state of raw device buffer
 * @param[in]       indx     disk index directory node
 * @param[in]       sockfd   opened socket descriptor
604
 * @return          0 in case disk index node was sent successfully and -1 otherwise
605
 */
606
static int send_file(rawdev_buffer *rawdev, struct disk_index *indx, int sockfd)
607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627
{
	uint64_t mm_file_start;
	struct range mmap_range;

	mmap_range.from = indx->f_offset & PAGE_BOUNDARY_MASK;
	mmap_range.to = indx->f_offset + indx->f_size;
	mm_file_start = indx->f_offset - mmap_range.from;
	if (mmap_disk(rawdev, &mmap_range) == 0) {
		send_buffer(sockfd, &rawdev->disk_mmap[mm_file_start], indx->f_size);
		if (munmap_disk(rawdev) != 0) {
			D0(fprintf(debug_file, "Unable to unmap memory region\n"));
			return -1;
		}
	} else {
		D0(fprintf(debug_file, "Unable to map disk to memory region:"
				"disk region start = 0x%llx, disk region end = 0x%llx\n", mmap_range.from, mmap_range.to));
		return -1;
	}
	 return 0;
}

628 629 630 631 632 633 634 635
/**
 * @brief Map a piece of raw device buffer to memory
 * @param[in,out]   rawdev   pointer to #rawdev_buffer structure containing
 * the current state of raw device buffer
 * @param[in]       range    pointer to #range structure holding the offsets of
 * mmaped region
 * @return          0 in case the region was mmaped successfully and -1 in case of an error
 */
636
static int mmap_disk(rawdev_buffer *rawdev, const struct range *range)
637 638
{
	int ret = 0;
639
	size_t mmap_size = range->to - range->from;
640 641 642 643 644

	rawdev->rawdev_fd = open(rawdev->rawdev_path, O_RDONLY);
	if (rawdev->rawdev_fd < 0) {
		return -1;
	}
645
	rawdev->disk_mmap = mmap(0, mmap_size, PROT_READ, MAP_SHARED, rawdev->rawdev_fd, range->from);
646
	if (rawdev->disk_mmap == MAP_FAILED) {
647
		rawdev->disk_mmap = NULL;
648 649 650
		close(rawdev->rawdev_fd);
		return -1;
	}
651 652
	rawdev->mmap_offset = range->from;
	rawdev->mmap_current_size = mmap_size;
653 654 655 656

	return ret;
}

657 658 659 660 661 662
/**
 * @brief Unmap currently mapped raw device buffer
 * @param[in,out]   rawdev   pointer to #rawdev_buffer structure containing
 * the current state of raw device buffer
 * @return          0 in case the region was unmapped successfully and -1 in case of an error
 */
663
static int munmap_disk(rawdev_buffer *rawdev)
664
{
665 666
	if (rawdev->disk_mmap == NULL)
		return 0;
667

668
	if (munmap(rawdev->disk_mmap, rawdev->mmap_current_size) != 0)
669 670 671
		return -1;
	if (close(rawdev->rawdev_fd) != 0)
		return -1;
672 673
	rawdev->mmap_offset = 0;
	rawdev->disk_mmap = NULL;
674

675 676 677
	return 0;
}

678 679 680 681 682 683 684 685
/**
 * @brief Check if the file pointed by disk index node is in the memory mapped region
 * @param[in]   range   pointer to #range structure holding the offsets of
 * memory mapped region
 * @param[in]   indx    pointer to the disk index node which should be checked for
 * presence in currently mapped region
 * @return      @b true if the file is the region and @b false otherwise
 */
686
static bool is_in_range(struct range *range, struct disk_index *indx)
687 688 689 690 691 692 693
{
	if (indx->f_offset >= range->from &&
			indx->f_offset <= range->to &&
			(indx->f_offset + indx->f_size) <= range->to)
		return true;
	else
		return false;
694 695
}

696 697 698
/**
 * @brief Prepare socket for communication
 * @param[out]   socket_fd   pointer to socket descriptor
699
 * @param[in]    port_num    socket port number
700
 * @return       None
701
 */
702
static void prep_socket(int *socket_fd, uint16_t port_num)
703 704 705 706 707 708
{
	int opt = 1;
	struct sockaddr_in sock;

	memset((char *)&sock, 0, sizeof(struct sockaddr_in));
	sock.sin_family = AF_INET;
709
	sock.sin_port = htons(port_num);
710 711 712 713 714 715 716
	*socket_fd = socket(AF_INET, SOCK_STREAM, 0);
	setsockopt(*socket_fd, SOL_SOCKET, SO_REUSEADDR, (char *)&opt, sizeof(opt));
	bind(*socket_fd, (struct sockaddr *) &sock, sizeof(struct sockaddr_in));
	listen(*socket_fd, 10);
}

/**
717 718 719
 * @brief Parse command line
 * @param[in]   cmd   pointer to command line buffer, this pointer is
 * updated to point to current command
720 721
 * @return      Positive command number from #socket_commands, -1 if command not
 * recognized and -2 to indicate that the command buffer has been fully processed
722
 */
723
static int parse_command(char **cmd)
724 725 726 727 728 729 730 731 732 733 734 735 736 737 738
{
	size_t cmd_len;
	int cmd_indx = -1;
	char *char_ptr;

	D6(fprintf(debug_file, "Parsing command line: %s\n", *cmd));
	char_ptr = strpbrk(*cmd, CMD_DELIMITER);
	if (char_ptr != NULL) {
		char_ptr[0] = '\0';
		char_ptr++;
		for (int i = 0; i < sizeof(cmd_list) / sizeof(cmd_list[0]); i++) {
			cmd_len = strlen(cmd_list[i]);
			if (strncmp(char_ptr, cmd_list[i], cmd_len) == 0) {
				cmd_indx = i;
				break;
739 740
			}
		}
741 742 743 744 745 746 747 748 749 750 751 752
		*cmd = char_ptr;
	} else {
		cmd_indx = -2;
	}

	return cmd_indx;
}

/**
 * @brief Break HTTP GET string after the command part as we do not need that part. The function
 * finds the first space character after the command part starts and replaces it with null.
 * @param[in,out]   cmd   pointer to HTTP GET string
753
 * @param[in]       cmd_len the length of the command in buffer
754
 * @return          None
755
 */
756
static void trim_command(char *cmd, ssize_t cmd_len)
757 758 759 760 761 762 763 764 765 766 767 768 769
{
	char *ptr_start, *ptr_end;

	if (cmd_len >= 0 && cmd_len < CMD_BUFF_LEN)
		cmd[cmd_len] = '\0';
	ptr_start = strpbrk(cmd, CMD_DELIMITER);
	if (ptr_start) {
		ptr_end = strchr(ptr_start, ' ');
		if (ptr_end)
			ptr_end[0] = '\0';
	}
}

770 771 772 773 774
/**
 * @brief Send a file crossing raw device buffer boundary.
 * Such a file is split into two parts, the one in the end of the buffer and the other
 * in the beginning, and should be sent in two steps
 * @param[in]   rawdev   pointer to #rawdev_buffer structure containing
775
 * the current state of raw device buffer
776 777
 * @param[in]   indx     pointer to the disk index node
 * @param[in]   sockfd   opened socket descriptor
778
 * @return      None
779
 */
780
static void send_split_file(rawdev_buffer *rawdev, struct disk_index *indx, int sockfd)
781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807
{
	ssize_t rcntr = 0;
	ssize_t scntr = 0;
	size_t head_sz = rawdev->end_pos - indx->f_offset;
	size_t tail_sz = indx->f_size - head_sz;
	uint64_t curr_pos = lseek64(rawdev->rawdev_fd, 0, SEEK_CUR);
	unsigned char *buff = malloc(indx->f_size);

	if (buff == NULL)
		return;

	lseek64(rawdev->rawdev_fd, indx->f_offset, SEEK_SET);
	while (rcntr < head_sz && rcntr != -1)
		rcntr += read(rawdev->rawdev_fd, &buff[rcntr], head_sz - rcntr);
	rcntr = 0;
	lseek64(rawdev->rawdev_fd, rawdev->start_pos, SEEK_SET);
	while (rcntr < tail_sz && rcntr != -1)
		rcntr += read(rawdev->rawdev_fd, &buff[head_sz + rcntr], tail_sz - rcntr);

	while (scntr < indx->f_size && scntr != -1)
		scntr += write(sockfd, &buff[scntr], head_sz - scntr);

	fprintf(debug_file, "%s: %d bytes sent\n", __func__, scntr);

	lseek64(rawdev->rawdev_fd, curr_pos, SEEK_SET);
}

808 809 810 811
/**
 * @brief Send the number of files (or disk chunks) found over socket connection
 * @param[in]   sockfd   opened socket descriptor
 * @param[in]   num      the number to be sent
812
 * @return      None
813
 */
814
static void send_fnum(int sockfd, size_t num)
815 816 817 818 819 820 821 822 823
{
	char buff[SMALL_BUFF_LEN] = {0};
	int len;

	len = snprintf(buff, SMALL_BUFF_LEN - 1, "Number of files: %d\n", num);
	buff[len] = '\0';
	write(sockfd, buff, len);
}

824 825 826 827
/**
 * @brief Read file parameters from a string and fill in disk index node structure
 * @param[in]    cmd   pointer to a string with file parameters
 * @param[out]   indx  pointer to disk index node structure
828
 * @return       The number of parameters extracted from the string or -1 in case of an error
829
 */
830
static int get_indx_args(char *cmd, struct disk_index *indx)
831 832 833 834 835 836 837 838
{
	char *cmd_start = strchr(cmd, ':');

	if (cmd_start == NULL)
		return -1;
	return sscanf(++cmd_start, INDEX_FORMAT_STR, &indx->port, &indx->rawtime, &indx->usec, &indx->f_offset, &indx->f_size);
}

839 840 841 842 843 844 845 846
/**
 * @brief Read time stamp from a string and copy time in UNIX format to disk index
 * node structure
 * @param[in]   cmd   pointer to a string withe time stamp
 * @param[out]  indx  pointer to disk index node structure
 * @return      The number of input items matched from the string or -1 in case of an error
 */
static int get_timestamp_args(char *cmd, struct disk_index *indx)
847 848 849 850 851 852 853 854 855 856 857 858 859 860 861
{
	int ret;
	struct tm tm;
	char *cmd_start = strchr(cmd, ':');

	if (cmd_start == NULL)
		return -1;
	ret = sscanf(++cmd_start, EXIF_TIMESTAMP_FORMAT, &tm.tm_year, &tm.tm_mon,
			&tm.tm_mday, &tm.tm_hour, &tm.tm_min, &tm.tm_sec);
	tm.tm_year -= 1900;
	tm.tm_mon -= 1;
	indx->rawtime = mktime(&tm);
	return ret;
}

862 863 864 865 866 867 868 869
/**
 * @brief Define memory mapped disk window where files will be searched
 * @param[in]   r   disk offsets range where memory mapped window will be located
 * @param[out]  s   search window
 * @return      0 if the window was successfully located in the region specified and
 * -1 in case of an error
 */
static int get_search_window(const struct range *r, struct range *s)
870
{
871
	if ((r->to - r->from) < SEARCH_SIZE_WINDOW || r->from > r->to)
872 873 874 875 876 877 878 879
		return -1;

	uint64_t middle = (r->to + r->from) / 2;
	s->from = (middle - SEARCH_SIZE_WINDOW / 2) & PAGE_BOUNDARY_MASK;
	s->to = middle + SEARCH_SIZE_WINDOW / 2;
	return 0;
}

880 881 882 883 884 885 886
/**
 * @brief Find JPEG file in a specified window. The window should be large enough to
 * contain at least one file.
 * @param[in]   rawdev   pointer to #rawdev_buffer structure containing
 * the current state of raw device buffer
 * @param[in]   wnd      pointer to a structure containing the offsets of a search window
 * @param[out]  indx     disk index structure which will hold the offset and size of a file
887
 * @return      0 if a file was found and -1 otherwise
888
 */
889
static int find_in_window(rawdev_buffer *rawdev, const struct range *wnd, struct disk_index **indx)
890 891
{
	int ret = -1;
892
	int pos_start, pos_stop;
893 894 895 896 897

	if (mmap_disk(rawdev, (const struct range *)wnd) == 0) {
		pos_start = find_marker(rawdev->disk_mmap, rawdev->mmap_current_size, elphel_st.iov_base, elphel_st.iov_len, 0);
		if (pos_start >= 0) {
			rawdev->file_start = rawdev->mmap_offset + pos_start;
898 899 900 901 902 903
			if (read_index(rawdev, indx) == 0) {
				pos_stop = find_marker(rawdev->disk_mmap + pos_start, rawdev->mmap_current_size - pos_start,
						elphel_en.iov_base, elphel_en.iov_len, 1);
				stop_index(*indx, rawdev->mmap_offset + pos_stop + pos_start);
				ret = 0;
			}
904 905 906
		}
		munmap_disk(rawdev);
	} else {
907
		D0(fprintf(debug_file, "ERROR: can not memory map region from 0x%llx to 0x%llx\n", wnd->from, wnd->to));
908 909 910 911 912 913 914
	}

	return ret;
}

/**
 * @brief Find a file on disk having time stamp close to the time stamp given.
915
 * @param[in]   rawdev   pointer to #rawdev_buffer structure containing
916
 * the current state of raw device buffer
917
 * @param[in]   idir     pointer to sparse disk index directory where indexes are sorted
918
 * in time order
919 920 921
 * @param[in]   rawtime  time (in UNIX format) of a possible index candidate
 * @return      A pointer to disk index node found or NULL if there were no close
 * index candidates
922
 */
923
static struct disk_index *find_disk_index(rawdev_buffer *rawdev, struct disk_idir *idir, time_t *rawtime)
924 925 926 927
{
	bool process = true;
	struct range range;
	struct range search_window;
928
	struct disk_index *indx_found = NULL;
929 930
	struct disk_index *indx_ret = NULL;
	struct disk_index *nearest_indx = find_nearest_by_time((const struct disk_idir *)idir, *rawtime);
931 932 933 934 935 936

	// define disk offsets where search will be performed
	if (nearest_indx == NULL) {
		range.from = rawdev->start_pos;
		range.to = rawdev->end_pos;
	} else {
937
		if (*rawtime > nearest_indx->rawtime) {
938 939 940 941 942 943 944 945 946 947 948 949 950 951
			range.from = nearest_indx->f_offset;
			if (nearest_indx->next != NULL)
				range.to = nearest_indx->next->f_offset;
			else
				range.to = rawdev->end_pos;
		} else {
			range.to = nearest_indx->f_offset;
			if (nearest_indx->prev != NULL)
				range.from = nearest_indx->prev->f_offset;
			else
				range.from = rawdev->start_pos;
		}
	}

952
	D6(fprintf(debug_file, "Starting search in range: from 0x%llx, to 0x%llx\n", range.from, range.to));
953 954 955

	while (process && get_search_window(&range, &search_window) == 0) {
		indx_found = NULL;
956 957
		if (find_in_window(rawdev, &search_window, &indx_found) == 0) {
			double time_diff = difftime(indx_found->rawtime, *rawtime);
958 959 960 961 962 963 964 965 966 967 968
			if (fabs(time_diff) > SEARCH_TIME_WINDOW) {
				// the index found is not within search time window, update sparse index directory and
				// define a new search window
				if (time_diff > 0) {
					range.to = search_window.from;
				} else {
					range.from = search_window.to;
				}
			} else {
				// the index found is within search time window, stop search and return
				process = false;
969
				indx_ret = indx_found;
970 971 972
			}
			insert_node(idir, indx_found);
		} else {
973
			// index is not found in the search window, move toward the start of the range
974 975 976 977
			range.to = search_window.from;
		}
	}

978
	D6(fprintf(debug_file, "\nSparse index directory dump, %d nodes:\n", idir->size));
979 980
	dump_index_dir(idir);

981
	return indx_ret;
982 983
}

984
/**
985
 * @brief Raw device buffer reading function.
986
 *
987 988 989
 * This function is started in a separated thread right after the application has started. It opens a
 * communication socket, waits for commands sent over the socket and process them.
 * @param[in, out]   arg   pointer to #camogm_state structure
990
 * @return           None
991 992
 * @warning The main processing loop of the function is enclosed in @e pthread_cleanup_push and @e pthread_cleanup_pop
 * calls. The effect of use of normal @b return or @b break to prematurely leave this loop is undefined.
993
 * @todo Print unrecognized command to debug output file
994 995 996 997
 */
void *reader(void *arg)
{
	int sockfd, fd;
998
	int disk_chunks;
999 1000 1001
	int cmd;
	char cmd_buff[CMD_BUFF_LEN] = {0};
	char *cmd_ptr;
1002
	char send_buff[CMD_BUFF_LEN] = {0};
1003
	bool transfer;
1004
	ssize_t cmd_len;
1005 1006
	size_t mm_file_start, mm_file_size;
	size_t file_cntr;
1007 1008
	camogm_state *state = (camogm_state *)arg;
	rawdev_buffer *rawdev = &state->rawdev;
1009 1010
	struct range mmap_range;
	struct disk_index *disk_indx, *cross_boundary_indx;
1011 1012
	struct disk_idir index_dir;
	struct disk_idir index_sparse;
1013 1014
	struct exit_state exit_state = {
			.state = state,
1015
			.idir = &index_dir,
1016
			.sparse_idir = &index_sparse,
1017
			.sockfd_const = &sockfd,
1018
			.sockfd_temp = &fd
1019
	};
1020 1021
	memset(&index_dir, 0, sizeof(struct disk_idir));
	memset(&index_sparse, 0, sizeof(struct disk_idir));
1022

1023
	prep_socket(&sockfd, state->sock_port);
1024
	pthread_cleanup_push(exit_thread, &exit_state);
1025 1026 1027 1028
	while (true) {
		fd = accept(sockfd, NULL, 0);
		if (fd == -1)
			continue;
1029 1030 1031 1032 1033 1034 1035 1036 1037
		if (state->prog_state == STATE_STOPPED && state->rawdev_op) {
			pthread_mutex_lock(&state->mutex);
			state->prog_state = STATE_READING;
			pthread_mutex_unlock(&state->mutex);
		} else {
			close(fd);
			D0(fprintf(debug_file, "Can not change state of the program, check settings\n"));
			continue;
		}
1038 1039 1040
		cmd_len = read(fd, cmd_buff, sizeof(cmd_buff) - 1);
		cmd_ptr = cmd_buff;
		trim_command(cmd_ptr, cmd_len);
1041
		while ((cmd = parse_command(&cmd_ptr)) != -2 && state->rawdev.thread_state != STATE_CANCEL) {
1042
			if (cmd >= 0)
1043
				D6(fprintf(debug_file, "Got command '%s', number %d\n", cmd_list[cmd], cmd));
1044 1045
			switch (cmd) {
			case CMD_BUILD_INDEX:
1046
				// scan raw device buffer and create disk index directory
1047 1048 1049 1050 1051
				if (index_dir.size != 0) {
					delete_idir(&index_dir);
				}
				build_index(state, &index_dir);
				D3(fprintf(debug_file, "%d files read from %s\n", index_dir.size, state->rawdev.rawdev_path));
1052 1053
				break;
			case CMD_GET_INDEX:
1054 1055 1056 1057 1058
				// send the content of disk index directory over socket
				if (index_dir.size > 0) {
					int len;
					disk_indx = index_dir.head;
					while (disk_indx != NULL) {
1059
						len = snprintf(send_buff, CMD_BUFF_LEN - 1, INDEX_FORMAT_STR,
1060 1061 1062 1063 1064 1065 1066 1067 1068
								disk_indx->port, disk_indx->rawtime, disk_indx->usec, disk_indx->f_offset, disk_indx->f_size);
						send_buff[len] = '\0';
						write(fd, send_buff, len);
						disk_indx = disk_indx->next;
					}
				} else {
					D0(fprintf(debug_file, "Index directory does not contain any files. Try to rebuild index "
							"directory with 'build_index' command\n"));
				}
1069 1070
				break;
			case CMD_READ_DISK:
1071
				// mmap raw device buffer in MMAP_CHUNK_SIZE chunks and send them over socket
1072
				mmap_range.from = rawdev->start_pos & PAGE_BOUNDARY_MASK;
1073
				mmap_range.to = mmap_range.from + rawdev->mmap_default_size;
1074 1075 1076
				disk_chunks = (size_t)ceil((double)(rawdev->end_pos - rawdev->start_pos) / (double)rawdev->mmap_default_size);
				transfer = true;
				mm_file_start = rawdev->start_pos;
1077
				mm_file_size = rawdev->mmap_default_size - rawdev->start_pos;
1078
				send_fnum(fd, disk_chunks);
1079
				close(fd);
1080
				while (disk_chunks > 0 && transfer && state->rawdev.thread_state != STATE_CANCEL) {
1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093
					fd = accept(sockfd, NULL, 0);
					if (mmap_disk(rawdev, &mmap_range) == 0) {
						send_buffer(fd, &rawdev->disk_mmap[mm_file_start], mm_file_size);
					} else {
						transfer = false;
						D0(fprintf(debug_file, "Unable to map disk to memory region:"
								"disk region start = 0x%llx, disk region end = 0x%llx\n", mmap_range.from, mmap_range.to));
					}
					if (munmap_disk(rawdev) != 0) {
						transfer = false;
						D0(fprintf(debug_file, "Unable to unmap memory region\n"));
					}
					mm_file_start = 0;
1094
					mm_file_size = rawdev->mmap_default_size;
1095
					disk_chunks--;
1096
					mmap_range.from = mmap_range.to;
1097
					mmap_range.to = mmap_range.from + rawdev->mmap_default_size;
1098
					if (mmap_range.to > rawdev->end_pos) {
1099
						mmap_range.to = rawdev->end_pos;
1100 1101
						mm_file_size = mmap_range.to - mmap_range.from;
					}
1102 1103
					close(fd);
				}
1104 1105
				break;
			case CMD_READ_FILE:
1106
				// read single file by offset given
1107 1108 1109 1110
				if (index_dir.size > 0) {
					struct disk_index indx;
					if (get_indx_args(cmd_ptr, &indx) > 0 &&
							(disk_indx = find_by_offset(&index_dir, indx.f_offset)) != NULL){
1111 1112 1113 1114 1115
						send_file(rawdev, disk_indx, fd);
					}
				}
				break;
			case CMD_FIND_FILE: {
1116
				// find file by time stamp
1117
				struct disk_index indx;
1118
				struct disk_index *indx_ptr = NULL;
1119
				if (get_timestamp_args(cmd_ptr, &indx) > 0) {
1120 1121 1122 1123
					if (index_dir.size == 0) {
						indx_ptr = find_disk_index(rawdev, &index_sparse, &indx.rawtime);
						if (indx_ptr != NULL)
							index_sparse.curr_indx = indx_ptr;
1124
					} else {
1125
						indx_ptr = find_nearest_by_time(&index_dir, indx.rawtime);
1126
					}
1127 1128 1129 1130 1131 1132
					if (indx_ptr != NULL)
						send_file(rawdev, indx_ptr, fd);
				}
				break;
			}
			case CMD_NEXT_FILE: {
1133
				// read next file after previously found file
1134 1135 1136 1137
				struct range rng;
				struct disk_index *new_indx = NULL;
				struct disk_index *indx_ptr = NULL;
				uint64_t len;
1138
				if (index_sparse.curr_indx != NULL) {
1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154
					if (index_sparse.curr_indx->next != NULL) {
						len = index_sparse.curr_indx->next->f_offset - index_sparse.curr_indx->f_offset - 1;
						if (len > 0) {
							rng.from = index_sparse.curr_indx->f_offset + index_sparse.curr_indx->f_size + 1;
							rng.to = index_sparse.curr_indx->next->f_offset;
						} else {
							indx_ptr = index_sparse.curr_indx->next;
						}
					} else {
						rng.from = index_sparse.curr_indx->f_offset + index_sparse.curr_indx->f_size;
						rng.to = rawdev->end_pos;
					}
					if (indx_ptr == NULL) {
						rng.from &= PAGE_BOUNDARY_MASK;
						if (rng.to - rng.from > rawdev->mmap_default_size)
							rng.to = rng.from + rawdev->mmap_default_size;
1155
						if (find_in_window(rawdev, &rng, &new_indx) == 0) {
1156 1157 1158 1159 1160 1161 1162
							insert_next(&index_sparse, index_sparse.curr_indx, new_indx);
							send_file(rawdev, new_indx, fd);
							index_sparse.curr_indx = new_indx;
						}
					} else {
						send_file(rawdev, indx_ptr, fd);
					}
1163
				}
1164
				break;
1165
			}
1166 1167 1168
			case CMD_PREV_FILE: {
				break;
			}
1169
			case CMD_READ_ALL_FILES:
1170 1171
				// read files from raw device buffer and send them over socket; the disk index directory
				// should be built beforehand
1172
				if (index_dir.size > 0) {
1173 1174
					send_fnum(fd, index_dir.size);
					close(fd);
1175 1176 1177 1178 1179 1180
					mmap_range.from = rawdev->start_pos;
					mmap_range.to = rawdev->start_pos + rawdev->mmap_default_size;
					disk_indx = index_dir.head;
					cross_boundary_indx = NULL;
					file_cntr = 0;
					transfer = true;
1181
					while (file_cntr < index_dir.size && disk_indx != NULL && state->rawdev.thread_state != STATE_CANCEL) {
1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220
						if (is_in_range(&mmap_range, disk_indx) && rawdev->disk_mmap != NULL) {
							fd = accept(sockfd, NULL, 0);
							mm_file_start = disk_indx->f_offset - rawdev->mmap_offset;
							send_buffer(fd, &rawdev->disk_mmap[mm_file_start], disk_indx->f_size);
							close(fd);
							disk_indx = disk_indx->next;
							file_cntr++;
						} else {
							if (munmap_disk(rawdev) == 0) {
								mmap_range.from = disk_indx->f_offset & PAGE_BOUNDARY_MASK;
								mmap_range.to = mmap_range.from + rawdev->mmap_default_size;
								if (mmap_range.to > rawdev->end_pos) {
									mmap_range.to = rawdev->end_pos;
								}
								if (disk_indx->f_offset + disk_indx->f_size <= mmap_range.to) {
									if (mmap_disk(rawdev, &mmap_range) < 0) {
										disk_indx = NULL;
										D0(fprintf(debug_file, "Unable to map disk to memory region:"
												" disk region start = 0x%llx, disk region end = 0x%llx\n", mmap_range.from, mmap_range.to));
									}
								} else {
									cross_boundary_indx = disk_indx;
									disk_indx = NULL;
								}
							} else {
								disk_indx = NULL;
								D0(fprintf(debug_file, "Unable to unmap memory region\n"));
							}
						}
					}
					munmap_disk(rawdev);
					if (cross_boundary_indx != NULL) {
						send_split_file(rawdev, cross_boundary_indx, fd);
						close(fd);
					}
				} else {
					D0(fprintf(debug_file, "Index directory does not contain any files. Try to rebuild index "
							"directory with 'build_index' command\n"));
				}
1221 1222 1223 1224 1225 1226
				break;
			case CMD_STATUS:
				break;
			default:
				D0(fprintf(debug_file, "Unrecognized command is skipped\n"));
			}
1227
		}
1228
		if (is_fd_valid(fd))
1229
			close(fd);
1230 1231 1232
		pthread_mutex_lock(&state->mutex);
		state->prog_state = STATE_STOPPED;
		pthread_mutex_unlock(&state->mutex);
1233
		usleep(COMMAND_LOOP_DELAY);
1234
	}
1235
	pthread_cleanup_pop(0);
1236 1237

	return (void *) 0;
1238 1239
}

1240 1241 1242 1243 1244
/**
 * @brief Clean up after the reading thread is closed. This function is thread-cancellation handler and it is
 * passed to @e pthread_cleanup_push() function.
 * @param[in]   arg   pointer to #exit_state structure containing resources that
 * should be closed
1245
 * @return      None
1246
 */
1247
static inline void exit_thread(void *arg)
1248 1249 1250
{
	struct exit_state *s = (struct exit_state *)arg;

1251 1252 1253
	if (s->state->rawdev.disk_mmap != NULL)
		munmap(s->state->rawdev.disk_mmap, s->state->rawdev.mmap_current_size);
	if (is_fd_valid(s->state->rawdev.rawdev_fd))
1254
		close(s->state->rawdev.rawdev_fd);
1255 1256
	if (s->idir->size != 0)
		delete_idir(s->idir);
1257 1258
	if (s->sparse_idir-> size != 0)
		delete_idir(s->sparse_idir);
1259
	if (is_fd_valid(*s->sockfd_const))
1260
		close(*s->sockfd_const);
1261
	if (is_fd_valid(*s->sockfd_temp))
1262
		close(*s->sockfd_temp);
1263 1264
}

1265
/**
1266 1267
 * @brief Extract the position and parameters of JPEG files in raw device buffer and
 * build disk index directory for further file extraction.
1268 1269
 *
 * Data from raw device is read to a buffer in #PHY_BLK_SZ blocks. The buffer is
1270 1271
 * then analyzed for JPEG markers stored in #elphelst and #elphelen arrays. The offsets and
 * sizes of the files found in the buffer are recorded to disk index directory.
1272
 * @param[in]   state   a pointer to a structure containing current state
1273 1274
 * @param[out]  idir    a pointer to disk index directory. This directory will contain
 * offset of the files found in the raw device buffer.
1275
 * @return      None
1276
 */
1277
static void build_index(camogm_state *state, struct disk_idir *idir)
1278 1279 1280 1281
{
	const int include_markers = INCLUDE_MARKERS;
	int process;
	int zero_cross;
1282
	int err;
1283 1284
	int pos_start, pos_stop;
	int buff_processed;
1285 1286
	int search_state;
	int idir_result;
1287
	ssize_t rd;
1288 1289
	unsigned char buff[PHY_BLK_SZ];
	unsigned char next_buff[PHY_BLK_SZ];
1290
	unsigned char *active_buff = buff;
1291 1292 1293
	unsigned char *save_from = NULL;
	unsigned char *save_to = NULL;
	uint64_t dev_curr_pos = 0;
1294
	uint64_t include_st_marker, include_en_marker;
1295
	size_t add_stm_len, add_enm_len;
1296
	struct disk_index *node = NULL;
1297

1298 1299 1300 1301
	state->rawdev.rawdev_fd = open(state->rawdev.rawdev_path, O_RDONLY);
	if (state->rawdev.rawdev_fd < 0) {
		D0(perror(__func__));
		D0(fprintf(debug_file, "Error opening raw device %s\n", state->rawdev.rawdev_path));
1302
		return;
1303 1304 1305
	}

	if (include_markers) {
1306 1307 1308 1309
		include_st_marker = 0;
		add_stm_len = elphel_st.iov_len;
		include_en_marker = 1;
		add_enm_len = 0;
1310 1311
	} else {
		include_st_marker = 0;
1312 1313 1314
		add_stm_len = 0;
		include_en_marker = 1;
		add_enm_len = elphel_en.iov_len;
1315
	}
1316

1317 1318
	process = 1;
	zero_cross = 0;
1319 1320
	search_state = SEARCH_SKIP;
	idir_result = 0;
1321
	while (process && state->rawdev.thread_state != STATE_CANCEL) {
1322 1323 1324
		rd = read(state->rawdev.rawdev_fd, buff, sizeof(buff));
		err = errno;
		if ((rd > 0) && (dev_curr_pos + rd > state->rawdev.end_pos)) {
1325
			// read pointer jumped over the raw storage buffer end, truncate excessive data
Mikhail Karpenko's avatar
Mikhail Karpenko committed
1326
			D3(fprintf(debug_file, "End of raw storage buffer is reached, will start from the beginning\n"));
1327
			rd = state->rawdev.end_pos - dev_curr_pos;
1328
			zero_cross = 1;
1329 1330
			lseek64(state->rawdev.rawdev_fd, state->rawdev.start_pos, SEEK_SET);
			dev_curr_pos = state->rawdev.start_pos;
1331 1332 1333 1334 1335 1336
			if (rd == 0) {
				continue;
			}
		} else if (rd < 0) {
			// read error or read pointer exceeded raw storage capacity, close files and terminate
			process = 0;
Mikhail Karpenko's avatar
Mikhail Karpenko committed
1337
			D0(fprintf(debug_file, "Raw device read was unsuccessful: %s\n", strerror(err)));
1338 1339
		} else if (rd == 0) {
			// end of device file reached
Mikhail Karpenko's avatar
Mikhail Karpenko committed
1340
			D3(fprintf(debug_file, "End of raw storage device file is reached, will start from the beginning\n"));
1341
			zero_cross = 1;
1342 1343
			lseek64(state->rawdev.rawdev_fd, state->rawdev.start_pos, SEEK_SET);
			dev_curr_pos = state->rawdev.start_pos;
1344 1345 1346 1347
		}
		if (process) {
			save_from = buff;
			save_to = buff + rd;
1348
			active_buff = buff;
1349 1350 1351
			buff_processed = 0;
			do {
				// process data in read buffer
1352 1353 1354
				pos_start = find_marker(save_from, save_to - save_from, elphel_st.iov_base, elphel_st.iov_len, include_st_marker);
				pos_stop = find_marker(save_from, save_to - save_from, elphel_en.iov_base, elphel_en.iov_len, include_en_marker);

1355
				node = NULL;
1356 1357 1358
				if (pos_start == MATCH_NOT_FOUND && pos_stop == MATCH_NOT_FOUND) {
					// normal condition, search in progress
					buff_processed = 1;
1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378
				} else if (pos_start >= 0 && pos_stop >= 0 && pos_start > pos_stop) {
					// normal condition, start marker following stop marker found - this indicates a new file
					if (search_state == SEARCH_FILE_DATA) {
						uint64_t disk_pos = dev_curr_pos + pos_stop + (save_from - active_buff);
						idir_result = stop_index(idir->tail, disk_pos);
					}
					if (zero_cross == 0) {
						state->rawdev.file_start = dev_curr_pos + pos_start + (save_from - active_buff);
						idir_result = read_index(&state->rawdev, &node);
						if (idir_result == 0)
							add_node(idir, node);
						search_state = SEARCH_FILE_DATA;
						save_from = save_from + pos_start + add_stm_len;
						// @todo: replace with pointer to current buffer
						save_to = buff + rd;
					} else {
						buff_processed = 1;
						process = 0;
					}
					D6(fprintf(debug_file, "State 'stop current file and start new file'\n"));
1379 1380 1381 1382
				} else if (pos_start >= 0 && pos_stop == MATCH_NOT_FOUND && search_state == SEARCH_SKIP) {
					// normal condition, new file found
					search_state = SEARCH_FILE_DATA;
					state->rawdev.file_start = dev_curr_pos + pos_start + (save_from - active_buff);
1383 1384 1385
					idir_result = read_index(&state->rawdev, &node);
					if (idir_result == 0)
						add_node(idir, node);
1386
					buff_processed = 1;
Mikhail Karpenko's avatar
Mikhail Karpenko committed
1387
					D6(fprintf(debug_file, "New file found. File start position: %llu\n", state->rawdev.file_start));
1388 1389 1390
					D6(fprintf(debug_file, "State 'starting file'\n"));
				} else if (pos_start >= 0 && pos_stop == MATCH_NOT_FOUND && search_state == SEARCH_FILE_DATA) {
					// error condition (normally should not happen), discard current index and start a new one
1391
					buff_processed = 1;
1392
					remove_node(idir, idir->tail);
1393 1394
					if (zero_cross == 0) {
						state->rawdev.file_start = dev_curr_pos + pos_start + (save_from - active_buff);
1395 1396 1397
						idir_result = read_index(&state->rawdev, &node);
						if (idir_result == 0)
							add_node(idir, node);
1398 1399 1400 1401
					} else {
						process = 0;
					}
					D6(fprintf(debug_file, "State 'abnormal start marker, remove current disk index from directory and skip data'\n"));
1402
				} else if (pos_start == MATCH_NOT_FOUND && pos_stop >= 0 &&
1403 1404 1405 1406
						search_state == SEARCH_FILE_DATA) {
					// normal condition, save current file size to index directory
					uint64_t disk_pos = dev_curr_pos + pos_stop + (save_from - active_buff);
					search_state = SEARCH_SKIP;
1407
					idir_result = stop_index(idir->tail, disk_pos);
1408 1409 1410
					buff_processed = 1;
					if (zero_cross)
						process = 0;
Mikhail Karpenko's avatar
Mikhail Karpenko committed
1411
					D6(fprintf(debug_file, "State 'finishing file'\n"));
1412 1413 1414 1415 1416 1417
				} else if (pos_start == MATCH_NOT_FOUND && pos_stop >= 0 && search_state == SEARCH_SKIP) {
					// error condition (normally should not happen), drop current read buffer and do nothing
					buff_processed = 1;
					D6(fprintf(debug_file, "State 'abnormal stop marker, skip data'\n"));
				} else if (pos_start == MATCH_PARTIAL && search_state == SEARCH_SKIP) {
					// partial start marker found in the end of read buffer, get next chunk of data and try to find marker there
1418 1419 1420 1421 1422 1423 1424 1425 1426
					enum match_result result;
					struct crb_ptrs field_markers;
					struct iovec next_chunk = {
							.iov_base = next_buff,
					};
					struct iovec curr_chunk = {
							.iov_base = save_from,
							.iov_len = save_to - save_from
					};
1427
					ssize_t next_rd = read(state->rawdev.rawdev_fd, next_buff, sizeof(next_buff));
1428 1429
					next_chunk.iov_len = next_rd;
					result = check_edge_case(&curr_chunk, &next_chunk, &elphel_st, &field_markers);
1430 1431 1432
					if (result == MATCH_FOUND) {
						search_state = SEARCH_FILE_DATA;
						state->rawdev.file_start = dev_curr_pos + pos_start + (save_from - active_buff);
1433 1434 1435
						idir_result = read_index(&state->rawdev, &node);
						if (idir_result == 0)
							add_node(idir, node);
Mikhail Karpenko's avatar
Mikhail Karpenko committed
1436
						D6(fprintf(debug_file, "File start position: %llu\n", state->rawdev.file_start));
1437 1438 1439 1440 1441 1442
						save_from = next_chunk.iov_base + field_markers.second_buff.iov_len;
						save_to = next_chunk.iov_base + next_chunk.iov_len;
					} else {
						save_from = next_chunk.iov_base;
						save_to = next_chunk.iov_base + next_chunk.iov_len;
					}
1443 1444
					dev_curr_pos += next_rd;
					active_buff = next_buff;
Mikhail Karpenko's avatar
Mikhail Karpenko committed
1445
					D6(fprintf(debug_file, "State 'check elphel_st cross boundary'; result = %d\n", result));
1446
				} else if (pos_stop == MATCH_PARTIAL && search_state == SEARCH_FILE_DATA) {
1447 1448 1449 1450 1451 1452 1453 1454 1455 1456
					// partial end marker found in the end of read buffer - get next chunk of data and try to find marker there
					enum match_result result;
					struct crb_ptrs field_markers;
					struct iovec next_chunk = {
							.iov_base = next_buff,
					};
					struct iovec curr_chunk = {
							.iov_base = save_from,
							.iov_len = save_to - save_from
					};
1457
					ssize_t next_rd = read(state->rawdev.rawdev_fd, next_buff, sizeof(next_buff));
1458 1459 1460
					next_chunk.iov_len = next_rd;
					result = check_edge_case(&curr_chunk, &next_chunk, &elphel_en, &field_markers);
					if (result == MATCH_FOUND) {
1461 1462
						search_state = SEARCH_SKIP;
						uint64_t disk_pos = dev_curr_pos + pos_stop + (save_from - active_buff);
1463
						idir_result = stop_index(idir->tail, disk_pos);
1464 1465 1466 1467 1468 1469
						save_from = next_chunk.iov_base + field_markers.second_buff.iov_len;
						save_to = next_chunk.iov_base + next_chunk.iov_len;
					} else {
						save_from = next_chunk.iov_base;
						save_to = next_chunk.iov_base + next_chunk.iov_len;
					}
1470 1471
					dev_curr_pos += next_rd;
					active_buff = next_buff;
Mikhail Karpenko's avatar
Mikhail Karpenko committed
1472
					D6(fprintf(debug_file, "State 'check elphel_en' cross boundary:; result = %d\n", result));
1473 1474
				} else {
					// no markers found and new file has not bee started yet - skip data
1475 1476
					D6(fprintf(debug_file, "Undefined state: pos_start = %d, pos_stop = %d, search_state = %d\n",
							pos_start, pos_stop, search_state));
1477 1478 1479 1480
					buff_processed = 1;
					if (zero_cross)
						process = 0;
				}
1481 1482
			} while (buff_processed == 0 && idir_result == 0);
			if (idir_result != 0) {
1483