Loading camogm.c +2 −2 Original line number Diff line number Diff line Loading @@ -812,7 +812,7 @@ void camogm_set_prefix(camogm_state *state, const char * p, path_type type) D0(fprintf(debug_file, "WARNING: raw device write initiated\n")); state->rawdev_op = 1; /* debug code follows */ state->rawdev.end_pos = 134217728; state->rawdev.end_pos = 128 * 1048576; /* end of debug code */ } } Loading Loading @@ -1687,7 +1687,7 @@ int main(int argc, char *argv[]) "format and stores the result files.\n\n"; int ret; int opt; uint16_t port_num; uint16_t port_num = 0; char pipe_name_str[ELPHEL_PATH_MAX] = {0}; if ((argc < 5) || (argv[1][1] == '-')) { Loading camogm_read.c +309 −28 Original line number Diff line number Diff line Loading @@ -53,6 +53,8 @@ #define TIFF_HDR_OFFSET 12 /** @brief The date and time format of Exif field */ #define EXIF_DATE_TIME_FORMAT "%Y:%m:%d %H:%M:%S" #define EXIF_TIMESTAMP_FORMAT "%d:%d:%d_%d:%d:%d" //#define EXIF_TIMESTAMP_FORMAT "%04d:%02d:%02d_%02d:%02d:%02d" /** @brief The format string used for file parameters reporting. Time and port number are extracted from Exif */ #define INDEX_FORMAT_STR "port_number=%d;unix_time=%ld;usec_time=%06u;offset=0x%010llx;file_size=%u\n" /** @brief The delimiters used to separate several commands in one command string sent over socket */ Loading Loading @@ -88,6 +90,7 @@ static const struct iovec elphel_en = { X(CMD_GET_INDEX, "get_index") \ X(CMD_READ_DISK, "read_disk") \ X(CMD_READ_FILE, "read_file") \ X(CMD_FIND_FILE, "find_file") \ X(CMD_READ_ALL_FILES, "read_all_files") \ X(CMD_STATUS, "status") Loading Loading @@ -225,12 +228,15 @@ struct crb_ptrs { struct exit_state { camogm_state *state; struct disk_idir *idir; struct disk_idir *sparse_idir; int *sockfd_const; int *sockfd_temp; }; static inline void exit_thread(void *arg); static void build_index(camogm_state *state, struct disk_idir *idir); static int mmap_disk(rawdev_buffer *rawdev, const struct range *range); static int munmap_disk(rawdev_buffer *rawdev); /** * @brief Debug function, prints the content of disk index directory Loading @@ -241,9 +247,11 @@ void dump_index_dir(const struct disk_idir *idir) { struct disk_index *ind = idir->head; printf("Head pointer = 0x%p, tail pointer = 0x%p\n", idir->head, idir->tail); while (ind != NULL) { fprintf(debug_file, INDEX_FORMAT_STR, ind->port, ind->rawtime, ind->usec, ind->f_offset, ind->f_size); fprintf(debug_file, "\tCurrent pointer: 0x%p, Prev pointer: 0x%p, next pointer: 0x%p\n", ind, ind->prev, ind->next); ind = ind->next; } } Loading Loading @@ -326,14 +334,14 @@ static void hdr_byte_order(struct tiff_hdr *hdr) * @param[out] buff buffer for the string to be read * @return The number of bytes placed to the read buffer */ static size_t exif_get_text(camogm_state *state, struct ifd_entry *tag, char *buff) static size_t exif_get_text(rawdev_buffer *rawdev, struct ifd_entry *tag, char *buff) { size_t bytes = 0; size_t str_len = tag->len * exif_data_fmt[tag->format]; uint64_t curr_pos = state->rawdev.file_start + TIFF_HDR_OFFSET + tag->offset; uint64_t curr_pos = rawdev->file_start + TIFF_HDR_OFFSET + tag->offset; lseek64(state->rawdev.rawdev_fd, curr_pos, SEEK_SET); bytes = read(state->rawdev.rawdev_fd, buff, str_len); lseek64(rawdev->rawdev_fd, curr_pos, SEEK_SET); bytes = read(rawdev->rawdev_fd, buff, str_len); return bytes; } Loading Loading @@ -418,12 +426,12 @@ static int save_index(camogm_state *state, struct disk_idir *idir) } if (ifd_date_time.len != 0) { struct tm tm; exif_get_text(state, &ifd_date_time, str_buff); exif_get_text(&state->rawdev, &ifd_date_time, str_buff); strptime(str_buff, EXIF_DATE_TIME_FORMAT, &tm); node->rawtime = mktime(&tm); } if (ifd_subsec.len != 0) { exif_get_text(state, &ifd_subsec, str_buff); exif_get_text(&state->rawdev, &ifd_subsec, str_buff); node->usec = strtoul(str_buff, NULL, 10); } if (node->rawtime != -1) Loading @@ -436,6 +444,96 @@ static int save_index(camogm_state *state, struct disk_idir *idir) return ret; } static int read_index(rawdev_buffer *rawdev, struct disk_index *indx) { int ret = 0; int process = 2; uint32_t data32; uint16_t num_entries = 0; uint64_t curr_pos; uint64_t subifd_offset = 0; struct tiff_hdr hdr; struct ifd_entry ifd; struct ifd_entry ifd_page_num = {0}; struct ifd_entry ifd_date_time = {0}; struct ifd_entry ifd_subsec = {0}; struct disk_index *node = NULL; unsigned char read_buff[TIFF_HDR_OFFSET] = {0}; char str_buff[SMALL_BUFF_LEN] = {0}; uint64_t save_pos = lseek64(rawdev->rawdev_fd, 0, SEEK_CUR); if (indx == NULL) return -1; if (create_node(&node) != 0) return -1; curr_pos = rawdev->file_start; lseek64(rawdev->rawdev_fd, curr_pos, SEEK_SET); if (read(rawdev->rawdev_fd, read_buff, sizeof(read_buff)) <= 0) { lseek64(rawdev->rawdev_fd, save_pos, SEEK_SET); return -1; } if (read_buff[2] == 0xff && read_buff[3] == 0xe1) { // get IFD0 offset from TIFF header read(rawdev->rawdev_fd, &hdr, sizeof(struct tiff_hdr)); hdr_byte_order(&hdr); curr_pos = rawdev->file_start + TIFF_HDR_OFFSET + hdr.offset; lseek64(rawdev->rawdev_fd, curr_pos, SEEK_SET); // process IFD0 and SubIFD fields do { read(rawdev->rawdev_fd, &num_entries, sizeof(num_entries)); num_entries = __be16_to_cpu(num_entries); for (int i = 0; i < num_entries; i++) { read(rawdev->rawdev_fd, &ifd, sizeof(struct ifd_entry)); ifd_byte_order(&ifd); switch (ifd.tag) { case Exif_Image_PageNumber: ifd_page_num = ifd; break; case Exif_Photo_DateTimeOriginal & 0xffff: ifd_date_time = ifd; break; case Exif_Image_ExifTag: subifd_offset = ifd.offset; break; case Exif_Photo_SubSecTimeOriginal & 0xffff: ifd_subsec = ifd; break; } } // ensure that IFD0 finished correctly (0x00000000 in the end), set file pointer to SubIFD and // process remaining fields read(rawdev->rawdev_fd, &data32, sizeof(data32)); process -= (subifd_offset == 0 || data32 != 0) ? 2 : 1; curr_pos = rawdev->file_start + TIFF_HDR_OFFSET + subifd_offset; lseek64(rawdev->rawdev_fd, curr_pos, SEEK_SET); } while (process > 0); // fill disk index node with Exif data and add it to disk index directory node->f_offset = rawdev->file_start; if (ifd_page_num.len != 0) { node->port = (uint32_t)ifd_page_num.offset; } if (ifd_date_time.len != 0) { struct tm tm; exif_get_text(rawdev, &ifd_date_time, str_buff); strptime(str_buff, EXIF_DATE_TIME_FORMAT, &tm); node->rawtime = mktime(&tm); } if (ifd_subsec.len != 0) { exif_get_text(rawdev, &ifd_subsec, str_buff); node->usec = strtoul(str_buff, NULL, 10); } if (node->rawtime != -1) *indx = *node; else ret = -1; } lseek64(rawdev->rawdev_fd, save_pos, SEEK_SET); return ret; } /** * @brief Calculate the size of current file and update the value in disk index directory * @param[in,out] idir pointer to disk index directory Loading Loading @@ -534,6 +632,28 @@ void send_buffer(int sockfd, unsigned char *buff, size_t sz) } } int send_file(rawdev_buffer *rawdev, struct disk_index *indx, int sockfd) { uint64_t mm_file_start; struct range mmap_range; mmap_range.from = indx->f_offset & PAGE_BOUNDARY_MASK; mmap_range.to = indx->f_offset + indx->f_size; mm_file_start = indx->f_offset - mmap_range.from; if (mmap_disk(rawdev, &mmap_range) == 0) { send_buffer(sockfd, &rawdev->disk_mmap[mm_file_start], indx->f_size); if (munmap_disk(rawdev) != 0) { D0(fprintf(debug_file, "Unable to unmap memory region\n")); return -1; } } else { D0(fprintf(debug_file, "Unable to map disk to memory region:" "disk region start = 0x%llx, disk region end = 0x%llx\n", mmap_range.from, mmap_range.to)); return -1; } return 0; } /** * @brief Map a piece of raw device buffer to memory * @param[in,out] rawdev pointer to #rawdev_buffer structure containing Loading @@ -542,7 +662,7 @@ void send_buffer(int sockfd, unsigned char *buff, size_t sz) * mmaped region * @return 0 in case the region was mmaped successfully and -1 in case of an error */ int mmap_disk(rawdev_buffer *rawdev, const struct range *range) static int mmap_disk(rawdev_buffer *rawdev, const struct range *range) { int ret = 0; size_t mmap_size = range->to - range->from; Loading @@ -569,7 +689,7 @@ int mmap_disk(rawdev_buffer *rawdev, const struct range *range) * the current state of raw device buffer * @return 0 in case the region was unmapped successfully and -1 in case of an error */ int munmap_disk(rawdev_buffer *rawdev) static int munmap_disk(rawdev_buffer *rawdev) { if (rawdev->disk_mmap == NULL) return 0; Loading Loading @@ -681,6 +801,7 @@ void trim_command(char *cmd, ssize_t cmd_len) * Such a file is split into two parts, the one in the end of the buffer and the other * in the beginning, and should be sent in two steps * @param[in] rawdev pointer to #rawdev_buffer structure containing * the current state of raw device buffer * @param[in] indx pointer to the disk index node * @param[in] sockfd opened socket descriptor */ Loading Loading @@ -743,6 +864,154 @@ int get_indx_args(char *cmd, struct disk_index *indx) return sscanf(++cmd_start, INDEX_FORMAT_STR, &indx->port, &indx->rawtime, &indx->usec, &indx->f_offset, &indx->f_size); } int get_timestamp_args(char *cmd, struct disk_index *indx) { int ret; struct tm tm; char *cmd_start = strchr(cmd, ':'); if (cmd_start == NULL) return -1; ret = sscanf(++cmd_start, EXIF_TIMESTAMP_FORMAT, &tm.tm_year, &tm.tm_mon, &tm.tm_mday, &tm.tm_hour, &tm.tm_min, &tm.tm_sec); tm.tm_year -= 1900; tm.tm_mon -= 1; indx->rawtime = mktime(&tm); return ret; } #define SEARCH_SIZE_WINDOW (2 * 1048576) int get_search_window(const struct range *r, struct range *s) { if ((r->to - r->from) < SEARCH_SIZE_WINDOW) return -1; uint64_t middle = (r->to + r->from) / 2; s->from = (middle - SEARCH_SIZE_WINDOW / 2) & PAGE_BOUNDARY_MASK; s->to = middle + SEARCH_SIZE_WINDOW / 2; return 0; } int find_in_window(rawdev_buffer *rawdev, const struct range *wnd, struct disk_index *indx) { int ret = -1; int pos_start; if (mmap_disk(rawdev, (const struct range *)wnd) == 0) { fprintf(debug_file, "Searching in mmapped window: from 0x%llx, to 0x%llx\n", wnd->from, wnd->from + rawdev->mmap_current_size); pos_start = find_marker(rawdev->disk_mmap, rawdev->mmap_current_size, elphel_st.iov_base, elphel_st.iov_len, 0); fprintf(debug_file, "pos_start = %d\n", pos_start); if (pos_start >= 0) { rawdev->file_start = rawdev->mmap_offset + pos_start; read_index(rawdev, indx); ret = 0; } munmap_disk(rawdev); } else { fprintf(debug_file, "Error mmaping region\n"); } return ret; } /** * @brief Find a file on disk having time stamp close to the time stamp given. * @param[in] rawdev pointer to #rawdev_buffer structure containing * the current state of raw device buffer * @param[in] idir pointer to sparse disk index directory where indexes are sorted * in time order * @param[in,out] indx pointer to #disk_index structure with time stamp fields filled in. * This structure will contain the disk index found (if any) and thus the time stamp * provided through this structure will be overwritten. * @return 0 if an index was found and -1 otherwise. The @b indx will contain the * information about the index found on disk. */ // Time window used for disk index search. Index within this window is considered a candidate #define SEARCH_TIME_WINDOW 600 int find_disk_offset(rawdev_buffer *rawdev, struct disk_idir *idir, struct disk_index *indx) { int ret = 0; bool indx_appended; bool process = true; struct range range; struct range search_window; struct disk_index *indx_found; struct disk_index *nearest_indx = find_nearest_by_time((const struct disk_idir *)idir, indx->rawtime); struct tm *tm = gmtime(&indx->rawtime); fprintf(debug_file, "%s: looking for offset near %04d-%02d-%02d %02d:%02d:%02d\n", __func__, tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec); uint64_t nr; nr = (nearest_indx == NULL) ? 0 : nearest_indx->f_offset; fprintf(debug_file, "Nearest index offset: 0x%llx\n", nr); // define disk offsets where search will be performed if (nearest_indx == NULL) { range.from = rawdev->start_pos; range.to = rawdev->end_pos; } else { if (indx->rawtime > nearest_indx->rawtime) { range.from = nearest_indx->f_offset; if (nearest_indx->next != NULL) range.to = nearest_indx->next->f_offset; else range.to = rawdev->end_pos; } else { range.to = nearest_indx->f_offset; if (nearest_indx->prev != NULL) range.from = nearest_indx->prev->f_offset; else range.from = rawdev->start_pos; } } fprintf(debug_file, "Starting search in range: from 0x%llx, to 0x%llx\n", range.from, range.to); while (process && get_search_window(&range, &search_window) == 0) { indx_found = NULL; if (create_node(&indx_found) == -1) break; indx_appended = false; fprintf(debug_file, "Search window: from 0x%llx, to 0x%llx\n", search_window.from, search_window.to); if (find_in_window(rawdev, &search_window, indx_found) == 0) { double time_diff = difftime(indx_found->rawtime, indx->rawtime); fprintf(debug_file, "Index found: time diff %f, file offset 0x%llx, rawtime found %d, rawtime given %d\n", time_diff, indx_found->f_offset, indx_found->rawtime, indx->rawtime); if (fabs(time_diff) > SEARCH_TIME_WINDOW) { // the index found is not within search time window, update sparse index directory and // define a new search window if (time_diff > 0) { range.to = search_window.from; } else { range.from = search_window.to; } } else { // the index found is within search time window, stop search and return process = false; *indx = *indx_found; } insert_node(idir, indx_found); indx_appended = true; } else { // index not found in the search window, move toward the start of the range range.to = search_window.from; } fprintf(debug_file, "Updating range, new range: from 0x%llx, to 0x%llx\n", range.from, range.to); } if (indx->f_offset == 0 && indx->f_size == 0) { // no files were found within specified time window ret = -1; } if (indx_appended == false) free(indx_found); fprintf(debug_file, "\nSparse index directory dump, %d nodes:\n", idir->size); dump_index_dir(idir); return ret; } /** * @brief Raw device buffer reading function. * Loading Loading @@ -771,17 +1040,17 @@ void *reader(void *arg) struct stat stat_buff; struct range mmap_range; struct disk_index *disk_indx, *cross_boundary_indx; struct disk_idir index_dir = { .head = NULL, .tail = NULL, .size = 0 }; struct disk_idir index_dir; struct disk_idir index_sparse; struct exit_state exit_state = { .state = state, .idir = &index_dir, .sparse_idir = &index_sparse, .sockfd_const = &sockfd, .sockfd_temp = &fd }; memset(&index_dir, 0, sizeof(struct disk_index)); memset(&index_sparse, 0, sizeof(struct disk_index)); prep_socket(&sockfd, state->sock_port); pthread_cleanup_push(exit_thread, &exit_state); Loading Loading @@ -870,21 +1139,28 @@ void *reader(void *arg) struct disk_index indx; if (get_indx_args(cmd_ptr, &indx) > 0 && (disk_indx = find_by_offset(&index_dir, indx.f_offset)) != NULL){ mmap_range.from = indx.f_offset & PAGE_BOUNDARY_MASK; mmap_range.to = indx.f_offset + indx.f_size; mm_file_start = indx.f_offset - mmap_range.from; if (mmap_disk(rawdev, &mmap_range) == 0) { send_buffer(fd, &rawdev->disk_mmap[mm_file_start], indx.f_size); if (munmap_disk(rawdev) != 0) { D0(fprintf(debug_file, "Unable to unmap memory region\n")); send_file(rawdev, disk_indx, fd); } } else { D0(fprintf(debug_file, "Unable to map disk to memory region:" "disk region start = 0x%llx, disk region end = 0x%llx\n", mmap_range.from, mmap_range.to)); } break; case CMD_FIND_FILE: { struct disk_index indx; if (get_timestamp_args(cmd_ptr, &indx) > 0) { if (index_dir.size == 0) { find_disk_offset(rawdev, &index_sparse, &indx); } else { struct disk_index *indx_ptr; indx_ptr = find_nearest_by_time(&index_sparse, indx.rawtime); /* debug code follows */ if (indx_ptr != NULL) printf("Index found in pre-built index directory: offset = 0x%llx\n", indx_ptr->f_offset); else printf("Index NOT found in pre-built index directory. Probably it should be rebuilt\n"); /* end of debug code */ } } break; } case CMD_READ_ALL_FILES: // read files from raw device buffer and send them over socket; the disk index directory // should be built beforehand Loading Loading @@ -976,6 +1252,8 @@ static inline void exit_thread(void *arg) } if (s->idir->size != 0) delete_idir(s->idir); if (s->sparse_idir-> size != 0) delete_idir(s->sparse_idir); if (fstat(*s->sockfd_const, &buff) != EBADF) close(*s->sockfd_const); if (fstat(*s->sockfd_temp, &buff) != EBADF) Loading Loading @@ -1109,13 +1387,16 @@ void build_index(camogm_state *state, struct disk_idir *idir) // error condition (normally should not happen), drop current read buffer and do nothing buff_processed = 1; D6(fprintf(debug_file, "State 'abnormal stop marker, skip data'\n")); } else if (pos_start >= 0 && pos_stop >= 0 && pos_start > pos_stop && search_state == SEARCH_FILE_DATA) { } else if (pos_start >= 0 && pos_stop >= 0 && pos_start > pos_stop) { // normal condition, start marker following stop marker found - this indicates a new file if (search_state == SEARCH_FILE_DATA) { uint64_t disk_pos = dev_curr_pos + pos_stop + (save_from - active_buff); idir_result = stop_index(idir, disk_pos); } if (zero_cross == 0) { state->rawdev.file_start = dev_curr_pos + pos_start + (save_from - active_buff); idir_result = save_index(state, idir); search_state = SEARCH_FILE_DATA; save_from = save_from + pos_start + add_stm_len; // @todo: replace with pointer to current buffer save_to = buff + rd; Loading index_list.c +98 −0 Original line number Diff line number Diff line Loading @@ -17,6 +17,8 @@ #include <stdlib.h> #include <string.h> #include <time.h> #include <math.h> #include "index_list.h" Loading Loading @@ -61,6 +63,81 @@ int add_node(struct disk_idir *idir, struct disk_index *index) return idir->size; } int insert_node(struct disk_idir *idir, struct disk_index *indx) { int ret = 0; struct disk_index *node; if (idir->head == NULL) { add_node(idir, indx); return 1; } node = idir->head; while (node != NULL) { if (indx->rawtime < node->rawtime) { ret = insert_prev(idir, node, indx); break; } node = node->next; } if (node == NULL) ret = insert_next(idir, idir->tail, indx); return ret; } /** * @brief Insert new index to the list before the index specified. It means that new index will * be inserted toward the head. * @param[in,out] idir pointer to the lined list of indexes * @param[in] parent pointer to the element before which the new element will be inserted * @param[in] new_indx pointer to the element which will be inserted * @return The number of nodes in the list */ int insert_prev(struct disk_idir *idir, struct disk_index *parent, struct disk_index *new_indx) { if (parent->prev != NULL) { new_indx->next = parent; new_indx->prev = parent->prev; parent->prev->next = new_indx; parent->prev = new_indx; } else { new_indx->next = parent; new_indx->prev = NULL; parent->prev = new_indx; idir->head = new_indx; } idir->size++; return idir->size; } /** * @brief Insert new index to the list after the index specified. It means that new index will * be inserted toward the tail. * @param[in,out] idir pointer to the linked list of indexes * @param[in] parent pointer to the element after which the new element will be inserted * @param[in] new_indx pointer to the element which will be inserted * @return The number of nodes in the list */ int insert_next(struct disk_idir *idir, struct disk_index *parent, struct disk_index *new_indx) { if (parent->next != NULL) { new_indx->next = parent->next; new_indx->prev = parent; parent->next->prev = new_indx; parent->next = new_indx; } else { new_indx->next = NULL; new_indx->prev = parent; parent->next = new_indx; idir->tail = new_indx; } idir->size++; return idir->size; } /** * @brief Find index node by its start offset * @param[in] idir pointer to disk index directory Loading @@ -80,6 +157,27 @@ struct disk_index *find_by_offset(const struct disk_idir *idir, uint64_t offset) return index; } struct disk_index *find_nearest_by_time(const struct disk_idir *idir, time_t time) { struct disk_index *ptr = NULL; struct disk_index *index = idir->head; if (idir->size == 0) return NULL; if (index->next != NULL) ptr = index->next; else return index; while (index != NULL) { if (fabs(difftime(index->rawtime, time)) < fabs(difftime(ptr->rawtime, time))) ptr = index; index = index->next; } return ptr; } /** * @brief Remove a single index node from disk index directory * @param[in,out] idir pointer to disk index directory Loading index_list.h +4 −0 Original line number Diff line number Diff line Loading @@ -68,7 +68,11 @@ struct disk_idir { void dump_index_dir(const struct disk_idir *idir); int create_node(struct disk_index **index); int add_node(struct disk_idir *idir, struct disk_index *index); int insert_prev(struct disk_idir *idir, struct disk_index *parent, struct disk_index *new_indx); int insert_next(struct disk_idir *idir, struct disk_index *parent, struct disk_index *new_indx); int insert_node(struct disk_idir *idir, struct disk_index *indx); struct disk_index *find_by_offset(const struct disk_idir *idir, uint64_t offset); struct disk_index *find_nearest_by_time(const struct disk_idir *idir, time_t time); int remove_node(struct disk_idir *idir, struct disk_index *node); int delete_idir(struct disk_idir *idir); Loading Loading
camogm.c +2 −2 Original line number Diff line number Diff line Loading @@ -812,7 +812,7 @@ void camogm_set_prefix(camogm_state *state, const char * p, path_type type) D0(fprintf(debug_file, "WARNING: raw device write initiated\n")); state->rawdev_op = 1; /* debug code follows */ state->rawdev.end_pos = 134217728; state->rawdev.end_pos = 128 * 1048576; /* end of debug code */ } } Loading Loading @@ -1687,7 +1687,7 @@ int main(int argc, char *argv[]) "format and stores the result files.\n\n"; int ret; int opt; uint16_t port_num; uint16_t port_num = 0; char pipe_name_str[ELPHEL_PATH_MAX] = {0}; if ((argc < 5) || (argv[1][1] == '-')) { Loading
camogm_read.c +309 −28 Original line number Diff line number Diff line Loading @@ -53,6 +53,8 @@ #define TIFF_HDR_OFFSET 12 /** @brief The date and time format of Exif field */ #define EXIF_DATE_TIME_FORMAT "%Y:%m:%d %H:%M:%S" #define EXIF_TIMESTAMP_FORMAT "%d:%d:%d_%d:%d:%d" //#define EXIF_TIMESTAMP_FORMAT "%04d:%02d:%02d_%02d:%02d:%02d" /** @brief The format string used for file parameters reporting. Time and port number are extracted from Exif */ #define INDEX_FORMAT_STR "port_number=%d;unix_time=%ld;usec_time=%06u;offset=0x%010llx;file_size=%u\n" /** @brief The delimiters used to separate several commands in one command string sent over socket */ Loading Loading @@ -88,6 +90,7 @@ static const struct iovec elphel_en = { X(CMD_GET_INDEX, "get_index") \ X(CMD_READ_DISK, "read_disk") \ X(CMD_READ_FILE, "read_file") \ X(CMD_FIND_FILE, "find_file") \ X(CMD_READ_ALL_FILES, "read_all_files") \ X(CMD_STATUS, "status") Loading Loading @@ -225,12 +228,15 @@ struct crb_ptrs { struct exit_state { camogm_state *state; struct disk_idir *idir; struct disk_idir *sparse_idir; int *sockfd_const; int *sockfd_temp; }; static inline void exit_thread(void *arg); static void build_index(camogm_state *state, struct disk_idir *idir); static int mmap_disk(rawdev_buffer *rawdev, const struct range *range); static int munmap_disk(rawdev_buffer *rawdev); /** * @brief Debug function, prints the content of disk index directory Loading @@ -241,9 +247,11 @@ void dump_index_dir(const struct disk_idir *idir) { struct disk_index *ind = idir->head; printf("Head pointer = 0x%p, tail pointer = 0x%p\n", idir->head, idir->tail); while (ind != NULL) { fprintf(debug_file, INDEX_FORMAT_STR, ind->port, ind->rawtime, ind->usec, ind->f_offset, ind->f_size); fprintf(debug_file, "\tCurrent pointer: 0x%p, Prev pointer: 0x%p, next pointer: 0x%p\n", ind, ind->prev, ind->next); ind = ind->next; } } Loading Loading @@ -326,14 +334,14 @@ static void hdr_byte_order(struct tiff_hdr *hdr) * @param[out] buff buffer for the string to be read * @return The number of bytes placed to the read buffer */ static size_t exif_get_text(camogm_state *state, struct ifd_entry *tag, char *buff) static size_t exif_get_text(rawdev_buffer *rawdev, struct ifd_entry *tag, char *buff) { size_t bytes = 0; size_t str_len = tag->len * exif_data_fmt[tag->format]; uint64_t curr_pos = state->rawdev.file_start + TIFF_HDR_OFFSET + tag->offset; uint64_t curr_pos = rawdev->file_start + TIFF_HDR_OFFSET + tag->offset; lseek64(state->rawdev.rawdev_fd, curr_pos, SEEK_SET); bytes = read(state->rawdev.rawdev_fd, buff, str_len); lseek64(rawdev->rawdev_fd, curr_pos, SEEK_SET); bytes = read(rawdev->rawdev_fd, buff, str_len); return bytes; } Loading Loading @@ -418,12 +426,12 @@ static int save_index(camogm_state *state, struct disk_idir *idir) } if (ifd_date_time.len != 0) { struct tm tm; exif_get_text(state, &ifd_date_time, str_buff); exif_get_text(&state->rawdev, &ifd_date_time, str_buff); strptime(str_buff, EXIF_DATE_TIME_FORMAT, &tm); node->rawtime = mktime(&tm); } if (ifd_subsec.len != 0) { exif_get_text(state, &ifd_subsec, str_buff); exif_get_text(&state->rawdev, &ifd_subsec, str_buff); node->usec = strtoul(str_buff, NULL, 10); } if (node->rawtime != -1) Loading @@ -436,6 +444,96 @@ static int save_index(camogm_state *state, struct disk_idir *idir) return ret; } static int read_index(rawdev_buffer *rawdev, struct disk_index *indx) { int ret = 0; int process = 2; uint32_t data32; uint16_t num_entries = 0; uint64_t curr_pos; uint64_t subifd_offset = 0; struct tiff_hdr hdr; struct ifd_entry ifd; struct ifd_entry ifd_page_num = {0}; struct ifd_entry ifd_date_time = {0}; struct ifd_entry ifd_subsec = {0}; struct disk_index *node = NULL; unsigned char read_buff[TIFF_HDR_OFFSET] = {0}; char str_buff[SMALL_BUFF_LEN] = {0}; uint64_t save_pos = lseek64(rawdev->rawdev_fd, 0, SEEK_CUR); if (indx == NULL) return -1; if (create_node(&node) != 0) return -1; curr_pos = rawdev->file_start; lseek64(rawdev->rawdev_fd, curr_pos, SEEK_SET); if (read(rawdev->rawdev_fd, read_buff, sizeof(read_buff)) <= 0) { lseek64(rawdev->rawdev_fd, save_pos, SEEK_SET); return -1; } if (read_buff[2] == 0xff && read_buff[3] == 0xe1) { // get IFD0 offset from TIFF header read(rawdev->rawdev_fd, &hdr, sizeof(struct tiff_hdr)); hdr_byte_order(&hdr); curr_pos = rawdev->file_start + TIFF_HDR_OFFSET + hdr.offset; lseek64(rawdev->rawdev_fd, curr_pos, SEEK_SET); // process IFD0 and SubIFD fields do { read(rawdev->rawdev_fd, &num_entries, sizeof(num_entries)); num_entries = __be16_to_cpu(num_entries); for (int i = 0; i < num_entries; i++) { read(rawdev->rawdev_fd, &ifd, sizeof(struct ifd_entry)); ifd_byte_order(&ifd); switch (ifd.tag) { case Exif_Image_PageNumber: ifd_page_num = ifd; break; case Exif_Photo_DateTimeOriginal & 0xffff: ifd_date_time = ifd; break; case Exif_Image_ExifTag: subifd_offset = ifd.offset; break; case Exif_Photo_SubSecTimeOriginal & 0xffff: ifd_subsec = ifd; break; } } // ensure that IFD0 finished correctly (0x00000000 in the end), set file pointer to SubIFD and // process remaining fields read(rawdev->rawdev_fd, &data32, sizeof(data32)); process -= (subifd_offset == 0 || data32 != 0) ? 2 : 1; curr_pos = rawdev->file_start + TIFF_HDR_OFFSET + subifd_offset; lseek64(rawdev->rawdev_fd, curr_pos, SEEK_SET); } while (process > 0); // fill disk index node with Exif data and add it to disk index directory node->f_offset = rawdev->file_start; if (ifd_page_num.len != 0) { node->port = (uint32_t)ifd_page_num.offset; } if (ifd_date_time.len != 0) { struct tm tm; exif_get_text(rawdev, &ifd_date_time, str_buff); strptime(str_buff, EXIF_DATE_TIME_FORMAT, &tm); node->rawtime = mktime(&tm); } if (ifd_subsec.len != 0) { exif_get_text(rawdev, &ifd_subsec, str_buff); node->usec = strtoul(str_buff, NULL, 10); } if (node->rawtime != -1) *indx = *node; else ret = -1; } lseek64(rawdev->rawdev_fd, save_pos, SEEK_SET); return ret; } /** * @brief Calculate the size of current file and update the value in disk index directory * @param[in,out] idir pointer to disk index directory Loading Loading @@ -534,6 +632,28 @@ void send_buffer(int sockfd, unsigned char *buff, size_t sz) } } int send_file(rawdev_buffer *rawdev, struct disk_index *indx, int sockfd) { uint64_t mm_file_start; struct range mmap_range; mmap_range.from = indx->f_offset & PAGE_BOUNDARY_MASK; mmap_range.to = indx->f_offset + indx->f_size; mm_file_start = indx->f_offset - mmap_range.from; if (mmap_disk(rawdev, &mmap_range) == 0) { send_buffer(sockfd, &rawdev->disk_mmap[mm_file_start], indx->f_size); if (munmap_disk(rawdev) != 0) { D0(fprintf(debug_file, "Unable to unmap memory region\n")); return -1; } } else { D0(fprintf(debug_file, "Unable to map disk to memory region:" "disk region start = 0x%llx, disk region end = 0x%llx\n", mmap_range.from, mmap_range.to)); return -1; } return 0; } /** * @brief Map a piece of raw device buffer to memory * @param[in,out] rawdev pointer to #rawdev_buffer structure containing Loading @@ -542,7 +662,7 @@ void send_buffer(int sockfd, unsigned char *buff, size_t sz) * mmaped region * @return 0 in case the region was mmaped successfully and -1 in case of an error */ int mmap_disk(rawdev_buffer *rawdev, const struct range *range) static int mmap_disk(rawdev_buffer *rawdev, const struct range *range) { int ret = 0; size_t mmap_size = range->to - range->from; Loading @@ -569,7 +689,7 @@ int mmap_disk(rawdev_buffer *rawdev, const struct range *range) * the current state of raw device buffer * @return 0 in case the region was unmapped successfully and -1 in case of an error */ int munmap_disk(rawdev_buffer *rawdev) static int munmap_disk(rawdev_buffer *rawdev) { if (rawdev->disk_mmap == NULL) return 0; Loading Loading @@ -681,6 +801,7 @@ void trim_command(char *cmd, ssize_t cmd_len) * Such a file is split into two parts, the one in the end of the buffer and the other * in the beginning, and should be sent in two steps * @param[in] rawdev pointer to #rawdev_buffer structure containing * the current state of raw device buffer * @param[in] indx pointer to the disk index node * @param[in] sockfd opened socket descriptor */ Loading Loading @@ -743,6 +864,154 @@ int get_indx_args(char *cmd, struct disk_index *indx) return sscanf(++cmd_start, INDEX_FORMAT_STR, &indx->port, &indx->rawtime, &indx->usec, &indx->f_offset, &indx->f_size); } int get_timestamp_args(char *cmd, struct disk_index *indx) { int ret; struct tm tm; char *cmd_start = strchr(cmd, ':'); if (cmd_start == NULL) return -1; ret = sscanf(++cmd_start, EXIF_TIMESTAMP_FORMAT, &tm.tm_year, &tm.tm_mon, &tm.tm_mday, &tm.tm_hour, &tm.tm_min, &tm.tm_sec); tm.tm_year -= 1900; tm.tm_mon -= 1; indx->rawtime = mktime(&tm); return ret; } #define SEARCH_SIZE_WINDOW (2 * 1048576) int get_search_window(const struct range *r, struct range *s) { if ((r->to - r->from) < SEARCH_SIZE_WINDOW) return -1; uint64_t middle = (r->to + r->from) / 2; s->from = (middle - SEARCH_SIZE_WINDOW / 2) & PAGE_BOUNDARY_MASK; s->to = middle + SEARCH_SIZE_WINDOW / 2; return 0; } int find_in_window(rawdev_buffer *rawdev, const struct range *wnd, struct disk_index *indx) { int ret = -1; int pos_start; if (mmap_disk(rawdev, (const struct range *)wnd) == 0) { fprintf(debug_file, "Searching in mmapped window: from 0x%llx, to 0x%llx\n", wnd->from, wnd->from + rawdev->mmap_current_size); pos_start = find_marker(rawdev->disk_mmap, rawdev->mmap_current_size, elphel_st.iov_base, elphel_st.iov_len, 0); fprintf(debug_file, "pos_start = %d\n", pos_start); if (pos_start >= 0) { rawdev->file_start = rawdev->mmap_offset + pos_start; read_index(rawdev, indx); ret = 0; } munmap_disk(rawdev); } else { fprintf(debug_file, "Error mmaping region\n"); } return ret; } /** * @brief Find a file on disk having time stamp close to the time stamp given. * @param[in] rawdev pointer to #rawdev_buffer structure containing * the current state of raw device buffer * @param[in] idir pointer to sparse disk index directory where indexes are sorted * in time order * @param[in,out] indx pointer to #disk_index structure with time stamp fields filled in. * This structure will contain the disk index found (if any) and thus the time stamp * provided through this structure will be overwritten. * @return 0 if an index was found and -1 otherwise. The @b indx will contain the * information about the index found on disk. */ // Time window used for disk index search. Index within this window is considered a candidate #define SEARCH_TIME_WINDOW 600 int find_disk_offset(rawdev_buffer *rawdev, struct disk_idir *idir, struct disk_index *indx) { int ret = 0; bool indx_appended; bool process = true; struct range range; struct range search_window; struct disk_index *indx_found; struct disk_index *nearest_indx = find_nearest_by_time((const struct disk_idir *)idir, indx->rawtime); struct tm *tm = gmtime(&indx->rawtime); fprintf(debug_file, "%s: looking for offset near %04d-%02d-%02d %02d:%02d:%02d\n", __func__, tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec); uint64_t nr; nr = (nearest_indx == NULL) ? 0 : nearest_indx->f_offset; fprintf(debug_file, "Nearest index offset: 0x%llx\n", nr); // define disk offsets where search will be performed if (nearest_indx == NULL) { range.from = rawdev->start_pos; range.to = rawdev->end_pos; } else { if (indx->rawtime > nearest_indx->rawtime) { range.from = nearest_indx->f_offset; if (nearest_indx->next != NULL) range.to = nearest_indx->next->f_offset; else range.to = rawdev->end_pos; } else { range.to = nearest_indx->f_offset; if (nearest_indx->prev != NULL) range.from = nearest_indx->prev->f_offset; else range.from = rawdev->start_pos; } } fprintf(debug_file, "Starting search in range: from 0x%llx, to 0x%llx\n", range.from, range.to); while (process && get_search_window(&range, &search_window) == 0) { indx_found = NULL; if (create_node(&indx_found) == -1) break; indx_appended = false; fprintf(debug_file, "Search window: from 0x%llx, to 0x%llx\n", search_window.from, search_window.to); if (find_in_window(rawdev, &search_window, indx_found) == 0) { double time_diff = difftime(indx_found->rawtime, indx->rawtime); fprintf(debug_file, "Index found: time diff %f, file offset 0x%llx, rawtime found %d, rawtime given %d\n", time_diff, indx_found->f_offset, indx_found->rawtime, indx->rawtime); if (fabs(time_diff) > SEARCH_TIME_WINDOW) { // the index found is not within search time window, update sparse index directory and // define a new search window if (time_diff > 0) { range.to = search_window.from; } else { range.from = search_window.to; } } else { // the index found is within search time window, stop search and return process = false; *indx = *indx_found; } insert_node(idir, indx_found); indx_appended = true; } else { // index not found in the search window, move toward the start of the range range.to = search_window.from; } fprintf(debug_file, "Updating range, new range: from 0x%llx, to 0x%llx\n", range.from, range.to); } if (indx->f_offset == 0 && indx->f_size == 0) { // no files were found within specified time window ret = -1; } if (indx_appended == false) free(indx_found); fprintf(debug_file, "\nSparse index directory dump, %d nodes:\n", idir->size); dump_index_dir(idir); return ret; } /** * @brief Raw device buffer reading function. * Loading Loading @@ -771,17 +1040,17 @@ void *reader(void *arg) struct stat stat_buff; struct range mmap_range; struct disk_index *disk_indx, *cross_boundary_indx; struct disk_idir index_dir = { .head = NULL, .tail = NULL, .size = 0 }; struct disk_idir index_dir; struct disk_idir index_sparse; struct exit_state exit_state = { .state = state, .idir = &index_dir, .sparse_idir = &index_sparse, .sockfd_const = &sockfd, .sockfd_temp = &fd }; memset(&index_dir, 0, sizeof(struct disk_index)); memset(&index_sparse, 0, sizeof(struct disk_index)); prep_socket(&sockfd, state->sock_port); pthread_cleanup_push(exit_thread, &exit_state); Loading Loading @@ -870,21 +1139,28 @@ void *reader(void *arg) struct disk_index indx; if (get_indx_args(cmd_ptr, &indx) > 0 && (disk_indx = find_by_offset(&index_dir, indx.f_offset)) != NULL){ mmap_range.from = indx.f_offset & PAGE_BOUNDARY_MASK; mmap_range.to = indx.f_offset + indx.f_size; mm_file_start = indx.f_offset - mmap_range.from; if (mmap_disk(rawdev, &mmap_range) == 0) { send_buffer(fd, &rawdev->disk_mmap[mm_file_start], indx.f_size); if (munmap_disk(rawdev) != 0) { D0(fprintf(debug_file, "Unable to unmap memory region\n")); send_file(rawdev, disk_indx, fd); } } else { D0(fprintf(debug_file, "Unable to map disk to memory region:" "disk region start = 0x%llx, disk region end = 0x%llx\n", mmap_range.from, mmap_range.to)); } break; case CMD_FIND_FILE: { struct disk_index indx; if (get_timestamp_args(cmd_ptr, &indx) > 0) { if (index_dir.size == 0) { find_disk_offset(rawdev, &index_sparse, &indx); } else { struct disk_index *indx_ptr; indx_ptr = find_nearest_by_time(&index_sparse, indx.rawtime); /* debug code follows */ if (indx_ptr != NULL) printf("Index found in pre-built index directory: offset = 0x%llx\n", indx_ptr->f_offset); else printf("Index NOT found in pre-built index directory. Probably it should be rebuilt\n"); /* end of debug code */ } } break; } case CMD_READ_ALL_FILES: // read files from raw device buffer and send them over socket; the disk index directory // should be built beforehand Loading Loading @@ -976,6 +1252,8 @@ static inline void exit_thread(void *arg) } if (s->idir->size != 0) delete_idir(s->idir); if (s->sparse_idir-> size != 0) delete_idir(s->sparse_idir); if (fstat(*s->sockfd_const, &buff) != EBADF) close(*s->sockfd_const); if (fstat(*s->sockfd_temp, &buff) != EBADF) Loading Loading @@ -1109,13 +1387,16 @@ void build_index(camogm_state *state, struct disk_idir *idir) // error condition (normally should not happen), drop current read buffer and do nothing buff_processed = 1; D6(fprintf(debug_file, "State 'abnormal stop marker, skip data'\n")); } else if (pos_start >= 0 && pos_stop >= 0 && pos_start > pos_stop && search_state == SEARCH_FILE_DATA) { } else if (pos_start >= 0 && pos_stop >= 0 && pos_start > pos_stop) { // normal condition, start marker following stop marker found - this indicates a new file if (search_state == SEARCH_FILE_DATA) { uint64_t disk_pos = dev_curr_pos + pos_stop + (save_from - active_buff); idir_result = stop_index(idir, disk_pos); } if (zero_cross == 0) { state->rawdev.file_start = dev_curr_pos + pos_start + (save_from - active_buff); idir_result = save_index(state, idir); search_state = SEARCH_FILE_DATA; save_from = save_from + pos_start + add_stm_len; // @todo: replace with pointer to current buffer save_to = buff + rd; Loading
index_list.c +98 −0 Original line number Diff line number Diff line Loading @@ -17,6 +17,8 @@ #include <stdlib.h> #include <string.h> #include <time.h> #include <math.h> #include "index_list.h" Loading Loading @@ -61,6 +63,81 @@ int add_node(struct disk_idir *idir, struct disk_index *index) return idir->size; } int insert_node(struct disk_idir *idir, struct disk_index *indx) { int ret = 0; struct disk_index *node; if (idir->head == NULL) { add_node(idir, indx); return 1; } node = idir->head; while (node != NULL) { if (indx->rawtime < node->rawtime) { ret = insert_prev(idir, node, indx); break; } node = node->next; } if (node == NULL) ret = insert_next(idir, idir->tail, indx); return ret; } /** * @brief Insert new index to the list before the index specified. It means that new index will * be inserted toward the head. * @param[in,out] idir pointer to the lined list of indexes * @param[in] parent pointer to the element before which the new element will be inserted * @param[in] new_indx pointer to the element which will be inserted * @return The number of nodes in the list */ int insert_prev(struct disk_idir *idir, struct disk_index *parent, struct disk_index *new_indx) { if (parent->prev != NULL) { new_indx->next = parent; new_indx->prev = parent->prev; parent->prev->next = new_indx; parent->prev = new_indx; } else { new_indx->next = parent; new_indx->prev = NULL; parent->prev = new_indx; idir->head = new_indx; } idir->size++; return idir->size; } /** * @brief Insert new index to the list after the index specified. It means that new index will * be inserted toward the tail. * @param[in,out] idir pointer to the linked list of indexes * @param[in] parent pointer to the element after which the new element will be inserted * @param[in] new_indx pointer to the element which will be inserted * @return The number of nodes in the list */ int insert_next(struct disk_idir *idir, struct disk_index *parent, struct disk_index *new_indx) { if (parent->next != NULL) { new_indx->next = parent->next; new_indx->prev = parent; parent->next->prev = new_indx; parent->next = new_indx; } else { new_indx->next = NULL; new_indx->prev = parent; parent->next = new_indx; idir->tail = new_indx; } idir->size++; return idir->size; } /** * @brief Find index node by its start offset * @param[in] idir pointer to disk index directory Loading @@ -80,6 +157,27 @@ struct disk_index *find_by_offset(const struct disk_idir *idir, uint64_t offset) return index; } struct disk_index *find_nearest_by_time(const struct disk_idir *idir, time_t time) { struct disk_index *ptr = NULL; struct disk_index *index = idir->head; if (idir->size == 0) return NULL; if (index->next != NULL) ptr = index->next; else return index; while (index != NULL) { if (fabs(difftime(index->rawtime, time)) < fabs(difftime(ptr->rawtime, time))) ptr = index; index = index->next; } return ptr; } /** * @brief Remove a single index node from disk index directory * @param[in,out] idir pointer to disk index directory Loading
index_list.h +4 −0 Original line number Diff line number Diff line Loading @@ -68,7 +68,11 @@ struct disk_idir { void dump_index_dir(const struct disk_idir *idir); int create_node(struct disk_index **index); int add_node(struct disk_idir *idir, struct disk_index *index); int insert_prev(struct disk_idir *idir, struct disk_index *parent, struct disk_index *new_indx); int insert_next(struct disk_idir *idir, struct disk_index *parent, struct disk_index *new_indx); int insert_node(struct disk_idir *idir, struct disk_index *indx); struct disk_index *find_by_offset(const struct disk_idir *idir, uint64_t offset); struct disk_index *find_nearest_by_time(const struct disk_idir *idir, time_t time); int remove_node(struct disk_idir *idir, struct disk_index *node); int delete_idir(struct disk_idir *idir); Loading