Loading MatrixNowString.txt −98.7 KiB (290 KiB) File changed.No diff preview for this file type. View original file View changed file SameTableEncoder.ipynb +24 −50 Original line number Original line Diff line number Diff line %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python import numpy as np import numpy as np from matplotlib import pyplot as plt from matplotlib import pyplot as plt from itertools import product from itertools import product import os import os import sys import sys from PIL import Image from PIL import Image from scipy.optimize import minimize,linprog from scipy.optimize import minimize,linprog from sklearn.neighbors import KernelDensity from sklearn.neighbors import KernelDensity from collections import Counter from collections import Counter import numpy.linalg as la import numpy.linalg as la ``` ``` %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python def file_extractor(dirname="images"): def file_extractor(dirname="images"): files = os.listdir(dirname) files = os.listdir(dirname) scenes = [] scenes = [] for file in files: for file in files: if file == '.DS_Store': if file == '.DS_Store': continue continue else: else: scenes.append(os.path.join(dirname, file)) scenes.append(os.path.join(dirname, file)) return scenes return scenes def image_extractor(scenes): def image_extractor(scenes): image_folder = [] image_folder = [] for scene in scenes: for scene in scenes: files = os.listdir(scene) files = os.listdir(scene) for file in files: for file in files: if file[-5:] != ".tiff" or file[-7:] == "_6.tiff": if file[-5:] != ".tiff" or file[-7:] == "_6.tiff": continue continue else: else: image_folder.append(os.path.join(scene, file)) image_folder.append(os.path.join(scene, file)) return image_folder #returns a list of file paths to .tiff files in the specified directory given in file_extractor return image_folder #returns a list of file paths to .tiff files in the specified directory given in file_extractor def im_distribution(images, num): def im_distribution(images, num): """ """ Function that extracts tiff files from specific cameras and returns a list of all Function that extracts tiff files from specific cameras and returns a list of all the tiff files corresponding to that camera. i.e. all pictures labeled "_7.tiff" or otherwise the tiff files corresponding to that camera. i.e. all pictures labeled "_7.tiff" or otherwise specified camera numbers. specified camera numbers. Parameters: Parameters: images (list): list of all tiff files, regardless of classification. This is NOT a list of directories but images (list): list of all tiff files, regardless of classification. This is NOT a list of directories but of specific tiff files that can be opened right away. This is the list that we iterate through and of specific tiff files that can be opened right away. This is the list that we iterate through and divide. divide. num (str): a string designation for the camera number that we want to extract i.e. "14" for double digits num (str): a string designation for the camera number that we want to extract i.e. "14" for double digits of "_1" for single digits. of "_1" for single digits. Returns: Returns: tiff (list): A list of tiff files that have the specified designation from num. They are the files extracted tiff (list): A list of tiff files that have the specified designation from num. They are the files extracted from the 'images' list that correspond to the given num. from the 'images' list that correspond to the given num. """ """ tiff = [] tiff = [] for im in images: for im in images: if im[-7:-5] == num: if im[-7:-5] == num: tiff.append(im) tiff.append(im) return tiff return tiff ``` ``` %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python def predict_pix(tiff_image_path, difference = True): def predict_pix(tiff_image_path, difference = True): """ """ This function predict the pixel values excluding the boundary. This function predict the pixel values excluding the boundary. Using the 4 neighbor pixel values and MSE to predict the next pixel value Using the 4 neighbor pixel values and MSE to predict the next pixel value (-1,1) (0,1) (1,1) => relative position of the 4 other given values (-1,1) (0,1) (1,1) => relative position of the 4 other given values (-1,0) (0,0) => (0,0) is the one we want to predict (-1,0) (0,0) => (0,0) is the one we want to predict take the derivative of mean square error to solve for the system of equation take the derivative of mean square error to solve for the system of equation A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) A @ [a, b, c] = [-z0+z2-z3, z0+z1+z2, -z0-z1-z2-z3] where z0 = (-1,1), z1 = (0,1), z2 = (1,1), z3 = (-1,0) A @ [a, b, c] = [-z0+z2-z3, z0+z1+z2, -z0-z1-z2-z3] where z0 = (-1,1), z1 = (0,1), z2 = (1,1), z3 = (-1,0) and the predicted pixel value is c. and the predicted pixel value is c. Input: Input: tiff_image_path (string): path to the tiff file tiff_image_path (string): path to the tiff file Return: Return: image ndarray(512 X 640): original image image ndarray(512 X 640): original image predict ndarray(325380,): predicted image excluding the boundary predict ndarray(325380,): predicted image excluding the boundary diff. ndarray(325380,): IF difference = TRUE, difference between the min and max of four neighbors exclude the boundary diff. ndarray(325380,): IF difference = TRUE, difference between the min and max of four neighbors exclude the boundary ELSE: the residuals of the four nearest pixels to a fitted hyperplane ELSE: the residuals of the four nearest pixels to a fitted hyperplane error ndarray(325380,): difference between the original image and predicted image error ndarray(325380,): difference between the original image and predicted image A ndarray(3 X 3): system of equation A ndarray(3 X 3): system of equation """ """ image_obj = Image.open(tiff_image_path) #Open the image and read it as an Image object image_obj = Image.open(tiff_image_path) #Open the image and read it as an Image object image_array = np.array(image_obj)[1:,:].astype(int) #Convert to an array, leaving out the first row because the first row is just housekeeping data image_array = np.array(image_obj)[1:,:].astype(int) #Convert to an array, leaving out the first row because the first row is just housekeeping data # image_array = image_array.astype(int) # image_array = image_array.astype(int) A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation # where z0 = (-1,1), z1 = (0,1), z2 = (1,1), z3 = (-1,0) # where z0 = (-1,1), z1 = (0,1), z2 = (1,1), z3 = (-1,0) z0 = image_array[0:-2,0:-2] # get all the first pixel for the entire image z0 = image_array[0:-2,0:-2] # get all the first pixel for the entire image z1 = image_array[0:-2,1:-1] # get all the second pixel for the entire image z1 = image_array[0:-2,1:-1] # get all the second pixel for the entire image z2 = image_array[0:-2,2::] # get all the third pixel for the entire image z2 = image_array[0:-2,2::] # get all the third pixel for the entire image z3 = image_array[1:-1,0:-2] # get all the forth pixel for the entire image z3 = image_array[1:-1,0:-2] # get all the forth pixel for the entire image # calculate the out put of the system of equation # calculate the out put of the system of equation y0 = np.ravel(-z0+z2-z3) y0 = np.ravel(-z0+z2-z3) y1 = np.ravel(z0+z1+z2) y1 = np.ravel(z0+z1+z2) y2 = np.ravel(-z0-z1-z2-z3) y2 = np.ravel(-z0-z1-z2-z3) y = np.vstack((y0,y1,y2)) y = np.vstack((y0,y1,y2)) # use numpy solver to solve the system of equations all at once # use numpy solver to solve the system of equations all at once #predict = np.floor(np.linalg.solve(A,y)[-1]) #predict = np.floor(np.linalg.solve(A,y)[-1]) predict = np.round(np.round((np.linalg.solve(A,y)[-1]),1)) predict = np.round(np.round((np.linalg.solve(A,y)[-1]),1)) #Matrix system of points that will be used to solve the least squares fitting hyperplane #Matrix system of points that will be used to solve the least squares fitting hyperplane points = np.array([[-1,-1,1], [-1,0,1], [-1,1,1], [0,-1,1]]) points = np.array([[-1,-1,1], [-1,0,1], [-1,1,1], [0,-1,1]]) # flatten the neighbor pixlels and stack them together # flatten the neighbor pixlels and stack them together z0 = np.ravel(z0) z0 = np.ravel(z0) z1 = np.ravel(z1) z1 = np.ravel(z1) z2 = np.ravel(z2) z2 = np.ravel(z2) z3 = np.ravel(z3) z3 = np.ravel(z3) neighbor = np.vstack((z0,z1,z2,z3)).T neighbor = np.vstack((z0,z1,z2,z3)).T if difference: if difference: # calculate the difference # calculate the difference diff = np.max(neighbor,axis = 1) - np.min(neighbor, axis=1) diff = np.max(neighbor,axis = 1) - np.min(neighbor, axis=1) else: else: #Compute the best fitting hyperplane using least squares #Compute the best fitting hyperplane using least squares #The res is the residuals of the four points used to fit the hyperplane (summed distance of each of the #The res is the residuals of the four points used to fit the hyperplane (summed distance of each of the #points to the hyperplane), it is a measure of gradient #points to the hyperplane), it is a measure of gradient f, diff, rank, s = la.lstsq(points, neighbor.T, rcond=None) f, diff, rank, s = la.lstsq(points, neighbor.T, rcond=None) diff = diff.astype(int) diff = diff.astype(int) # calculate the error # calculate the error error = np.ravel(image_array[1:-1,1:-1])-predict error = np.ravel(image_array[1:-1,1:-1])-predict return image_array, predict, diff, error return image_array, predict, diff, error ``` ``` %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python """ """ this huffman encoding code is found online this huffman encoding code is found online https://favtutor.com/blogs/huffman-coding https://favtutor.com/blogs/huffman-coding """ """ class NodeTree(object): class NodeTree(object): def __init__(self, left=None, right=None): def __init__(self, left=None, right=None): self.left = left self.left = left self.right = right self.right = right def children(self): def children(self): return self.left, self.right return self.left, self.right def __str__(self): def __str__(self): return self.left, self.right return self.left, self.right def huffman_code_tree(node, binString=''): def huffman_code_tree(node, binString=''): ''' ''' Function to find Huffman Code Function to find Huffman Code ''' ''' if type(node) is str: if type(node) is str: return {node: binString} return {node: binString} (l, r) = node.children() (l, r) = node.children() d = dict() d = dict() d.update(huffman_code_tree(l, binString + '0')) d.update(huffman_code_tree(l, binString + '0')) d.update(huffman_code_tree(r, binString + '1')) d.update(huffman_code_tree(r, binString + '1')) return d return d def make_tree(nodes): def make_tree(nodes): ''' ''' Function to make tree Function to make tree :param nodes: Nodes :param nodes: Nodes :return: Root of the tree :return: Root of the tree ''' ''' while len(nodes) > 1: while len(nodes) > 1: (key1, c1) = nodes[-1] (key1, c1) = nodes[-1] (key2, c2) = nodes[-2] (key2, c2) = nodes[-2] nodes = nodes[:-2] nodes = nodes[:-2] node = NodeTree(key1, key2) node = NodeTree(key1, key2) nodes.append((node, c1 + c2)) nodes.append((node, c1 + c2)) #reverse True, decending order #reverse True, decending order #There is a huge memory leak here, no idea how or why #There is a huge memory leak here, no idea how or why nodes = sorted(nodes, key=lambda x: x[1], reverse=True) nodes = sorted(nodes, key=lambda x: x[1], reverse=True) return nodes[0][0] return nodes[0][0] def decode_string(huffman_string, the_keys, the_values): def decode_string(huffman_string, the_keys, the_values): for i in range(len(huffman_string)): for i in range(len(huffman_string)): try: try: return (int(the_keys[the_values.index(huffman_string[:i+1])]),huffman_string[i+1:]) return (int(the_keys[the_values.index(huffman_string[:i+1])]),huffman_string[i+1:]) except: except: pass pass ``` ``` %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python def make_dictionary(tiff_image_path_list, num_bins=4, difference = True): def make_dictionary(tiff_image_path_list, num_bins=4, difference = True): """ """ This function is used to encode the error based on the difference This function is used to encode the error based on the difference and split the difference into different bins and split the difference into different bins Input: Input: tiff_image_path (string): path to the tiff file tiff_image_path (string): path to the tiff file num_bins (int): number of bins num_bins (int): number of bins Return: Return: huffman_encoding_list list (num_bins + 1): a list of dictionary huffman_encoding_list list (num_bins + 1): a list of dictionary image_array ndarray (512, 640): original image image_array ndarray (512, 640): original image new_error ndarray (512, 640): error that includes the boundary new_error ndarray (512, 640): error that includes the boundary diff ndarray (510, 638): difference of min and max of the 4 neighbors diff ndarray (510, 638): difference of min and max of the 4 neighbors boundary ndarray (2300,): the boundary values after subtracting the very first pixel value boundary ndarray (2300,): the boundary values after subtracting the very first pixel value predict ndarray (325380,): the list of predicted values predict ndarray (325380,): the list of predicted values bins list (num_bins - 1,): a list of threshold to cut the bins bins list (num_bins - 1,): a list of threshold to cut the bins A ndarray (3 X 3): system of equation A ndarray (3 X 3): system of equation """ """ list_of_all_vals = [] list_of_all_vals = [] huffman_encoding_list = [] huffman_encoding_list = [] for i in range(num_bins+1): for i in range(num_bins+1): list_of_all_vals.append([]) list_of_all_vals.append([]) for i, tiff_image_path in enumerate(tiff_image_path_list): for i, tiff_image_path in enumerate(tiff_image_path_list): # get the image_array, etc # get the image_array, etc image_array, predict, diff, error= predict_pix(tiff_image_path, difference) image_array, predict, diff, error= predict_pix(tiff_image_path, difference) # calculate the number of points that will go in each bin # calculate the number of points that will go in each bin data_points_per_bin = diff.size // num_bins data_points_per_bin = diff.size // num_bins # sort the difference and create the bins # sort the difference and create the bins sorted_diff = np.sort(diff.copy()) sorted_diff = np.sort(diff.copy()) bins = [12,60,180] bins = [12,60,180] # get the boundary # get the boundary boundary = np.hstack((image_array[0,:],image_array[-1,:],image_array[1:-1,0],image_array[1:-1,-1])) boundary = np.hstack((image_array[0,:],image_array[-1,:],image_array[1:-1,0],image_array[1:-1,-1])) # take the difference of the boundary with the very first pixel # take the difference of the boundary with the very first pixel boundary = boundary - image_array[0,0] boundary = boundary - image_array[0,0] #boundary is 1dim, so boundary[0] is just the first element #boundary is 1dim, so boundary[0] is just the first element boundary[0] = image_array[0,0] boundary[0] = image_array[0,0] # huffman encode the boundary # huffman encode the boundary for j in boundary: for j in boundary: list_of_all_vals[0].append(str(j)) list_of_all_vals[0].append(str(j)) # create a list of huffman table # create a list of huffman table n = len(bins) n = len(bins) # loop through different bins # loop through different bins for k in range (0,n): for k in range (0,n): # the first bin # the first bin if k == 0 : if k == 0 : # get the point within the bin and huffman huffman_encoding_dict # get the point within the bin and huffman huffman_encoding_dict mask = diff <= bins[k] mask = diff <= bins[k] for j in error[mask].astype(int): for j in error[mask].astype(int): list_of_all_vals[k+1].append(str(j)) list_of_all_vals[k+1].append(str(j)) # the middle bins # the middle bins else: else: # get the point within the bin and huffman huffman_encoding_dict # get the point within the bin and huffman huffman_encoding_dict mask = diff > bins[k-1] mask = diff > bins[k-1] new_error = error[mask] new_error = error[mask] mask2 = diff[mask] <= bins[k] mask2 = diff[mask] <= bins[k] for j in new_error[mask2].astype(int): for j in new_error[mask2].astype(int): list_of_all_vals[k+1].append(str(j)) list_of_all_vals[k+1].append(str(j)) # the last bin # the last bin # get the point within the bin and huffman huffman_encoding_dict # get the point within the bin and huffman huffman_encoding_dict mask = diff > bins[-1] mask = diff > bins[-1] for j in error[mask].astype(int): for j in error[mask].astype(int): list_of_all_vals[-1].append(str(j)) list_of_all_vals[-1].append(str(j)) for item in list_of_all_vals: for item in list_of_all_vals: freq = dict(Counter(item)) freq = dict(Counter(item)) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) node = make_tree(freq) node = make_tree(freq) huffman_encoding_list.append(huffman_code_tree(node)) huffman_encoding_list.append(huffman_code_tree(node)) # create a error matrix that includes the boundary (used in encoding matrix) # create a error matrix that includes the boundary (used in encoding matrix) new_error = np.copy(image_array) new_error = np.copy(image_array) new_error[1:-1,1:-1] = np.reshape(error,(510, 638)) new_error[1:-1,1:-1] = np.reshape(error,(510, 638)) keep = new_error[0,0] keep = new_error[0,0] new_error[0,:] = new_error[0,:] - keep new_error[0,:] = new_error[0,:] - keep new_error[-1,:] = new_error[-1,:] - keep new_error[-1,:] = new_error[-1,:] - keep new_error[1:-1,0] = new_error[1:-1,0] - keep new_error[1:-1,0] = new_error[1:-1,0] - keep new_error[1:-1,-1] = new_error[1:-1,-1] - keep new_error[1:-1,-1] = new_error[1:-1,-1] - keep new_error[0,0] = keep new_error[0,0] = keep # huffman_encoding_list = list(set(huffman_encoding_list)) # huffman_encoding_list = list(set(huffman_encoding_list)) diff = np.reshape(diff,(510,638)) diff = np.reshape(diff,(510,638)) # return the huffman dictionary # return the huffman dictionary return huffman_encoding_list,bins return huffman_encoding_list,bins ``` ``` %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python def huffman(tiff_image_path, num_bins=4, difference = True): def huffman(tiff_image_path, num_bins=4, difference = True): """ """ This function is used to encode the error based on the difference This function is used to encode the error based on the difference and split the difference into different bins and split the difference into different bins Input: Input: tiff_image_path (string): path to the tiff file tiff_image_path (string): path to the tiff file num_bins (int): number of bins num_bins (int): number of bins Return: Return: huffman_encoding_list list (num_bins + 1): a list of dictionary huffman_encoding_list list (num_bins + 1): a list of dictionary image_array ndarray (512, 640): original image image_array ndarray (512, 640): original image new_error ndarray (512, 640): error that includes the boundary new_error ndarray (512, 640): error that includes the boundary diff ndarray (510, 638): difference of min and max of the 4 neighbors diff ndarray (510, 638): difference of min and max of the 4 neighbors boundary ndarray (2300,): the boundary values after subtracting the very first pixel value boundary ndarray (2300,): the boundary values after subtracting the very first pixel value predict ndarray (325380,): the list of predicted values predict ndarray (325380,): the list of predicted values bins list (num_bins - 1,): a list of threshold to cut the bins bins list (num_bins - 1,): a list of threshold to cut the bins A ndarray (3 X 3): system of equation A ndarray (3 X 3): system of equation """ """ # get the image_array, etc # get the image_array, etc image_array, predict, diff, error= predict_pix(tiff_image_path, difference) image_array, predict, diff, error= predict_pix(tiff_image_path, difference) # calculate the number of points that will go in each bin # calculate the number of points that will go in each bin data_points_per_bin = diff.size // num_bins data_points_per_bin = diff.size // num_bins # sort the difference and create the bins # sort the difference and create the bins sorted_diff = np.sort(diff.copy()) sorted_diff = np.sort(diff.copy()) # bins = [sorted_diff[i*data_points_per_bin] for i in range(1,num_bins)] # bins = [sorted_diff[i*data_points_per_bin] for i in range(1,num_bins)] bins = [12,60,180] bins = [12,60,180] # get the boundary # get the boundary boundary = np.hstack((image_array[0,:],image_array[-1,:],image_array[1:-1,0],image_array[1:-1,-1])) boundary = np.hstack((image_array[0,:],image_array[-1,:],image_array[1:-1,0],image_array[1:-1,-1])) # take the difference of the boundary with the very first pixel # take the difference of the boundary with the very first pixel boundary = boundary - image_array[0,0] boundary = boundary - image_array[0,0] #boundary is 1dim, so boundary[0] is just the first element #boundary is 1dim, so boundary[0] is just the first element boundary[0] = image_array[0,0] boundary[0] = image_array[0,0] # huffman encode the boundary # huffman encode the boundary bound_vals_as_string = [str(i) for i in boundary] bound_vals_as_string = [str(i) for i in boundary] freq = dict(Counter(bound_vals_as_string)) freq = dict(Counter(bound_vals_as_string)) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) node = make_tree(freq) node = make_tree(freq) huffman_encoding_dict = huffman_code_tree(node) huffman_encoding_dict = huffman_code_tree(node) # create a list of huffman table # create a list of huffman table huffman_encoding_list = [huffman_encoding_dict] huffman_encoding_list = [huffman_encoding_dict] n = len(bins) n = len(bins) # loop through different bins # loop through different bins for i in range (0,n): for i in range (0,n): # the first bin # the first bin if i == 0 : if i == 0 : # get the point within the bin and huffman huffman_encoding_dict # get the point within the bin and huffman huffman_encoding_dict mask = diff <= bins[i] mask = diff <= bins[i] line_as_string = [str(i) for i in error[mask].astype(int)] line_as_string = [str(i) for i in error[mask].astype(int)] freq = dict(Counter(line_as_string)) freq = dict(Counter(line_as_string)) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) node = make_tree(freq) node = make_tree(freq) huffman_encoding_dict = huffman_code_tree(node) huffman_encoding_dict = huffman_code_tree(node) huffman_encoding_list.append(huffman_encoding_dict) huffman_encoding_list.append(huffman_encoding_dict) # the middle bins # the middle bins else: else: # get the point within the bin and huffman huffman_encoding_dict # get the point within the bin and huffman huffman_encoding_dict mask = diff > bins[i-1] mask = diff > bins[i-1] new_error = error[mask] new_error = error[mask] mask2 = diff[mask] <= bins[i] mask2 = diff[mask] <= bins[i] line_as_string = [str(i) for i in new_error[mask2].astype(int)] line_as_string = [str(i) for i in new_error[mask2].astype(int)] freq = dict(Counter(line_as_string)) freq = dict(Counter(line_as_string)) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) node = make_tree(freq) node = make_tree(freq) huffman_encoding_dict = huffman_code_tree(node) huffman_encoding_dict = huffman_code_tree(node) huffman_encoding_list.append(huffman_encoding_dict) huffman_encoding_list.append(huffman_encoding_dict) # the last bin # the last bin # get the point within the bin and huffman huffman_encoding_dict # get the point within the bin and huffman huffman_encoding_dict mask = diff > bins[-1] mask = diff > bins[-1] line_as_string = [str(i) for i in error[mask].astype(int)] line_as_string = [str(i) for i in error[mask].astype(int)] freq = dict(Counter(line_as_string)) freq = dict(Counter(line_as_string)) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) node = make_tree(freq) node = make_tree(freq) huffman_encoding_dict = huffman_code_tree(node) huffman_encoding_dict = huffman_code_tree(node) huffman_encoding_list.append(huffman_encoding_dict) huffman_encoding_list.append(huffman_encoding_dict) # create a error matrix that includes the boundary (used in encoding matrix) # create a error matrix that includes the boundary (used in encoding matrix) new_error = np.copy(image_array) new_error = np.copy(image_array) new_error[1:-1,1:-1] = np.reshape(error,(510, 638)) new_error[1:-1,1:-1] = np.reshape(error,(510, 638)) keep = new_error[0,0] keep = new_error[0,0] new_error[0,:] = new_error[0,:] - keep new_error[0,:] = new_error[0,:] - keep new_error[-1,:] = new_error[-1,:] - keep new_error[-1,:] = new_error[-1,:] - keep new_error[1:-1,0] = new_error[1:-1,0] - keep new_error[1:-1,0] = new_error[1:-1,0] - keep new_error[1:-1,-1] = new_error[1:-1,-1] - keep new_error[1:-1,-1] = new_error[1:-1,-1] - keep new_error[0,0] = keep new_error[0,0] = keep # huffman_encoding_list = list(set(huffman_encoding_list)) # huffman_encoding_list = list(set(huffman_encoding_list)) diff = np.reshape(diff,(510,638)) diff = np.reshape(diff,(510,638)) # return the huffman dictionary # return the huffman dictionary return huffman_encoding_list, image_array, new_error, diff, boundary, predict, bins return huffman_encoding_list, image_array, new_error, diff, boundary, predict, bins ``` ``` %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python def encoder(error, list_dic, diff, bound, bins): def encoder(error, list_dic, diff, bound, bins): """ """ This function encode the matrix with huffman coding tables This function encode the matrix with huffman coding tables Input: Input: error (512, 640): a matrix with all the errors error (512, 640): a matrix with all the errors list_dic (num_dic + 1,): a list of huffman coding table list_dic (num_dic + 1,): a list of huffman coding table bound (2300,): the boundary values after subtracting the very first pixel value bound (2300,): the boundary values after subtracting the very first pixel value bins (num_bins - 1,): a list of threshold to cut the bins bins (num_bins - 1,): a list of threshold to cut the bins Return: Return: encoded (512, 640): encoded matrix encoded (512, 640): encoded matrix """ """ returnable_encode = "" returnable_encode = "" # copy the error matrix (including the boundary) # copy the error matrix (including the boundary) encoded = np.copy(error).astype(int).astype(str).astype(object) encoded = np.copy(error).astype(int).astype(str).astype(object) #diff = np.reshape(diff,(510,638)) #diff = np.reshape(diff,(510,638)) # loop through all the pixel to encode # loop through all the pixel to encode for i in range(encoded.shape[0]): for i in range(encoded.shape[0]): for j in range(encoded.shape[1]): for j in range(encoded.shape[1]): if i == 0 or i == encoded.shape[0]-1 or j == 0 or j == encoded.shape[1]-1: if i == 0 or i == encoded.shape[0]-1 or j == 0 or j == encoded.shape[1]-1: returnable_encode += list_dic[0][encoded[i][j]] returnable_encode += list_dic[0][encoded[i][j]] elif diff[i-1][j-1] <= bins[0]: elif diff[i-1][j-1] <= bins[0]: returnable_encode += list_dic[1][encoded[i][j]] returnable_encode += list_dic[1][encoded[i][j]] elif diff[i-1][j-1] <= bins[1] and diff[i-1][j-1] > bins[0]: elif diff[i-1][j-1] <= bins[1] and diff[i-1][j-1] > bins[0]: returnable_encode +=list_dic[2][encoded[i][j]] returnable_encode +=list_dic[2][encoded[i][j]] elif diff[i-1][j-1] <= bins[2] and diff[i-1][j-1] > bins[1]: elif diff[i-1][j-1] <= bins[2] and diff[i-1][j-1] > bins[1]: returnable_encode +=list_dic[3][encoded[i][j]] returnable_encode +=list_dic[3][encoded[i][j]] else: else: returnable_encode += list_dic[4][encoded[i][j]] returnable_encode += list_dic[4][encoded[i][j]] return returnable_encode return returnable_encode ``` ``` %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python # def bitstring_to_bytes(s): # def bitstring_to_bytes(s): # v = int(s, 2) # v = int(s, 2) # b = bytearray() # b = bytearray() # while v: # while v: # b.append(v & 0xff) # b.append(v & 0xff) # v >>= 8 # v >>= 8 # return bytes(b[::-1]) # return bytes(b[::-1]) def bitstring_to_bytes(input_string): def bitstring_to_bytes(input_string): int_array = [] int_array = [] length_of_string = len(input_string) length_of_string = len(input_string) while length_of_string >= 8: while length_of_string >= 8: int_array.append(int(input_string[:8],2)) int_array.append(int(input_string[:8],2)) input_string = input_string[8:] input_string = input_string[8:] length_of_string = len(input_string) length_of_string = len(input_string) if length_of_string > 0: if length_of_string > 0: int_array.append(int(input_string,2)) zerobuffer = "" print(int_array[-12:]) for _ in range(8-length_of_string): zerobuffer += "0" int_array.append(int(input_string+zerobuffer,2)) # print(int_array[0:20]) # print(int_array[-12:]) return bytes(int_array) return bytes(int_array) ``` ``` %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python def decoder(encoded_string, list_dic, bins, use_diff): def decoder(encoded_string, list_dic, bins, use_diff): """ """ This function decodes the encoded_matrix. This function decodes the encoded_matrix. Input: Input: A (3 X 3): system of equation A (3 X 3): system of equation list_dic (num_dic + 1,): a list of huffman coding table list_dic (num_dic + 1,): a list of huffman coding table encoded_matrix (512, 640): encoded matrix encoded_matrix (512, 640): encoded matrix bins (num_bins - 1,): a list of threshold to cut the bins bins (num_bins - 1,): a list of threshold to cut the bins Return: Return: decode_matrix (512, 640): decoded matrix decode_matrix (512, 640): decoded matrix """ """ A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation # change the dictionary back to list # change the dictionary back to list # !!!!!WARNING!!!! has to change this part, everytime you change the number of bins # !!!!!WARNING!!!! has to change this part, everytime you change the number of bins the_keys0 = list(list_dic[0].keys()) the_keys0 = list(list_dic[0].keys()) the_values0 = list(list_dic[0].values()) the_values0 = list(list_dic[0].values()) the_keys1 = list(list_dic[1].keys()) the_keys1 = list(list_dic[1].keys()) the_values1 = list(list_dic[1].values()) the_values1 = list(list_dic[1].values()) the_keys2 = list(list_dic[2].keys()) the_keys2 = list(list_dic[2].keys()) the_values2 = list(list_dic[2].values()) the_values2 = list(list_dic[2].values()) the_keys3 = list(list_dic[3].keys()) the_keys3 = list(list_dic[3].keys()) the_values3 = list(list_dic[3].values()) the_values3 = list(list_dic[3].values()) the_keys4 = list(list_dic[4].keys()) the_keys4 = list(list_dic[4].keys()) the_values4 = list(list_dic[4].values()) the_values4 = list(list_dic[4].values()) #Matrix system of points that will be used to solve the least squares fitting hyperplane #Matrix system of points that will be used to solve the least squares fitting hyperplane points = np.array([[-1,-1,1], [-1,0,1], [-1,1,1], [0,-1,1]]) points = np.array([[-1,-1,1], [-1,0,1], [-1,1,1], [0,-1,1]]) decode_matrix = np.zeros((512,640)) decode_matrix = np.zeros((512,640)) # loop through all the element in the matrix # loop through all the element in the matrix for i in range(decode_matrix.shape[0]): for i in range(decode_matrix.shape[0]): for j in range(decode_matrix.shape[1]): for j in range(decode_matrix.shape[1]): # if it's the very first pixel on the image # if it's the very first pixel on the image if i == 0 and j == 0: if i == 0 and j == 0: colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys0, the_values=the_values0) colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys0, the_values=the_values0) decode_matrix[i][j] = colorvalue decode_matrix[i][j] = colorvalue # if it's on the boundary (any of the 4 edges) # if it's on the boundary (any of the 4 edges) elif i == 0 or i == decode_matrix.shape[0]-1 or j == 0 or j == decode_matrix.shape[1]-1: elif i == 0 or i == decode_matrix.shape[0]-1 or j == 0 or j == decode_matrix.shape[1]-1: colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys0, the_values=the_values0) colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys0, the_values=the_values0) decode_matrix[i][j] = colorvalue + decode_matrix[0][0] decode_matrix[i][j] = colorvalue + decode_matrix[0][0] # if not the boundary # if not the boundary else: else: # predict the image with the known pixel value # predict the image with the known pixel value z0 = decode_matrix[i-1][j-1] z0 = decode_matrix[i-1][j-1] z1 = decode_matrix[i-1][j] z1 = decode_matrix[i-1][j] z2 = decode_matrix[i-1][j+1] z2 = decode_matrix[i-1][j+1] z3 = decode_matrix[i][j-1] z3 = decode_matrix[i][j-1] y0 = int(-z0+z2-z3) y0 = int(-z0+z2-z3) y1 = int(z0+z1+z2) y1 = int(z0+z1+z2) y2 = int(-z0-z1-z2-z3) y2 = int(-z0-z1-z2-z3) y = np.vstack((y0,y1,y2)) y = np.vstack((y0,y1,y2)) if use_diff: if use_diff: difference = max(z0,z1,z2,z3) - min(z0,z1,z2,z3) difference = max(z0,z1,z2,z3) - min(z0,z1,z2,z3) else: else: f, difference, rank, s = la.lstsq(points, [z0,z1,z2,z3], rcond=None) f, difference, rank, s = la.lstsq(points, [z0,z1,z2,z3], rcond=None) difference = difference.astype(int) difference = difference.astype(int) predict = np.round(np.round(np.linalg.solve(A,y)[-1][0],1)) predict = np.round(np.round(np.linalg.solve(A,y)[-1][0],1)) # add on the difference by searching the dictionary # add on the difference by searching the dictionary # !!!!!WARNING!!!! has to change this part, eveytime you change the number of bins # !!!!!WARNING!!!! has to change this part, eveytime you change the number of bins if difference <= bins[0]: if difference <= bins[0]: colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys1, the_values=the_values1) colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys1, the_values=the_values1) decode_matrix[i][j] = colorvalue + int(predict) decode_matrix[i][j] = colorvalue + int(predict) elif difference <= bins[1] and difference > bins[0]: elif difference <= bins[1] and difference > bins[0]: colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys2, the_values=the_values2) colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys2, the_values=the_values2) decode_matrix[i][j] = colorvalue + int(predict) decode_matrix[i][j] = colorvalue + int(predict) elif difference <= bins[2] and difference > bins[1]: elif difference <= bins[2] and difference > bins[1]: colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys3, the_values=the_values3) colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys3, the_values=the_values3) decode_matrix[i][j] = colorvalue + int(predict) decode_matrix[i][j] = colorvalue + int(predict) else: else: colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys4, the_values=the_values4) colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys4, the_values=the_values4) decode_matrix[i][j] = colorvalue + int(predict) decode_matrix[i][j] = colorvalue + int(predict) return decode_matrix.astype(int) return decode_matrix.astype(int) ``` ``` %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python def read_from_file(filename): def read_from_file(filename): with open(filename, 'rb') as file: with open(filename, 'rb') as file: return file.read() return file.read() ``` ``` %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python scenes = file_extractor() scenes = file_extractor() newnamesforlater = [] images = image_extractor(scenes) images = image_extractor(scenes) oglist_dic, ogbins = make_dictionary(images[:1], 4, False) oglist_dic, ogbins = make_dictionary(images[:10], 4, False) file_size_ratios = [] file_size_ratios = [] np.save("first_dic.npy", oglist_dic) np.save("first_dic.npy", oglist_dic) for i in range(1): for i in range(10): list_dic, image, new_error, diff, bound, predict, bins = huffman(images[i], 4, False) list_dic, image, new_error, diff, bound, predict, bins = huffman(images[i], 4, False) encoded_string1 = encoder(new_error, oglist_dic, diff, bound, ogbins) encoded_string1 = encoder(new_error, oglist_dic, diff, bound, ogbins) # reconstruct_image = decoder(A, encoded_string, list_dic, bins, False) # reconstruct_image = decoder(A, encoded_string, list_dic, bins, False) # print(np.allclose(image, reconstruct_image)) # print(np.allclose(image, reconstruct_image)) print(len(encoded_string1)) inletters = bitstring_to_bytes(encoded_string1) inletters = bitstring_to_bytes(encoded_string1) if images[i][:-5] == ".tiff": # reconstruct_image = decoder(encoded_string, oglist_dic, ogbins, False) newname = images[i][:-5] # print(np.allclose(image, reconstruct_image)) # print(len(list_dic)) if images[0][:-5] == ".tiff": newname = images[0][:-5] else: else: newname = images[0][:-4] newname = images[i][:-4] newnamesforlater.append(newname + "_Compressed.txt") # with open(newname + "_Compressed.txt", 'wb') as f: with open(newname + "_Compressed.txt", 'wb') as f: with open("MatrixNowString.txt", 'wb') as f: f.write(inletters) f.write(inletters) file_size_ratios.append((os.path.getsize('MatrixNowString.txt'))/os.path.getsize('images/1626032610_393963/1626032610_393963_0.tiff')) file_size_ratios.append((os.path.getsize(newname + "_Compressed.txt"))/os.path.getsize('images/1626032610_393963/1626032610_393963_0.tiff')) ``` ``` %% Output 2080559 [189, 237, 174, 241, 149, 209, 142, 187, 71, 107, 178, 113] %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python print(np.mean(file_size_ratios)) print(np.mean(file_size_ratios)) ``` ``` %% Output %% Output 0.39572609126335023 0.3953878365239038 %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python 0.6058229115470704 ``` ``` %% Output 0.6058229115470704 %% Cell type:markdown id: tags: %% Cell type:markdown id: tags: Tomorrow note: Make new thing so that it rounds better, can be used losslessly while maintaining the same dict. Currently: wildly off. Tomorrow note: Make new thing so that it rounds better, can be used losslessly while maintaining the same dict. Currently: wildly off. %% Cell type:markdown id: tags: %% Cell type:markdown id: tags: info@elphel.com info@elphel.com 801.599.6216 801.599.6216 pdf4eclipse pdf4eclipse Don't use 6, the channel is broken Don't use 6, the channel is broken %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python print(int("100",2)) print(bin(4)) ``` ``` %% Output 4 0b100 %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python def bytes_to_bitstring(input_bytearray): def bytes_to_bitstring(input_bytearray): end_string = "" end_string = "" print(input_bytearray[0:25]) # print(bytearray(input_bytearray,encoding="ISO-8859-1").decode("utf-8")[0:4]) int_array = [i for i in input_bytearray] int_array = [i for i in input_bytearray] for item in int_array: for i, item in enumerate(int_array): end_string += (bin(item)[2:]) end_string += (bin(item)[2:].zfill(8)) return end_string return end_string # oglist_dic = np.load("first_dic.npy", allow_pickle="TRUE") oglist_dic = np.load("first_dic.npy", allow_pickle="TRUE") ogbins = [12,60,180] ogbins = [12,60,180] encoded_string2 = bytes_to_bitstring(read_from_file("images/1626033496_437803/1626033496_437803_11._Compressed.txt")) for i,item in enumerate(newnamesforlater[0:10]): list_dic, image, new_error, diff, bound, predict, bins = huffman(images[i], 4, False) print(len(encoded_string2)) encoded_string2 = bytes_to_bitstring(read_from_file(item)) reconstruct_image = decoder(encoded_string2, oglist_dic, ogbins, False) reconstruct_image = decoder(encoded_string2, oglist_dic, ogbins, False) # print(np.allclose(image, reconstruct_image)) print(np.allclose(image, reconstruct_image)) ``` ``` %% Output b'\xe4\x05%4E\x1a\xd3 h\xb1\xb5\x982\xc9\x0e\xbc\x83\x90\x101\xd23\xd9\x16>' 1850876 %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python ``` ``` images/1626033496_437803/1626033496_437803_1._Compressed.txt 0 → 100644 +253 KiB File added.No diff preview for this file type. View file images/1626033496_437803/1626033496_437803_11._Compressed.txt −130 KiB (254 KiB) File changed.No diff preview for this file type. View original file View changed file images/1626033496_437803/1626033496_437803_12._Compressed.txt 0 → 100644 +252 KiB File added.No diff preview for this file type. View file Loading
MatrixNowString.txt −98.7 KiB (290 KiB) File changed.No diff preview for this file type. View original file View changed file
SameTableEncoder.ipynb +24 −50 Original line number Original line Diff line number Diff line %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python import numpy as np import numpy as np from matplotlib import pyplot as plt from matplotlib import pyplot as plt from itertools import product from itertools import product import os import os import sys import sys from PIL import Image from PIL import Image from scipy.optimize import minimize,linprog from scipy.optimize import minimize,linprog from sklearn.neighbors import KernelDensity from sklearn.neighbors import KernelDensity from collections import Counter from collections import Counter import numpy.linalg as la import numpy.linalg as la ``` ``` %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python def file_extractor(dirname="images"): def file_extractor(dirname="images"): files = os.listdir(dirname) files = os.listdir(dirname) scenes = [] scenes = [] for file in files: for file in files: if file == '.DS_Store': if file == '.DS_Store': continue continue else: else: scenes.append(os.path.join(dirname, file)) scenes.append(os.path.join(dirname, file)) return scenes return scenes def image_extractor(scenes): def image_extractor(scenes): image_folder = [] image_folder = [] for scene in scenes: for scene in scenes: files = os.listdir(scene) files = os.listdir(scene) for file in files: for file in files: if file[-5:] != ".tiff" or file[-7:] == "_6.tiff": if file[-5:] != ".tiff" or file[-7:] == "_6.tiff": continue continue else: else: image_folder.append(os.path.join(scene, file)) image_folder.append(os.path.join(scene, file)) return image_folder #returns a list of file paths to .tiff files in the specified directory given in file_extractor return image_folder #returns a list of file paths to .tiff files in the specified directory given in file_extractor def im_distribution(images, num): def im_distribution(images, num): """ """ Function that extracts tiff files from specific cameras and returns a list of all Function that extracts tiff files from specific cameras and returns a list of all the tiff files corresponding to that camera. i.e. all pictures labeled "_7.tiff" or otherwise the tiff files corresponding to that camera. i.e. all pictures labeled "_7.tiff" or otherwise specified camera numbers. specified camera numbers. Parameters: Parameters: images (list): list of all tiff files, regardless of classification. This is NOT a list of directories but images (list): list of all tiff files, regardless of classification. This is NOT a list of directories but of specific tiff files that can be opened right away. This is the list that we iterate through and of specific tiff files that can be opened right away. This is the list that we iterate through and divide. divide. num (str): a string designation for the camera number that we want to extract i.e. "14" for double digits num (str): a string designation for the camera number that we want to extract i.e. "14" for double digits of "_1" for single digits. of "_1" for single digits. Returns: Returns: tiff (list): A list of tiff files that have the specified designation from num. They are the files extracted tiff (list): A list of tiff files that have the specified designation from num. They are the files extracted from the 'images' list that correspond to the given num. from the 'images' list that correspond to the given num. """ """ tiff = [] tiff = [] for im in images: for im in images: if im[-7:-5] == num: if im[-7:-5] == num: tiff.append(im) tiff.append(im) return tiff return tiff ``` ``` %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python def predict_pix(tiff_image_path, difference = True): def predict_pix(tiff_image_path, difference = True): """ """ This function predict the pixel values excluding the boundary. This function predict the pixel values excluding the boundary. Using the 4 neighbor pixel values and MSE to predict the next pixel value Using the 4 neighbor pixel values and MSE to predict the next pixel value (-1,1) (0,1) (1,1) => relative position of the 4 other given values (-1,1) (0,1) (1,1) => relative position of the 4 other given values (-1,0) (0,0) => (0,0) is the one we want to predict (-1,0) (0,0) => (0,0) is the one we want to predict take the derivative of mean square error to solve for the system of equation take the derivative of mean square error to solve for the system of equation A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) A @ [a, b, c] = [-z0+z2-z3, z0+z1+z2, -z0-z1-z2-z3] where z0 = (-1,1), z1 = (0,1), z2 = (1,1), z3 = (-1,0) A @ [a, b, c] = [-z0+z2-z3, z0+z1+z2, -z0-z1-z2-z3] where z0 = (-1,1), z1 = (0,1), z2 = (1,1), z3 = (-1,0) and the predicted pixel value is c. and the predicted pixel value is c. Input: Input: tiff_image_path (string): path to the tiff file tiff_image_path (string): path to the tiff file Return: Return: image ndarray(512 X 640): original image image ndarray(512 X 640): original image predict ndarray(325380,): predicted image excluding the boundary predict ndarray(325380,): predicted image excluding the boundary diff. ndarray(325380,): IF difference = TRUE, difference between the min and max of four neighbors exclude the boundary diff. ndarray(325380,): IF difference = TRUE, difference between the min and max of four neighbors exclude the boundary ELSE: the residuals of the four nearest pixels to a fitted hyperplane ELSE: the residuals of the four nearest pixels to a fitted hyperplane error ndarray(325380,): difference between the original image and predicted image error ndarray(325380,): difference between the original image and predicted image A ndarray(3 X 3): system of equation A ndarray(3 X 3): system of equation """ """ image_obj = Image.open(tiff_image_path) #Open the image and read it as an Image object image_obj = Image.open(tiff_image_path) #Open the image and read it as an Image object image_array = np.array(image_obj)[1:,:].astype(int) #Convert to an array, leaving out the first row because the first row is just housekeeping data image_array = np.array(image_obj)[1:,:].astype(int) #Convert to an array, leaving out the first row because the first row is just housekeeping data # image_array = image_array.astype(int) # image_array = image_array.astype(int) A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation # where z0 = (-1,1), z1 = (0,1), z2 = (1,1), z3 = (-1,0) # where z0 = (-1,1), z1 = (0,1), z2 = (1,1), z3 = (-1,0) z0 = image_array[0:-2,0:-2] # get all the first pixel for the entire image z0 = image_array[0:-2,0:-2] # get all the first pixel for the entire image z1 = image_array[0:-2,1:-1] # get all the second pixel for the entire image z1 = image_array[0:-2,1:-1] # get all the second pixel for the entire image z2 = image_array[0:-2,2::] # get all the third pixel for the entire image z2 = image_array[0:-2,2::] # get all the third pixel for the entire image z3 = image_array[1:-1,0:-2] # get all the forth pixel for the entire image z3 = image_array[1:-1,0:-2] # get all the forth pixel for the entire image # calculate the out put of the system of equation # calculate the out put of the system of equation y0 = np.ravel(-z0+z2-z3) y0 = np.ravel(-z0+z2-z3) y1 = np.ravel(z0+z1+z2) y1 = np.ravel(z0+z1+z2) y2 = np.ravel(-z0-z1-z2-z3) y2 = np.ravel(-z0-z1-z2-z3) y = np.vstack((y0,y1,y2)) y = np.vstack((y0,y1,y2)) # use numpy solver to solve the system of equations all at once # use numpy solver to solve the system of equations all at once #predict = np.floor(np.linalg.solve(A,y)[-1]) #predict = np.floor(np.linalg.solve(A,y)[-1]) predict = np.round(np.round((np.linalg.solve(A,y)[-1]),1)) predict = np.round(np.round((np.linalg.solve(A,y)[-1]),1)) #Matrix system of points that will be used to solve the least squares fitting hyperplane #Matrix system of points that will be used to solve the least squares fitting hyperplane points = np.array([[-1,-1,1], [-1,0,1], [-1,1,1], [0,-1,1]]) points = np.array([[-1,-1,1], [-1,0,1], [-1,1,1], [0,-1,1]]) # flatten the neighbor pixlels and stack them together # flatten the neighbor pixlels and stack them together z0 = np.ravel(z0) z0 = np.ravel(z0) z1 = np.ravel(z1) z1 = np.ravel(z1) z2 = np.ravel(z2) z2 = np.ravel(z2) z3 = np.ravel(z3) z3 = np.ravel(z3) neighbor = np.vstack((z0,z1,z2,z3)).T neighbor = np.vstack((z0,z1,z2,z3)).T if difference: if difference: # calculate the difference # calculate the difference diff = np.max(neighbor,axis = 1) - np.min(neighbor, axis=1) diff = np.max(neighbor,axis = 1) - np.min(neighbor, axis=1) else: else: #Compute the best fitting hyperplane using least squares #Compute the best fitting hyperplane using least squares #The res is the residuals of the four points used to fit the hyperplane (summed distance of each of the #The res is the residuals of the four points used to fit the hyperplane (summed distance of each of the #points to the hyperplane), it is a measure of gradient #points to the hyperplane), it is a measure of gradient f, diff, rank, s = la.lstsq(points, neighbor.T, rcond=None) f, diff, rank, s = la.lstsq(points, neighbor.T, rcond=None) diff = diff.astype(int) diff = diff.astype(int) # calculate the error # calculate the error error = np.ravel(image_array[1:-1,1:-1])-predict error = np.ravel(image_array[1:-1,1:-1])-predict return image_array, predict, diff, error return image_array, predict, diff, error ``` ``` %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python """ """ this huffman encoding code is found online this huffman encoding code is found online https://favtutor.com/blogs/huffman-coding https://favtutor.com/blogs/huffman-coding """ """ class NodeTree(object): class NodeTree(object): def __init__(self, left=None, right=None): def __init__(self, left=None, right=None): self.left = left self.left = left self.right = right self.right = right def children(self): def children(self): return self.left, self.right return self.left, self.right def __str__(self): def __str__(self): return self.left, self.right return self.left, self.right def huffman_code_tree(node, binString=''): def huffman_code_tree(node, binString=''): ''' ''' Function to find Huffman Code Function to find Huffman Code ''' ''' if type(node) is str: if type(node) is str: return {node: binString} return {node: binString} (l, r) = node.children() (l, r) = node.children() d = dict() d = dict() d.update(huffman_code_tree(l, binString + '0')) d.update(huffman_code_tree(l, binString + '0')) d.update(huffman_code_tree(r, binString + '1')) d.update(huffman_code_tree(r, binString + '1')) return d return d def make_tree(nodes): def make_tree(nodes): ''' ''' Function to make tree Function to make tree :param nodes: Nodes :param nodes: Nodes :return: Root of the tree :return: Root of the tree ''' ''' while len(nodes) > 1: while len(nodes) > 1: (key1, c1) = nodes[-1] (key1, c1) = nodes[-1] (key2, c2) = nodes[-2] (key2, c2) = nodes[-2] nodes = nodes[:-2] nodes = nodes[:-2] node = NodeTree(key1, key2) node = NodeTree(key1, key2) nodes.append((node, c1 + c2)) nodes.append((node, c1 + c2)) #reverse True, decending order #reverse True, decending order #There is a huge memory leak here, no idea how or why #There is a huge memory leak here, no idea how or why nodes = sorted(nodes, key=lambda x: x[1], reverse=True) nodes = sorted(nodes, key=lambda x: x[1], reverse=True) return nodes[0][0] return nodes[0][0] def decode_string(huffman_string, the_keys, the_values): def decode_string(huffman_string, the_keys, the_values): for i in range(len(huffman_string)): for i in range(len(huffman_string)): try: try: return (int(the_keys[the_values.index(huffman_string[:i+1])]),huffman_string[i+1:]) return (int(the_keys[the_values.index(huffman_string[:i+1])]),huffman_string[i+1:]) except: except: pass pass ``` ``` %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python def make_dictionary(tiff_image_path_list, num_bins=4, difference = True): def make_dictionary(tiff_image_path_list, num_bins=4, difference = True): """ """ This function is used to encode the error based on the difference This function is used to encode the error based on the difference and split the difference into different bins and split the difference into different bins Input: Input: tiff_image_path (string): path to the tiff file tiff_image_path (string): path to the tiff file num_bins (int): number of bins num_bins (int): number of bins Return: Return: huffman_encoding_list list (num_bins + 1): a list of dictionary huffman_encoding_list list (num_bins + 1): a list of dictionary image_array ndarray (512, 640): original image image_array ndarray (512, 640): original image new_error ndarray (512, 640): error that includes the boundary new_error ndarray (512, 640): error that includes the boundary diff ndarray (510, 638): difference of min and max of the 4 neighbors diff ndarray (510, 638): difference of min and max of the 4 neighbors boundary ndarray (2300,): the boundary values after subtracting the very first pixel value boundary ndarray (2300,): the boundary values after subtracting the very first pixel value predict ndarray (325380,): the list of predicted values predict ndarray (325380,): the list of predicted values bins list (num_bins - 1,): a list of threshold to cut the bins bins list (num_bins - 1,): a list of threshold to cut the bins A ndarray (3 X 3): system of equation A ndarray (3 X 3): system of equation """ """ list_of_all_vals = [] list_of_all_vals = [] huffman_encoding_list = [] huffman_encoding_list = [] for i in range(num_bins+1): for i in range(num_bins+1): list_of_all_vals.append([]) list_of_all_vals.append([]) for i, tiff_image_path in enumerate(tiff_image_path_list): for i, tiff_image_path in enumerate(tiff_image_path_list): # get the image_array, etc # get the image_array, etc image_array, predict, diff, error= predict_pix(tiff_image_path, difference) image_array, predict, diff, error= predict_pix(tiff_image_path, difference) # calculate the number of points that will go in each bin # calculate the number of points that will go in each bin data_points_per_bin = diff.size // num_bins data_points_per_bin = diff.size // num_bins # sort the difference and create the bins # sort the difference and create the bins sorted_diff = np.sort(diff.copy()) sorted_diff = np.sort(diff.copy()) bins = [12,60,180] bins = [12,60,180] # get the boundary # get the boundary boundary = np.hstack((image_array[0,:],image_array[-1,:],image_array[1:-1,0],image_array[1:-1,-1])) boundary = np.hstack((image_array[0,:],image_array[-1,:],image_array[1:-1,0],image_array[1:-1,-1])) # take the difference of the boundary with the very first pixel # take the difference of the boundary with the very first pixel boundary = boundary - image_array[0,0] boundary = boundary - image_array[0,0] #boundary is 1dim, so boundary[0] is just the first element #boundary is 1dim, so boundary[0] is just the first element boundary[0] = image_array[0,0] boundary[0] = image_array[0,0] # huffman encode the boundary # huffman encode the boundary for j in boundary: for j in boundary: list_of_all_vals[0].append(str(j)) list_of_all_vals[0].append(str(j)) # create a list of huffman table # create a list of huffman table n = len(bins) n = len(bins) # loop through different bins # loop through different bins for k in range (0,n): for k in range (0,n): # the first bin # the first bin if k == 0 : if k == 0 : # get the point within the bin and huffman huffman_encoding_dict # get the point within the bin and huffman huffman_encoding_dict mask = diff <= bins[k] mask = diff <= bins[k] for j in error[mask].astype(int): for j in error[mask].astype(int): list_of_all_vals[k+1].append(str(j)) list_of_all_vals[k+1].append(str(j)) # the middle bins # the middle bins else: else: # get the point within the bin and huffman huffman_encoding_dict # get the point within the bin and huffman huffman_encoding_dict mask = diff > bins[k-1] mask = diff > bins[k-1] new_error = error[mask] new_error = error[mask] mask2 = diff[mask] <= bins[k] mask2 = diff[mask] <= bins[k] for j in new_error[mask2].astype(int): for j in new_error[mask2].astype(int): list_of_all_vals[k+1].append(str(j)) list_of_all_vals[k+1].append(str(j)) # the last bin # the last bin # get the point within the bin and huffman huffman_encoding_dict # get the point within the bin and huffman huffman_encoding_dict mask = diff > bins[-1] mask = diff > bins[-1] for j in error[mask].astype(int): for j in error[mask].astype(int): list_of_all_vals[-1].append(str(j)) list_of_all_vals[-1].append(str(j)) for item in list_of_all_vals: for item in list_of_all_vals: freq = dict(Counter(item)) freq = dict(Counter(item)) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) node = make_tree(freq) node = make_tree(freq) huffman_encoding_list.append(huffman_code_tree(node)) huffman_encoding_list.append(huffman_code_tree(node)) # create a error matrix that includes the boundary (used in encoding matrix) # create a error matrix that includes the boundary (used in encoding matrix) new_error = np.copy(image_array) new_error = np.copy(image_array) new_error[1:-1,1:-1] = np.reshape(error,(510, 638)) new_error[1:-1,1:-1] = np.reshape(error,(510, 638)) keep = new_error[0,0] keep = new_error[0,0] new_error[0,:] = new_error[0,:] - keep new_error[0,:] = new_error[0,:] - keep new_error[-1,:] = new_error[-1,:] - keep new_error[-1,:] = new_error[-1,:] - keep new_error[1:-1,0] = new_error[1:-1,0] - keep new_error[1:-1,0] = new_error[1:-1,0] - keep new_error[1:-1,-1] = new_error[1:-1,-1] - keep new_error[1:-1,-1] = new_error[1:-1,-1] - keep new_error[0,0] = keep new_error[0,0] = keep # huffman_encoding_list = list(set(huffman_encoding_list)) # huffman_encoding_list = list(set(huffman_encoding_list)) diff = np.reshape(diff,(510,638)) diff = np.reshape(diff,(510,638)) # return the huffman dictionary # return the huffman dictionary return huffman_encoding_list,bins return huffman_encoding_list,bins ``` ``` %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python def huffman(tiff_image_path, num_bins=4, difference = True): def huffman(tiff_image_path, num_bins=4, difference = True): """ """ This function is used to encode the error based on the difference This function is used to encode the error based on the difference and split the difference into different bins and split the difference into different bins Input: Input: tiff_image_path (string): path to the tiff file tiff_image_path (string): path to the tiff file num_bins (int): number of bins num_bins (int): number of bins Return: Return: huffman_encoding_list list (num_bins + 1): a list of dictionary huffman_encoding_list list (num_bins + 1): a list of dictionary image_array ndarray (512, 640): original image image_array ndarray (512, 640): original image new_error ndarray (512, 640): error that includes the boundary new_error ndarray (512, 640): error that includes the boundary diff ndarray (510, 638): difference of min and max of the 4 neighbors diff ndarray (510, 638): difference of min and max of the 4 neighbors boundary ndarray (2300,): the boundary values after subtracting the very first pixel value boundary ndarray (2300,): the boundary values after subtracting the very first pixel value predict ndarray (325380,): the list of predicted values predict ndarray (325380,): the list of predicted values bins list (num_bins - 1,): a list of threshold to cut the bins bins list (num_bins - 1,): a list of threshold to cut the bins A ndarray (3 X 3): system of equation A ndarray (3 X 3): system of equation """ """ # get the image_array, etc # get the image_array, etc image_array, predict, diff, error= predict_pix(tiff_image_path, difference) image_array, predict, diff, error= predict_pix(tiff_image_path, difference) # calculate the number of points that will go in each bin # calculate the number of points that will go in each bin data_points_per_bin = diff.size // num_bins data_points_per_bin = diff.size // num_bins # sort the difference and create the bins # sort the difference and create the bins sorted_diff = np.sort(diff.copy()) sorted_diff = np.sort(diff.copy()) # bins = [sorted_diff[i*data_points_per_bin] for i in range(1,num_bins)] # bins = [sorted_diff[i*data_points_per_bin] for i in range(1,num_bins)] bins = [12,60,180] bins = [12,60,180] # get the boundary # get the boundary boundary = np.hstack((image_array[0,:],image_array[-1,:],image_array[1:-1,0],image_array[1:-1,-1])) boundary = np.hstack((image_array[0,:],image_array[-1,:],image_array[1:-1,0],image_array[1:-1,-1])) # take the difference of the boundary with the very first pixel # take the difference of the boundary with the very first pixel boundary = boundary - image_array[0,0] boundary = boundary - image_array[0,0] #boundary is 1dim, so boundary[0] is just the first element #boundary is 1dim, so boundary[0] is just the first element boundary[0] = image_array[0,0] boundary[0] = image_array[0,0] # huffman encode the boundary # huffman encode the boundary bound_vals_as_string = [str(i) for i in boundary] bound_vals_as_string = [str(i) for i in boundary] freq = dict(Counter(bound_vals_as_string)) freq = dict(Counter(bound_vals_as_string)) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) node = make_tree(freq) node = make_tree(freq) huffman_encoding_dict = huffman_code_tree(node) huffman_encoding_dict = huffman_code_tree(node) # create a list of huffman table # create a list of huffman table huffman_encoding_list = [huffman_encoding_dict] huffman_encoding_list = [huffman_encoding_dict] n = len(bins) n = len(bins) # loop through different bins # loop through different bins for i in range (0,n): for i in range (0,n): # the first bin # the first bin if i == 0 : if i == 0 : # get the point within the bin and huffman huffman_encoding_dict # get the point within the bin and huffman huffman_encoding_dict mask = diff <= bins[i] mask = diff <= bins[i] line_as_string = [str(i) for i in error[mask].astype(int)] line_as_string = [str(i) for i in error[mask].astype(int)] freq = dict(Counter(line_as_string)) freq = dict(Counter(line_as_string)) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) node = make_tree(freq) node = make_tree(freq) huffman_encoding_dict = huffman_code_tree(node) huffman_encoding_dict = huffman_code_tree(node) huffman_encoding_list.append(huffman_encoding_dict) huffman_encoding_list.append(huffman_encoding_dict) # the middle bins # the middle bins else: else: # get the point within the bin and huffman huffman_encoding_dict # get the point within the bin and huffman huffman_encoding_dict mask = diff > bins[i-1] mask = diff > bins[i-1] new_error = error[mask] new_error = error[mask] mask2 = diff[mask] <= bins[i] mask2 = diff[mask] <= bins[i] line_as_string = [str(i) for i in new_error[mask2].astype(int)] line_as_string = [str(i) for i in new_error[mask2].astype(int)] freq = dict(Counter(line_as_string)) freq = dict(Counter(line_as_string)) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) node = make_tree(freq) node = make_tree(freq) huffman_encoding_dict = huffman_code_tree(node) huffman_encoding_dict = huffman_code_tree(node) huffman_encoding_list.append(huffman_encoding_dict) huffman_encoding_list.append(huffman_encoding_dict) # the last bin # the last bin # get the point within the bin and huffman huffman_encoding_dict # get the point within the bin and huffman huffman_encoding_dict mask = diff > bins[-1] mask = diff > bins[-1] line_as_string = [str(i) for i in error[mask].astype(int)] line_as_string = [str(i) for i in error[mask].astype(int)] freq = dict(Counter(line_as_string)) freq = dict(Counter(line_as_string)) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) node = make_tree(freq) node = make_tree(freq) huffman_encoding_dict = huffman_code_tree(node) huffman_encoding_dict = huffman_code_tree(node) huffman_encoding_list.append(huffman_encoding_dict) huffman_encoding_list.append(huffman_encoding_dict) # create a error matrix that includes the boundary (used in encoding matrix) # create a error matrix that includes the boundary (used in encoding matrix) new_error = np.copy(image_array) new_error = np.copy(image_array) new_error[1:-1,1:-1] = np.reshape(error,(510, 638)) new_error[1:-1,1:-1] = np.reshape(error,(510, 638)) keep = new_error[0,0] keep = new_error[0,0] new_error[0,:] = new_error[0,:] - keep new_error[0,:] = new_error[0,:] - keep new_error[-1,:] = new_error[-1,:] - keep new_error[-1,:] = new_error[-1,:] - keep new_error[1:-1,0] = new_error[1:-1,0] - keep new_error[1:-1,0] = new_error[1:-1,0] - keep new_error[1:-1,-1] = new_error[1:-1,-1] - keep new_error[1:-1,-1] = new_error[1:-1,-1] - keep new_error[0,0] = keep new_error[0,0] = keep # huffman_encoding_list = list(set(huffman_encoding_list)) # huffman_encoding_list = list(set(huffman_encoding_list)) diff = np.reshape(diff,(510,638)) diff = np.reshape(diff,(510,638)) # return the huffman dictionary # return the huffman dictionary return huffman_encoding_list, image_array, new_error, diff, boundary, predict, bins return huffman_encoding_list, image_array, new_error, diff, boundary, predict, bins ``` ``` %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python def encoder(error, list_dic, diff, bound, bins): def encoder(error, list_dic, diff, bound, bins): """ """ This function encode the matrix with huffman coding tables This function encode the matrix with huffman coding tables Input: Input: error (512, 640): a matrix with all the errors error (512, 640): a matrix with all the errors list_dic (num_dic + 1,): a list of huffman coding table list_dic (num_dic + 1,): a list of huffman coding table bound (2300,): the boundary values after subtracting the very first pixel value bound (2300,): the boundary values after subtracting the very first pixel value bins (num_bins - 1,): a list of threshold to cut the bins bins (num_bins - 1,): a list of threshold to cut the bins Return: Return: encoded (512, 640): encoded matrix encoded (512, 640): encoded matrix """ """ returnable_encode = "" returnable_encode = "" # copy the error matrix (including the boundary) # copy the error matrix (including the boundary) encoded = np.copy(error).astype(int).astype(str).astype(object) encoded = np.copy(error).astype(int).astype(str).astype(object) #diff = np.reshape(diff,(510,638)) #diff = np.reshape(diff,(510,638)) # loop through all the pixel to encode # loop through all the pixel to encode for i in range(encoded.shape[0]): for i in range(encoded.shape[0]): for j in range(encoded.shape[1]): for j in range(encoded.shape[1]): if i == 0 or i == encoded.shape[0]-1 or j == 0 or j == encoded.shape[1]-1: if i == 0 or i == encoded.shape[0]-1 or j == 0 or j == encoded.shape[1]-1: returnable_encode += list_dic[0][encoded[i][j]] returnable_encode += list_dic[0][encoded[i][j]] elif diff[i-1][j-1] <= bins[0]: elif diff[i-1][j-1] <= bins[0]: returnable_encode += list_dic[1][encoded[i][j]] returnable_encode += list_dic[1][encoded[i][j]] elif diff[i-1][j-1] <= bins[1] and diff[i-1][j-1] > bins[0]: elif diff[i-1][j-1] <= bins[1] and diff[i-1][j-1] > bins[0]: returnable_encode +=list_dic[2][encoded[i][j]] returnable_encode +=list_dic[2][encoded[i][j]] elif diff[i-1][j-1] <= bins[2] and diff[i-1][j-1] > bins[1]: elif diff[i-1][j-1] <= bins[2] and diff[i-1][j-1] > bins[1]: returnable_encode +=list_dic[3][encoded[i][j]] returnable_encode +=list_dic[3][encoded[i][j]] else: else: returnable_encode += list_dic[4][encoded[i][j]] returnable_encode += list_dic[4][encoded[i][j]] return returnable_encode return returnable_encode ``` ``` %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python # def bitstring_to_bytes(s): # def bitstring_to_bytes(s): # v = int(s, 2) # v = int(s, 2) # b = bytearray() # b = bytearray() # while v: # while v: # b.append(v & 0xff) # b.append(v & 0xff) # v >>= 8 # v >>= 8 # return bytes(b[::-1]) # return bytes(b[::-1]) def bitstring_to_bytes(input_string): def bitstring_to_bytes(input_string): int_array = [] int_array = [] length_of_string = len(input_string) length_of_string = len(input_string) while length_of_string >= 8: while length_of_string >= 8: int_array.append(int(input_string[:8],2)) int_array.append(int(input_string[:8],2)) input_string = input_string[8:] input_string = input_string[8:] length_of_string = len(input_string) length_of_string = len(input_string) if length_of_string > 0: if length_of_string > 0: int_array.append(int(input_string,2)) zerobuffer = "" print(int_array[-12:]) for _ in range(8-length_of_string): zerobuffer += "0" int_array.append(int(input_string+zerobuffer,2)) # print(int_array[0:20]) # print(int_array[-12:]) return bytes(int_array) return bytes(int_array) ``` ``` %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python def decoder(encoded_string, list_dic, bins, use_diff): def decoder(encoded_string, list_dic, bins, use_diff): """ """ This function decodes the encoded_matrix. This function decodes the encoded_matrix. Input: Input: A (3 X 3): system of equation A (3 X 3): system of equation list_dic (num_dic + 1,): a list of huffman coding table list_dic (num_dic + 1,): a list of huffman coding table encoded_matrix (512, 640): encoded matrix encoded_matrix (512, 640): encoded matrix bins (num_bins - 1,): a list of threshold to cut the bins bins (num_bins - 1,): a list of threshold to cut the bins Return: Return: decode_matrix (512, 640): decoded matrix decode_matrix (512, 640): decoded matrix """ """ A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation # change the dictionary back to list # change the dictionary back to list # !!!!!WARNING!!!! has to change this part, everytime you change the number of bins # !!!!!WARNING!!!! has to change this part, everytime you change the number of bins the_keys0 = list(list_dic[0].keys()) the_keys0 = list(list_dic[0].keys()) the_values0 = list(list_dic[0].values()) the_values0 = list(list_dic[0].values()) the_keys1 = list(list_dic[1].keys()) the_keys1 = list(list_dic[1].keys()) the_values1 = list(list_dic[1].values()) the_values1 = list(list_dic[1].values()) the_keys2 = list(list_dic[2].keys()) the_keys2 = list(list_dic[2].keys()) the_values2 = list(list_dic[2].values()) the_values2 = list(list_dic[2].values()) the_keys3 = list(list_dic[3].keys()) the_keys3 = list(list_dic[3].keys()) the_values3 = list(list_dic[3].values()) the_values3 = list(list_dic[3].values()) the_keys4 = list(list_dic[4].keys()) the_keys4 = list(list_dic[4].keys()) the_values4 = list(list_dic[4].values()) the_values4 = list(list_dic[4].values()) #Matrix system of points that will be used to solve the least squares fitting hyperplane #Matrix system of points that will be used to solve the least squares fitting hyperplane points = np.array([[-1,-1,1], [-1,0,1], [-1,1,1], [0,-1,1]]) points = np.array([[-1,-1,1], [-1,0,1], [-1,1,1], [0,-1,1]]) decode_matrix = np.zeros((512,640)) decode_matrix = np.zeros((512,640)) # loop through all the element in the matrix # loop through all the element in the matrix for i in range(decode_matrix.shape[0]): for i in range(decode_matrix.shape[0]): for j in range(decode_matrix.shape[1]): for j in range(decode_matrix.shape[1]): # if it's the very first pixel on the image # if it's the very first pixel on the image if i == 0 and j == 0: if i == 0 and j == 0: colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys0, the_values=the_values0) colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys0, the_values=the_values0) decode_matrix[i][j] = colorvalue decode_matrix[i][j] = colorvalue # if it's on the boundary (any of the 4 edges) # if it's on the boundary (any of the 4 edges) elif i == 0 or i == decode_matrix.shape[0]-1 or j == 0 or j == decode_matrix.shape[1]-1: elif i == 0 or i == decode_matrix.shape[0]-1 or j == 0 or j == decode_matrix.shape[1]-1: colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys0, the_values=the_values0) colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys0, the_values=the_values0) decode_matrix[i][j] = colorvalue + decode_matrix[0][0] decode_matrix[i][j] = colorvalue + decode_matrix[0][0] # if not the boundary # if not the boundary else: else: # predict the image with the known pixel value # predict the image with the known pixel value z0 = decode_matrix[i-1][j-1] z0 = decode_matrix[i-1][j-1] z1 = decode_matrix[i-1][j] z1 = decode_matrix[i-1][j] z2 = decode_matrix[i-1][j+1] z2 = decode_matrix[i-1][j+1] z3 = decode_matrix[i][j-1] z3 = decode_matrix[i][j-1] y0 = int(-z0+z2-z3) y0 = int(-z0+z2-z3) y1 = int(z0+z1+z2) y1 = int(z0+z1+z2) y2 = int(-z0-z1-z2-z3) y2 = int(-z0-z1-z2-z3) y = np.vstack((y0,y1,y2)) y = np.vstack((y0,y1,y2)) if use_diff: if use_diff: difference = max(z0,z1,z2,z3) - min(z0,z1,z2,z3) difference = max(z0,z1,z2,z3) - min(z0,z1,z2,z3) else: else: f, difference, rank, s = la.lstsq(points, [z0,z1,z2,z3], rcond=None) f, difference, rank, s = la.lstsq(points, [z0,z1,z2,z3], rcond=None) difference = difference.astype(int) difference = difference.astype(int) predict = np.round(np.round(np.linalg.solve(A,y)[-1][0],1)) predict = np.round(np.round(np.linalg.solve(A,y)[-1][0],1)) # add on the difference by searching the dictionary # add on the difference by searching the dictionary # !!!!!WARNING!!!! has to change this part, eveytime you change the number of bins # !!!!!WARNING!!!! has to change this part, eveytime you change the number of bins if difference <= bins[0]: if difference <= bins[0]: colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys1, the_values=the_values1) colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys1, the_values=the_values1) decode_matrix[i][j] = colorvalue + int(predict) decode_matrix[i][j] = colorvalue + int(predict) elif difference <= bins[1] and difference > bins[0]: elif difference <= bins[1] and difference > bins[0]: colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys2, the_values=the_values2) colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys2, the_values=the_values2) decode_matrix[i][j] = colorvalue + int(predict) decode_matrix[i][j] = colorvalue + int(predict) elif difference <= bins[2] and difference > bins[1]: elif difference <= bins[2] and difference > bins[1]: colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys3, the_values=the_values3) colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys3, the_values=the_values3) decode_matrix[i][j] = colorvalue + int(predict) decode_matrix[i][j] = colorvalue + int(predict) else: else: colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys4, the_values=the_values4) colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys4, the_values=the_values4) decode_matrix[i][j] = colorvalue + int(predict) decode_matrix[i][j] = colorvalue + int(predict) return decode_matrix.astype(int) return decode_matrix.astype(int) ``` ``` %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python def read_from_file(filename): def read_from_file(filename): with open(filename, 'rb') as file: with open(filename, 'rb') as file: return file.read() return file.read() ``` ``` %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python scenes = file_extractor() scenes = file_extractor() newnamesforlater = [] images = image_extractor(scenes) images = image_extractor(scenes) oglist_dic, ogbins = make_dictionary(images[:1], 4, False) oglist_dic, ogbins = make_dictionary(images[:10], 4, False) file_size_ratios = [] file_size_ratios = [] np.save("first_dic.npy", oglist_dic) np.save("first_dic.npy", oglist_dic) for i in range(1): for i in range(10): list_dic, image, new_error, diff, bound, predict, bins = huffman(images[i], 4, False) list_dic, image, new_error, diff, bound, predict, bins = huffman(images[i], 4, False) encoded_string1 = encoder(new_error, oglist_dic, diff, bound, ogbins) encoded_string1 = encoder(new_error, oglist_dic, diff, bound, ogbins) # reconstruct_image = decoder(A, encoded_string, list_dic, bins, False) # reconstruct_image = decoder(A, encoded_string, list_dic, bins, False) # print(np.allclose(image, reconstruct_image)) # print(np.allclose(image, reconstruct_image)) print(len(encoded_string1)) inletters = bitstring_to_bytes(encoded_string1) inletters = bitstring_to_bytes(encoded_string1) if images[i][:-5] == ".tiff": # reconstruct_image = decoder(encoded_string, oglist_dic, ogbins, False) newname = images[i][:-5] # print(np.allclose(image, reconstruct_image)) # print(len(list_dic)) if images[0][:-5] == ".tiff": newname = images[0][:-5] else: else: newname = images[0][:-4] newname = images[i][:-4] newnamesforlater.append(newname + "_Compressed.txt") # with open(newname + "_Compressed.txt", 'wb') as f: with open(newname + "_Compressed.txt", 'wb') as f: with open("MatrixNowString.txt", 'wb') as f: f.write(inletters) f.write(inletters) file_size_ratios.append((os.path.getsize('MatrixNowString.txt'))/os.path.getsize('images/1626032610_393963/1626032610_393963_0.tiff')) file_size_ratios.append((os.path.getsize(newname + "_Compressed.txt"))/os.path.getsize('images/1626032610_393963/1626032610_393963_0.tiff')) ``` ``` %% Output 2080559 [189, 237, 174, 241, 149, 209, 142, 187, 71, 107, 178, 113] %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python print(np.mean(file_size_ratios)) print(np.mean(file_size_ratios)) ``` ``` %% Output %% Output 0.39572609126335023 0.3953878365239038 %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python 0.6058229115470704 ``` ``` %% Output 0.6058229115470704 %% Cell type:markdown id: tags: %% Cell type:markdown id: tags: Tomorrow note: Make new thing so that it rounds better, can be used losslessly while maintaining the same dict. Currently: wildly off. Tomorrow note: Make new thing so that it rounds better, can be used losslessly while maintaining the same dict. Currently: wildly off. %% Cell type:markdown id: tags: %% Cell type:markdown id: tags: info@elphel.com info@elphel.com 801.599.6216 801.599.6216 pdf4eclipse pdf4eclipse Don't use 6, the channel is broken Don't use 6, the channel is broken %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python print(int("100",2)) print(bin(4)) ``` ``` %% Output 4 0b100 %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python def bytes_to_bitstring(input_bytearray): def bytes_to_bitstring(input_bytearray): end_string = "" end_string = "" print(input_bytearray[0:25]) # print(bytearray(input_bytearray,encoding="ISO-8859-1").decode("utf-8")[0:4]) int_array = [i for i in input_bytearray] int_array = [i for i in input_bytearray] for item in int_array: for i, item in enumerate(int_array): end_string += (bin(item)[2:]) end_string += (bin(item)[2:].zfill(8)) return end_string return end_string # oglist_dic = np.load("first_dic.npy", allow_pickle="TRUE") oglist_dic = np.load("first_dic.npy", allow_pickle="TRUE") ogbins = [12,60,180] ogbins = [12,60,180] encoded_string2 = bytes_to_bitstring(read_from_file("images/1626033496_437803/1626033496_437803_11._Compressed.txt")) for i,item in enumerate(newnamesforlater[0:10]): list_dic, image, new_error, diff, bound, predict, bins = huffman(images[i], 4, False) print(len(encoded_string2)) encoded_string2 = bytes_to_bitstring(read_from_file(item)) reconstruct_image = decoder(encoded_string2, oglist_dic, ogbins, False) reconstruct_image = decoder(encoded_string2, oglist_dic, ogbins, False) # print(np.allclose(image, reconstruct_image)) print(np.allclose(image, reconstruct_image)) ``` ``` %% Output b'\xe4\x05%4E\x1a\xd3 h\xb1\xb5\x982\xc9\x0e\xbc\x83\x90\x101\xd23\xd9\x16>' 1850876 %% Cell type:code id: tags: %% Cell type:code id: tags: ``` python ``` python ``` ```
images/1626033496_437803/1626033496_437803_1._Compressed.txt 0 → 100644 +253 KiB File added.No diff preview for this file type. View file
images/1626033496_437803/1626033496_437803_11._Compressed.txt −130 KiB (254 KiB) File changed.No diff preview for this file type. View original file View changed file
images/1626033496_437803/1626033496_437803_12._Compressed.txt 0 → 100644 +252 KiB File added.No diff preview for this file type. View file
