Loading SameTableEncoder.ipynb 0 → 100644 +462 −0 Original line number Diff line number Diff line %% Cell type:code id: tags: ``` python import numpy as np from matplotlib import pyplot as plt from itertools import product import os import sys from PIL import Image from scipy.optimize import minimize,linprog from sklearn.neighbors import KernelDensity from collections import Counter import numpy.linalg as la ``` %% Cell type:code id: tags: ``` python def file_extractor(dirname="images"): files = os.listdir(dirname) scenes = [] for file in files: if file == '.DS_Store': continue else: scenes.append(os.path.join(dirname, file)) return scenes def image_extractor(scenes): image_folder = [] for scene in scenes: files = os.listdir(scene) for file in files: if file[-5:] != ".tiff" or file[-7:] == "_6.tiff": continue else: image_folder.append(os.path.join(scene, file)) return image_folder #returns a list of file paths to .tiff files in the specified directory given in file_extractor def im_distribution(images, num): """ Function that extracts tiff files from specific cameras and returns a list of all the tiff files corresponding to that camera. i.e. all pictures labeled "_7.tiff" or otherwise specified camera numbers. Parameters: images (list): list of all tiff files, regardless of classification. This is NOT a list of directories but of specific tiff files that can be opened right away. This is the list that we iterate through and divide. num (str): a string designation for the camera number that we want to extract i.e. "14" for double digits of "_1" for single digits. Returns: tiff (list): A list of tiff files that have the specified designation from num. They are the files extracted from the 'images' list that correspond to the given num. """ tiff = [] for im in images: if im[-7:-5] == num: tiff.append(im) return tiff ``` %% Cell type:code id: tags: ``` python def predict_pix(tiff_image_path, difference = True): """ This function predict the pixel values excluding the boundary. Using the 4 neighbor pixel values and MSE to predict the next pixel value (-1,1) (0,1) (1,1) => relative position of the 4 other given values (-1,0) (0,0) => (0,0) is the one we want to predict take the derivative of mean square error to solve for the system of equation A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) A @ [a, b, c] = [-z0+z2-z3, z0+z1+z2, -z0-z1-z2-z3] where z0 = (-1,1), z1 = (0,1), z2 = (1,1), z3 = (-1,0) and the predicted pixel value is c. Input: tiff_image_path (string): path to the tiff file Return: image ndarray(512 X 640): original image predict ndarray(325380,): predicted image excluding the boundary diff. ndarray(325380,): IF difference = TRUE, difference between the min and max of four neighbors exclude the boundary ELSE: the residuals of the four nearest pixels to a fitted hyperplane error ndarray(325380,): difference between the original image and predicted image A ndarray(3 X 3): system of equation """ image_obj = Image.open(tiff_image_path) #Open the image and read it as an Image object image_array = np.array(image_obj)[1:,:].astype(int) #Convert to an array, leaving out the first row because the first row is just housekeeping data # image_array = image_array.astype(int) A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation # where z0 = (-1,1), z1 = (0,1), z2 = (1,1), z3 = (-1,0) z0 = image_array[0:-2,0:-2] # get all the first pixel for the entire image z1 = image_array[0:-2,1:-1] # get all the second pixel for the entire image z2 = image_array[0:-2,2::] # get all the third pixel for the entire image z3 = image_array[1:-1,0:-2] # get all the forth pixel for the entire image # calculate the out put of the system of equation y0 = np.ravel(-z0+z2-z3) y1 = np.ravel(z0+z1+z2) y2 = np.ravel(-z0-z1-z2-z3) y = np.vstack((y0,y1,y2)) # use numpy solver to solve the system of equations all at once #predict = np.floor(np.linalg.solve(A,y)[-1]) predict = np.round(np.round((np.linalg.solve(A,y)[-1]),1)) #Matrix system of points that will be used to solve the least squares fitting hyperplane points = np.array([[-1,-1,1], [-1,0,1], [-1,1,1], [0,-1,1]]) # flatten the neighbor pixlels and stack them together z0 = np.ravel(z0) z1 = np.ravel(z1) z2 = np.ravel(z2) z3 = np.ravel(z3) neighbor = np.vstack((z0,z1,z2,z3)).T if difference: # calculate the difference diff = np.max(neighbor,axis = 1) - np.min(neighbor, axis=1) else: #Compute the best fitting hyperplane using least squares #The res is the residuals of the four points used to fit the hyperplane (summed distance of each of the #points to the hyperplane), it is a measure of gradient f, diff, rank, s = la.lstsq(points, neighbor.T, rcond=None) diff = diff.astype(int) # calculate the error error = np.ravel(image_array[1:-1,1:-1])-predict return image_array, predict, diff, error, A ``` %% Cell type:code id: tags: ``` python """ this huffman encoding code is found online https://favtutor.com/blogs/huffman-coding """ class NodeTree(object): def __init__(self, left=None, right=None): self.left = left self.right = right def children(self): return self.left, self.right def __str__(self): return self.left, self.right def huffman_code_tree(node, binString=''): ''' Function to find Huffman Code ''' if type(node) is str: return {node: binString} (l, r) = node.children() d = dict() d.update(huffman_code_tree(l, binString + '0')) d.update(huffman_code_tree(r, binString + '1')) return d def make_tree(nodes): ''' Function to make tree :param nodes: Nodes :return: Root of the tree ''' while len(nodes) > 1: (key1, c1) = nodes[-1] (key2, c2) = nodes[-2] nodes = nodes[:-2] node = NodeTree(key1, key2) nodes.append((node, c1 + c2)) #reverse True, decending order #There is a huge memory leak here, no idea how or why nodes = sorted(nodes, key=lambda x: x[1], reverse=True) return nodes[0][0] def decode_string(huffman_string, the_keys, the_values): for i in range(len(huffman_string)): try: return (int(the_keys[the_values.index(huffman_string[:i+1])]),huffman_string[i+1:]) except: pass ``` %% Cell type:code id: tags: ``` python def huffman(tiff_image_path, num_bins=4, difference = True): """ This function is used to encode the error based on the difference and split the difference into different bins Input: tiff_image_path (string): path to the tiff file num_bins (int): number of bins Return: huffman_encoding_list list (num_bins + 1): a list of dictionary image_array ndarray (512, 640): original image new_error ndarray (512, 640): error that includes the boundary diff ndarray (510, 638): difference of min and max of the 4 neighbors boundary ndarray (2300,): the boundary values after subtracting the very first pixel value predict ndarray (325380,): the list of predicted values bins list (num_bins - 1,): a list of threshold to cut the bins A ndarray (3 X 3): system of equation """ # get the image_array, etc image_array, predict, diff, error, A = predict_pix(tiff_image_path, difference) # calculate the number of points that will go in each bin data_points_per_bin = diff.size // num_bins # sort the difference and create the bins sorted_diff = np.sort(diff.copy()) bins = [sorted_diff[i*data_points_per_bin] for i in range(1,num_bins)] # get the boundary boundary = np.hstack((image_array[0,:],image_array[-1,:],image_array[1:-1,0],image_array[1:-1,-1])) # take the difference of the boundary with the very first pixel boundary = boundary - image_array[0,0] #boundary is 1dim, so boundary[0] is just the first element boundary[0] = image_array[0,0] # huffman encode the boundary bound_vals_as_string = [str(i) for i in boundary] freq = dict(Counter(bound_vals_as_string)) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) node = make_tree(freq) huffman_encoding_dict = huffman_code_tree(node) # create a list of huffman table huffman_encoding_list = [huffman_encoding_dict] n = len(bins) # loop through different bins for i in range (0,n): # the first bin if i == 0 : # get the point within the bin and huffman huffman_encoding_dict mask = diff <= bins[i] line_as_string = [str(i) for i in error[mask].astype(int)] freq = dict(Counter(line_as_string)) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) node = make_tree(freq) huffman_encoding_dict = huffman_code_tree(node) huffman_encoding_list.append(huffman_encoding_dict) # the middle bins else: # get the point within the bin and huffman huffman_encoding_dict mask = diff > bins[i-1] new_error = error[mask] mask2 = diff[mask] <= bins[i] line_as_string = [str(i) for i in new_error[mask2].astype(int)] freq = dict(Counter(line_as_string)) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) node = make_tree(freq) huffman_encoding_dict = huffman_code_tree(node) huffman_encoding_list.append(huffman_encoding_dict) # the last bin # get the point within the bin and huffman huffman_encoding_dict mask = diff > bins[-1] line_as_string = [str(i) for i in error[mask].astype(int)] freq = dict(Counter(line_as_string)) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) node = make_tree(freq) huffman_encoding_dict = huffman_code_tree(node) huffman_encoding_list.append(huffman_encoding_dict) # create a error matrix that includes the boundary (used in encoding matrix) new_error = np.copy(image_array) new_error[1:-1,1:-1] = np.reshape(error,(510, 638)) keep = new_error[0,0] new_error[0,:] = new_error[0,:] - keep new_error[-1,:] = new_error[-1,:] - keep new_error[1:-1,0] = new_error[1:-1,0] - keep new_error[1:-1,-1] = new_error[1:-1,-1] - keep new_error[0,0] = keep # huffman_encoding_list = list(set(huffman_encoding_list)) diff = np.reshape(diff,(510,638)) # return the huffman dictionary return huffman_encoding_list, image_array, new_error, diff, boundary, predict, bins, A ``` %% Cell type:code id: tags: ``` python def encoder(error, list_dic, diff, bound, bins): """ This function encode the matrix with huffman coding tables Input: error (512, 640): a matrix with all the errors list_dic (num_dic + 1,): a list of huffman coding table bound (2300,): the boundary values after subtracting the very first pixel value bins (num_bins - 1,): a list of threshold to cut the bins Return: encoded (512, 640): encoded matrix """ returnable_encode = "" # copy the error matrix (including the boundary) encoded = np.copy(error).astype(int).astype(str).astype(object) #diff = np.reshape(diff,(510,638)) # loop through all the pixel to encode for i in range(encoded.shape[0]): for j in range(encoded.shape[1]): if i == 0 or i == encoded.shape[0]-1 or j == 0 or j == encoded.shape[1]-1: returnable_encode += list_dic[0][encoded[i][j]] elif diff[i-1][j-1] <= bins[0]: returnable_encode += list_dic[1][encoded[i][j]] elif diff[i-1][j-1] <= bins[1] and diff[i-1][j-1] > bins[0]: returnable_encode +=list_dic[2][encoded[i][j]] elif diff[i-1][j-1] <= bins[2] and diff[i-1][j-1] > bins[1]: returnable_encode +=list_dic[3][encoded[i][j]] else: returnable_encode += list_dic[4][encoded[i][j]] return returnable_encode ``` %% Cell type:code id: tags: ``` python def bitstring_to_bytes(s): v = int(s, 2) b = bytearray() while v: b.append(v & 0xff) v >>= 8 return bytes(b[::-1]) ``` %% Cell type:code id: tags: ``` python scenes = file_extractor() images = image_extractor(scenes) oglist_dic, ogimage, ognew_error, ogdiff, ogbound, ogpredict, ogbins, ogA = huffman(images[0], 4, False) file_size_ratios = [] for i in range(25,27): list_dic, image, new_error, diff, bound, predict, bins, A = huffman(images[i], 4, False) encoded_string = encoder(new_error, oglist_dic, diff, bound, ogbins) # reconstruct_image = decoder(A, encoded_string, list_dic, bins, False) # print(np.allclose(image, reconstruct_image)) inletters = bitstring_to_bytes(encoded_string) with open("MatrixNowString.txt", 'w') as f: f.write(inletters.decode("ISO-8859-1")) file_size_ratios.append((os.path.getsize('MatrixNowString.txt'))/os.path.getsize('images/1626032610_393963/1626032610_393963_0.tiff')) ``` %% Output Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here --------------------------------------------------------------------------- KeyboardInterrupt Traceback (most recent call last) /home/bryce/git/master/SameTableEncoder.ipynb Cell 8' in <cell line: 5>() <a href='vscode-notebook-cell:/home/bryce/git/master/SameTableEncoder.ipynb#ch0000006?line=6'>7</a> encoded_string = encoder(new_error, oglist_dic, diff, bound, ogbins) <a href='vscode-notebook-cell:/home/bryce/git/master/SameTableEncoder.ipynb#ch0000006?line=7'>8</a> # reconstruct_image = decoder(A, encoded_string, list_dic, bins, False) <a href='vscode-notebook-cell:/home/bryce/git/master/SameTableEncoder.ipynb#ch0000006?line=8'>9</a> # print(np.allclose(image, reconstruct_image)) ---> <a href='vscode-notebook-cell:/home/bryce/git/master/SameTableEncoder.ipynb#ch0000006?line=9'>10</a> inletters = bitstring_to_bytes(encoded_string) <a href='vscode-notebook-cell:/home/bryce/git/master/SameTableEncoder.ipynb#ch0000006?line=10'>11</a> with open("MatrixNowString.txt", 'w') as f: <a href='vscode-notebook-cell:/home/bryce/git/master/SameTableEncoder.ipynb#ch0000006?line=11'>12</a> f.write(inletters.decode("ISO-8859-1")) /home/bryce/git/master/SameTableEncoder.ipynb Cell 7' in bitstring_to_bytes(s) <a href='vscode-notebook-cell:/home/bryce/git/master/SameTableEncoder.ipynb#ch0000007?line=3'>4</a> while v: <a href='vscode-notebook-cell:/home/bryce/git/master/SameTableEncoder.ipynb#ch0000007?line=4'>5</a> b.append(v & 0xff) ----> <a href='vscode-notebook-cell:/home/bryce/git/master/SameTableEncoder.ipynb#ch0000007?line=5'>6</a> v >>= 8 <a href='vscode-notebook-cell:/home/bryce/git/master/SameTableEncoder.ipynb#ch0000007?line=6'>7</a> return bytes(b[::-1]) KeyboardInterrupt: %% Cell type:code id: tags: ``` python print(np.mean(file_size_ratios)) ``` %% Output 0.6058229115470704 %% Cell type:code id: tags: ``` python 0.6058229115470704 ``` %% Cell type:markdown id: tags: Tomorrow note: Make new thing so that it rounds better, can be used losslessly while maintaining the same dict. Currently: wildly off. %% Cell type:markdown id: tags: info@elphel.com 801.599.6216 pdf4eclipse Don't use 6, the channel is broken %% Cell type:markdown id: tags: Loading
SameTableEncoder.ipynb 0 → 100644 +462 −0 Original line number Diff line number Diff line %% Cell type:code id: tags: ``` python import numpy as np from matplotlib import pyplot as plt from itertools import product import os import sys from PIL import Image from scipy.optimize import minimize,linprog from sklearn.neighbors import KernelDensity from collections import Counter import numpy.linalg as la ``` %% Cell type:code id: tags: ``` python def file_extractor(dirname="images"): files = os.listdir(dirname) scenes = [] for file in files: if file == '.DS_Store': continue else: scenes.append(os.path.join(dirname, file)) return scenes def image_extractor(scenes): image_folder = [] for scene in scenes: files = os.listdir(scene) for file in files: if file[-5:] != ".tiff" or file[-7:] == "_6.tiff": continue else: image_folder.append(os.path.join(scene, file)) return image_folder #returns a list of file paths to .tiff files in the specified directory given in file_extractor def im_distribution(images, num): """ Function that extracts tiff files from specific cameras and returns a list of all the tiff files corresponding to that camera. i.e. all pictures labeled "_7.tiff" or otherwise specified camera numbers. Parameters: images (list): list of all tiff files, regardless of classification. This is NOT a list of directories but of specific tiff files that can be opened right away. This is the list that we iterate through and divide. num (str): a string designation for the camera number that we want to extract i.e. "14" for double digits of "_1" for single digits. Returns: tiff (list): A list of tiff files that have the specified designation from num. They are the files extracted from the 'images' list that correspond to the given num. """ tiff = [] for im in images: if im[-7:-5] == num: tiff.append(im) return tiff ``` %% Cell type:code id: tags: ``` python def predict_pix(tiff_image_path, difference = True): """ This function predict the pixel values excluding the boundary. Using the 4 neighbor pixel values and MSE to predict the next pixel value (-1,1) (0,1) (1,1) => relative position of the 4 other given values (-1,0) (0,0) => (0,0) is the one we want to predict take the derivative of mean square error to solve for the system of equation A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) A @ [a, b, c] = [-z0+z2-z3, z0+z1+z2, -z0-z1-z2-z3] where z0 = (-1,1), z1 = (0,1), z2 = (1,1), z3 = (-1,0) and the predicted pixel value is c. Input: tiff_image_path (string): path to the tiff file Return: image ndarray(512 X 640): original image predict ndarray(325380,): predicted image excluding the boundary diff. ndarray(325380,): IF difference = TRUE, difference between the min and max of four neighbors exclude the boundary ELSE: the residuals of the four nearest pixels to a fitted hyperplane error ndarray(325380,): difference between the original image and predicted image A ndarray(3 X 3): system of equation """ image_obj = Image.open(tiff_image_path) #Open the image and read it as an Image object image_array = np.array(image_obj)[1:,:].astype(int) #Convert to an array, leaving out the first row because the first row is just housekeeping data # image_array = image_array.astype(int) A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation # where z0 = (-1,1), z1 = (0,1), z2 = (1,1), z3 = (-1,0) z0 = image_array[0:-2,0:-2] # get all the first pixel for the entire image z1 = image_array[0:-2,1:-1] # get all the second pixel for the entire image z2 = image_array[0:-2,2::] # get all the third pixel for the entire image z3 = image_array[1:-1,0:-2] # get all the forth pixel for the entire image # calculate the out put of the system of equation y0 = np.ravel(-z0+z2-z3) y1 = np.ravel(z0+z1+z2) y2 = np.ravel(-z0-z1-z2-z3) y = np.vstack((y0,y1,y2)) # use numpy solver to solve the system of equations all at once #predict = np.floor(np.linalg.solve(A,y)[-1]) predict = np.round(np.round((np.linalg.solve(A,y)[-1]),1)) #Matrix system of points that will be used to solve the least squares fitting hyperplane points = np.array([[-1,-1,1], [-1,0,1], [-1,1,1], [0,-1,1]]) # flatten the neighbor pixlels and stack them together z0 = np.ravel(z0) z1 = np.ravel(z1) z2 = np.ravel(z2) z3 = np.ravel(z3) neighbor = np.vstack((z0,z1,z2,z3)).T if difference: # calculate the difference diff = np.max(neighbor,axis = 1) - np.min(neighbor, axis=1) else: #Compute the best fitting hyperplane using least squares #The res is the residuals of the four points used to fit the hyperplane (summed distance of each of the #points to the hyperplane), it is a measure of gradient f, diff, rank, s = la.lstsq(points, neighbor.T, rcond=None) diff = diff.astype(int) # calculate the error error = np.ravel(image_array[1:-1,1:-1])-predict return image_array, predict, diff, error, A ``` %% Cell type:code id: tags: ``` python """ this huffman encoding code is found online https://favtutor.com/blogs/huffman-coding """ class NodeTree(object): def __init__(self, left=None, right=None): self.left = left self.right = right def children(self): return self.left, self.right def __str__(self): return self.left, self.right def huffman_code_tree(node, binString=''): ''' Function to find Huffman Code ''' if type(node) is str: return {node: binString} (l, r) = node.children() d = dict() d.update(huffman_code_tree(l, binString + '0')) d.update(huffman_code_tree(r, binString + '1')) return d def make_tree(nodes): ''' Function to make tree :param nodes: Nodes :return: Root of the tree ''' while len(nodes) > 1: (key1, c1) = nodes[-1] (key2, c2) = nodes[-2] nodes = nodes[:-2] node = NodeTree(key1, key2) nodes.append((node, c1 + c2)) #reverse True, decending order #There is a huge memory leak here, no idea how or why nodes = sorted(nodes, key=lambda x: x[1], reverse=True) return nodes[0][0] def decode_string(huffman_string, the_keys, the_values): for i in range(len(huffman_string)): try: return (int(the_keys[the_values.index(huffman_string[:i+1])]),huffman_string[i+1:]) except: pass ``` %% Cell type:code id: tags: ``` python def huffman(tiff_image_path, num_bins=4, difference = True): """ This function is used to encode the error based on the difference and split the difference into different bins Input: tiff_image_path (string): path to the tiff file num_bins (int): number of bins Return: huffman_encoding_list list (num_bins + 1): a list of dictionary image_array ndarray (512, 640): original image new_error ndarray (512, 640): error that includes the boundary diff ndarray (510, 638): difference of min and max of the 4 neighbors boundary ndarray (2300,): the boundary values after subtracting the very first pixel value predict ndarray (325380,): the list of predicted values bins list (num_bins - 1,): a list of threshold to cut the bins A ndarray (3 X 3): system of equation """ # get the image_array, etc image_array, predict, diff, error, A = predict_pix(tiff_image_path, difference) # calculate the number of points that will go in each bin data_points_per_bin = diff.size // num_bins # sort the difference and create the bins sorted_diff = np.sort(diff.copy()) bins = [sorted_diff[i*data_points_per_bin] for i in range(1,num_bins)] # get the boundary boundary = np.hstack((image_array[0,:],image_array[-1,:],image_array[1:-1,0],image_array[1:-1,-1])) # take the difference of the boundary with the very first pixel boundary = boundary - image_array[0,0] #boundary is 1dim, so boundary[0] is just the first element boundary[0] = image_array[0,0] # huffman encode the boundary bound_vals_as_string = [str(i) for i in boundary] freq = dict(Counter(bound_vals_as_string)) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) node = make_tree(freq) huffman_encoding_dict = huffman_code_tree(node) # create a list of huffman table huffman_encoding_list = [huffman_encoding_dict] n = len(bins) # loop through different bins for i in range (0,n): # the first bin if i == 0 : # get the point within the bin and huffman huffman_encoding_dict mask = diff <= bins[i] line_as_string = [str(i) for i in error[mask].astype(int)] freq = dict(Counter(line_as_string)) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) node = make_tree(freq) huffman_encoding_dict = huffman_code_tree(node) huffman_encoding_list.append(huffman_encoding_dict) # the middle bins else: # get the point within the bin and huffman huffman_encoding_dict mask = diff > bins[i-1] new_error = error[mask] mask2 = diff[mask] <= bins[i] line_as_string = [str(i) for i in new_error[mask2].astype(int)] freq = dict(Counter(line_as_string)) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) node = make_tree(freq) huffman_encoding_dict = huffman_code_tree(node) huffman_encoding_list.append(huffman_encoding_dict) # the last bin # get the point within the bin and huffman huffman_encoding_dict mask = diff > bins[-1] line_as_string = [str(i) for i in error[mask].astype(int)] freq = dict(Counter(line_as_string)) freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) node = make_tree(freq) huffman_encoding_dict = huffman_code_tree(node) huffman_encoding_list.append(huffman_encoding_dict) # create a error matrix that includes the boundary (used in encoding matrix) new_error = np.copy(image_array) new_error[1:-1,1:-1] = np.reshape(error,(510, 638)) keep = new_error[0,0] new_error[0,:] = new_error[0,:] - keep new_error[-1,:] = new_error[-1,:] - keep new_error[1:-1,0] = new_error[1:-1,0] - keep new_error[1:-1,-1] = new_error[1:-1,-1] - keep new_error[0,0] = keep # huffman_encoding_list = list(set(huffman_encoding_list)) diff = np.reshape(diff,(510,638)) # return the huffman dictionary return huffman_encoding_list, image_array, new_error, diff, boundary, predict, bins, A ``` %% Cell type:code id: tags: ``` python def encoder(error, list_dic, diff, bound, bins): """ This function encode the matrix with huffman coding tables Input: error (512, 640): a matrix with all the errors list_dic (num_dic + 1,): a list of huffman coding table bound (2300,): the boundary values after subtracting the very first pixel value bins (num_bins - 1,): a list of threshold to cut the bins Return: encoded (512, 640): encoded matrix """ returnable_encode = "" # copy the error matrix (including the boundary) encoded = np.copy(error).astype(int).astype(str).astype(object) #diff = np.reshape(diff,(510,638)) # loop through all the pixel to encode for i in range(encoded.shape[0]): for j in range(encoded.shape[1]): if i == 0 or i == encoded.shape[0]-1 or j == 0 or j == encoded.shape[1]-1: returnable_encode += list_dic[0][encoded[i][j]] elif diff[i-1][j-1] <= bins[0]: returnable_encode += list_dic[1][encoded[i][j]] elif diff[i-1][j-1] <= bins[1] and diff[i-1][j-1] > bins[0]: returnable_encode +=list_dic[2][encoded[i][j]] elif diff[i-1][j-1] <= bins[2] and diff[i-1][j-1] > bins[1]: returnable_encode +=list_dic[3][encoded[i][j]] else: returnable_encode += list_dic[4][encoded[i][j]] return returnable_encode ``` %% Cell type:code id: tags: ``` python def bitstring_to_bytes(s): v = int(s, 2) b = bytearray() while v: b.append(v & 0xff) v >>= 8 return bytes(b[::-1]) ``` %% Cell type:code id: tags: ``` python scenes = file_extractor() images = image_extractor(scenes) oglist_dic, ogimage, ognew_error, ogdiff, ogbound, ogpredict, ogbins, ogA = huffman(images[0], 4, False) file_size_ratios = [] for i in range(25,27): list_dic, image, new_error, diff, bound, predict, bins, A = huffman(images[i], 4, False) encoded_string = encoder(new_error, oglist_dic, diff, bound, ogbins) # reconstruct_image = decoder(A, encoded_string, list_dic, bins, False) # print(np.allclose(image, reconstruct_image)) inletters = bitstring_to_bytes(encoded_string) with open("MatrixNowString.txt", 'w') as f: f.write(inletters.decode("ISO-8859-1")) file_size_ratios.append((os.path.getsize('MatrixNowString.txt'))/os.path.getsize('images/1626032610_393963/1626032610_393963_0.tiff')) ``` %% Output Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here Not here --------------------------------------------------------------------------- KeyboardInterrupt Traceback (most recent call last) /home/bryce/git/master/SameTableEncoder.ipynb Cell 8' in <cell line: 5>() <a href='vscode-notebook-cell:/home/bryce/git/master/SameTableEncoder.ipynb#ch0000006?line=6'>7</a> encoded_string = encoder(new_error, oglist_dic, diff, bound, ogbins) <a href='vscode-notebook-cell:/home/bryce/git/master/SameTableEncoder.ipynb#ch0000006?line=7'>8</a> # reconstruct_image = decoder(A, encoded_string, list_dic, bins, False) <a href='vscode-notebook-cell:/home/bryce/git/master/SameTableEncoder.ipynb#ch0000006?line=8'>9</a> # print(np.allclose(image, reconstruct_image)) ---> <a href='vscode-notebook-cell:/home/bryce/git/master/SameTableEncoder.ipynb#ch0000006?line=9'>10</a> inletters = bitstring_to_bytes(encoded_string) <a href='vscode-notebook-cell:/home/bryce/git/master/SameTableEncoder.ipynb#ch0000006?line=10'>11</a> with open("MatrixNowString.txt", 'w') as f: <a href='vscode-notebook-cell:/home/bryce/git/master/SameTableEncoder.ipynb#ch0000006?line=11'>12</a> f.write(inletters.decode("ISO-8859-1")) /home/bryce/git/master/SameTableEncoder.ipynb Cell 7' in bitstring_to_bytes(s) <a href='vscode-notebook-cell:/home/bryce/git/master/SameTableEncoder.ipynb#ch0000007?line=3'>4</a> while v: <a href='vscode-notebook-cell:/home/bryce/git/master/SameTableEncoder.ipynb#ch0000007?line=4'>5</a> b.append(v & 0xff) ----> <a href='vscode-notebook-cell:/home/bryce/git/master/SameTableEncoder.ipynb#ch0000007?line=5'>6</a> v >>= 8 <a href='vscode-notebook-cell:/home/bryce/git/master/SameTableEncoder.ipynb#ch0000007?line=6'>7</a> return bytes(b[::-1]) KeyboardInterrupt: %% Cell type:code id: tags: ``` python print(np.mean(file_size_ratios)) ``` %% Output 0.6058229115470704 %% Cell type:code id: tags: ``` python 0.6058229115470704 ``` %% Cell type:markdown id: tags: Tomorrow note: Make new thing so that it rounds better, can be used losslessly while maintaining the same dict. Currently: wildly off. %% Cell type:markdown id: tags: info@elphel.com 801.599.6216 pdf4eclipse Don't use 6, the channel is broken %% Cell type:markdown id: tags:
