moved to folder

testpyfiles/BryceCheckpoint.py

+# %%
+import numpy as np
+from matplotlib import pyplot as plt
+from itertools import product
+import os
+import sys
+from PIL import Image
+from scipy.optimize import minimize,linprog
+import time
+import seaborn as sns
+from sklearn.neighbors import KernelDensity
+import pandas as pd
+from collections import Counter
+import time
+import numpy.linalg as la
+
+# %%
+def file_extractor(dirname="images"):
+    files = os.listdir(dirname)
+    scenes = []
+    for file in files:
+        if file == '.DS_Store':
+            continue
+        else:
+            scenes.append(os.path.join(dirname, file))
+    return scenes
+
+def image_extractor(scenes):
+    image_folder = []
+    for scene in scenes:
+        files = os.listdir(scene)
+        for file in files:
+            if file[-5:] != ".tiff" or file[-7:] == "_6.tiff":
+                continue
+            else:
+                image_folder.append(os.path.join(scene, file))
+    return image_folder #returns a list of file paths to .tiff files in the specified directory given in file_extractor
+
+def im_distribution(images, num):
+    """
+    Function that extracts tiff files from specific cameras and returns a list of all
+    the tiff files corresponding to that camera. i.e. all pictures labeled "_7.tiff" or otherwise
+    specified camera numbers.
+    
+    Parameters:
+        images (list): list of all tiff files, regardless of classification. This is NOT a list of directories but
+        of specific tiff files that can be opened right away. This is the list that we iterate through and 
+        divide.
+        
+        num (str): a string designation for the camera number that we want to extract i.e. "14" for double digits
+        of "_1" for single digits.
+        
+    Returns:
+        tiff (list): A list of tiff files that have the specified designation from num. They are the files extracted
+        from the 'images' list that correspond to the given num.
+    """
+    tiff = []
+    for im in images:
+        if im[-7:-5] == num:
+            tiff.append(im)
+    return tiff
+
+# %%
+def predict_pix(tiff_image_path, difference = True):
+    """
+    This function predict the pixel values excluding the boundary.
+    Using the 4 neighbor pixel values and MSE to predict the next pixel value
+    (-1,1) (0,1) (1,1)  => relative position of the 4 other given values
+    (-1,0) (0,0)        => (0,0) is the one we want to predict
+    take the derivative of mean square error to solve for the system of equation 
+    A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]])
+    A @ [a, b, c] = [-z0+z2-z3, z0+z1+z2, -z0-z1-z2-z3] where z0 = (-1,1), z1 = (0,1), z2 = (1,1), z3 = (-1,0)
+    and the predicted pixel value is c.
+    
+    Input:
+    tiff_image_path (string): path to the tiff file
+    
+    Return:
+    image   ndarray(512 X 640): original image 
+    predict ndarray(325380,): predicted image excluding the boundary
+    diff.   ndarray(325380,): IF difference = TRUE, difference between the min and max of four neighbors exclude the boundary
+                            ELSE: the residuals of the four nearest pixels to a fitted hyperplane
+    error   ndarray(325380,): difference between the original image and predicted image
+    A       ndarray(3 X 3): system of equation
+    """
+    image_obj = Image.open(tiff_image_path)    #Open the image and read it as an Image object
+    image_array = np.array(image_obj)[1:,:].astype(int)    #Convert to an array, leaving out the first row because the first row is just housekeeping data
+    # image_array = image_array.astype(int) 
+    A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation
+    # where z0 = (-1,1), z1 = (0,1), z2 = (1,1), z3 = (-1,0)
+    z0 = image_array[0:-2,0:-2]   # get all the first pixel for the entire image
+    z1 = image_array[0:-2,1:-1]   # get all the second pixel for the entire image
+    z2 = image_array[0:-2,2::]    # get all the third pixel for the entire image
+    z3 = image_array[1:-1,0:-2]   # get all the forth pixel for the entire image
+    
+    # calculate the out put of the system of equation
+    y0 = np.ravel(-z0+z2-z3)
+    y1 = np.ravel(z0+z1+z2)
+    y2 = np.ravel(-z0-z1-z2-z3)
+    y = np.vstack((y0,y1,y2))
+    
+    # use numpy solver to solve the system of equations all at once
+    #predict = np.floor(np.linalg.solve(A,y)[-1])
+    predict = np.round(np.round((np.linalg.solve(A,y)[-1]),1))
+    
+    #Matrix system of points that will be used to solve the least squares fitting hyperplane
+    points = np.array([[-1,-1,1], [-1,0,1], [-1,1,1], [0,-1,1]])
+    
+    # flatten the neighbor pixlels and stack them together
+    z0 = np.ravel(z0)
+    z1 = np.ravel(z1)
+    z2 = np.ravel(z2)
+    z3 = np.ravel(z3)
+    neighbor = np.vstack((z0,z1,z2,z3)).T
+    
+    if difference:
+        # calculate the difference
+        diff = np.max(neighbor,axis = 1) - np.min(neighbor, axis=1)
+    
+    else:
+        #Compute the best fitting hyperplane using least squares
+        #The res is the residuals of the four points used to fit the hyperplane (summed distance of each of the 
+        #points to the hyperplane), it is a measure of gradient
+        f, diff, rank, s = la.lstsq(points, neighbor.T, rcond=None)
+        diff = diff.astype(int)
+    
+    # calculate the error
+    error = np.ravel(image_array[1:-1,1:-1])-predict
+    
+    return image_array, predict, diff, error, A
+
+# %%
+"""
+this huffman encoding code is found online
+https://favtutor.com/blogs/huffman-coding
+"""
+
+class NodeTree(object):
+    def __init__(self, left=None, right=None):
+        self.left = left
+        self.right = right
+
+    def children(self):
+        return self.left, self.right
+
+    def __str__(self):
+        return self.left, self.right
+
+
+def huffman_code_tree(node, binString=''):
+    '''
+    Function to find Huffman Code
+    '''
+    if type(node) is str:
+        return {node: binString}
+    (l, r) = node.children()
+    d = dict()
+    d.update(huffman_code_tree(l, binString + '0'))
+    d.update(huffman_code_tree(r, binString + '1'))
+    return d
+
+
+def make_tree(nodes):
+    '''
+    Function to make tree
+    :param nodes: Nodes
+    :return: Root of the tree
+    '''
+    while len(nodes) > 1:
+        (key1, c1) = nodes[-1]
+        (key2, c2) = nodes[-2]
+        nodes = nodes[:-2]
+        node = NodeTree(key1, key2)
+        nodes.append((node, c1 + c2))
+        #reverse True, decending order
+        sorted_nodes = sorted(nodes, key=lambda x: x[1], reverse=True)
+    return sorted_nodes[0][0]
+
+# %%
+def huffman(tiff_image_path, num_bins=4, difference = True):
+    """
+    This function is used to encode the error based on the difference
+    and split the difference into different bins
+    
+    Input:
+    tiff_image_path     (string): path to the tiff file
+    num_bins            (int): number of bins
+    
+    Return:
+    list_dic    list (num_bins + 1): a list of dictionary
+    image_array array (512, 640): original image
+    new_error   array (512, 640): error that includes the boundary
+    diff        array (510, 638): difference of min and max of the 4 neighbors
+    boundary   (2300,): the boundary values after subtracting the very first pixel value
+    predict    (325380,): the list of predicted values
+    bins       (num_bins - 1,): a list of threshold to cut the bins
+    A          (3 X 3): system of equation
+    
+    """
+    # get the image_array, etc
+    image_array, predict, diff, error, A = predict_pix(tiff_image_path, difference)
+    
+    # calculate the number of points that will go in each bin
+    data_points_per_bin = diff.size // num_bins
+
+    # sort the difference and create the bins
+    sorted_diff = np.sort(diff.copy())
+    bins = [sorted_diff[i*data_points_per_bin] for i in range(1,num_bins)]
+    
+    # get the boundary 
+    boundary = np.hstack((image_array[0,:],image_array[-1,:],image_array[1:-1,0],image_array[1:-1,-1]))
+    
+    # take the difference of the boundary with the very first pixel
+    boundary = boundary - image_array[0,0]
+    
+    #boundary is 1dim, so boundary[0] is just the first element
+    boundary[0] = image_array[0,0]
+    
+    # huffman encode the boundary
+    bound_vals_as_string = [str(i) for i in boundary]
+    freq = dict(Counter(bound_vals_as_string))
+    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)
+    node = make_tree(freq)
+    huffman_encoding_dict = huffman_code_tree(node)
+    
+    # create a list of huffman table
+    huffman_encoding_list = [huffman_encoding_dict]
+    n = len(bins)
+    
+    # loop through different bins
+    for i in range (0,n):
+        # the first bin
+        if i == 0 :
+            # get the point within the bin and huffman huffman_encoding_dict
+            mask = diff <= bins[i]
+            line_as_string = [str(i) for i in error[mask].astype(int)]
+            freq = dict(Counter(line_as_string))
+            freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)
+            node = make_tree(freq)
+            huffman_encoding_dict = huffman_code_tree(node)
+            huffman_encoding_list.append(huffman_encoding_dict)
+            
+        # the middle bins
+        else:
+            # get the point within the bin and huffman huffman_encoding_dict
+            mask = diff > bins[i-1]
+            new_error = error[mask]
+            mask2 = diff[mask] <= bins[i]
+            line_as_string = [str(i) for i in new_error[mask2].astype(int)]
+            freq = dict(Counter(line_as_string))
+            freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)
+            node = make_tree(freq)
+            huffman_encoding_dict = huffman_code_tree(node)
+            huffman_encoding_list.append(huffman_encoding_dict)
+            
+    # the last bin       
+    # get the point within the bin and huffman huffman_encoding_dict
+    mask = diff > bins[-1]
+    line_as_string = [str(i) for i in error[mask].astype(int)]
+    freq = dict(Counter(line_as_string))
+    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)
+    node = make_tree(freq)
+    huffman_encoding_dict = huffman_code_tree(node)
+    huffman_encoding_list.append(huffman_encoding_dict)
+
+    # create a error matrix that includes the boundary (used in encoding matrix)
+    new_error = np.copy(image_array)
+    new_error[1:-1,1:-1] = np.reshape(error,(510, 638))
+    keep = new_error[0,0]
+    new_error[0,:] = new_error[0,:] - keep
+    new_error[-1,:] = new_error[-1,:] - keep
+    new_error[1:-1,0] = new_error[1:-1,0] - keep
+    new_error[1:-1,-1] = new_error[1:-1,-1] - keep
+    new_error[0,0] = keep
+    
+    diff = np.reshape(diff,(510,638))
+    # return the huffman dictionary
+    return huffman_encoding_list, image_array, new_error, diff, boundary, predict, bins, A
+ 
+
+
+# %%
+def encoder(error, list_dic, diff, bound, bins):
+    """
+    This function encode the matrix with huffman coding tables
+    
+    Input:
+    error     (512, 640): a matrix with all the errors
+    list_dic  (num_dic + 1,): a list of huffman coding table 
+    bound     (2300,): the boundary values after subtracting the very first pixel value
+    bins       (num_bins - 1,): a list of threshold to cut the bins
+    
+    Return:
+    encoded   (512, 640): encoded matrix
+    """
+    # copy the error matrix (including the boundary)
+    encoded = np.copy(error).astype(int).astype(str).astype(object)
+    #diff = np.reshape(diff,(510,638))
+    # loop through all the pixel to encode
+    for i in range(encoded.shape[0]):
+        for j in range(encoded.shape[1]):
+            if i == 0 or i == encoded.shape[0]-1 or j == 0 or j == encoded.shape[1]-1:
+                encoded[i][j] = list_dic[0][encoded[i][j]]
+            elif diff[i-1][j-1] <= bins[0]:
+                encoded[i][j] = list_dic[1][encoded[i][j]]
+            elif diff[i-1][j-1] <= bins[1] and diff[i-1][j-1] > bins[0]:
+                encoded[i][j] = list_dic[2][encoded[i][j]]
+            elif diff[i-1][j-1] <= bins[2] and diff[i-1][j-1] > bins[1]:
+                encoded[i][j] = list_dic[3][encoded[i][j]]
+            else: 
+                encoded[i][j] = list_dic[4][encoded[i][j]]
+
+    return encoded
+
+# %%
+def decoder(A, encoded_matrix, list_dic, bins, use_diff):
+    """
+    This function decodes the encoded_matrix.
+    Input:
+    A               (3 X 3): system of equation
+    list_dic        (num_dic + 1,): a list of huffman coding table 
+    encoded_matrix  (512, 640): encoded matrix
+    bins            (num_bins - 1,): a list of threshold to cut the bins
+    
+    Return:
+    decode_matrix   (512, 640): decoded matrix
+    """
+    # change the dictionary back to list
+    # !!!!!WARNING!!!! has to change this part, eveytime you change the number of bins
+    the_keys0 = list(list_dic[0].keys())
+    the_values0 = list(list_dic[0].values())
+    
+    the_keys1 = list(list_dic[1].keys())
+    the_values1 = list(list_dic[1].values())
+    
+    the_keys2 = list(list_dic[2].keys())
+    the_values2 = list(list_dic[2].values())
+    
+    the_keys3 = list(list_dic[3].keys())
+    the_values3 = list(list_dic[3].values())
+    
+    the_keys4 = list(list_dic[4].keys())
+    the_values4 = list(list_dic[4].values())
+    
+    #Matrix system of points that will be used to solve the least squares fitting hyperplane
+    points = np.array([[-1,-1,1], [-1,0,1], [-1,1,1], [0,-1,1]])
+    
+    decode_matrix = np.zeros((512,640))
+    # loop through all the element in the matrix
+    for i in range(decode_matrix.shape[0]):
+        for j in range(decode_matrix.shape[1]):
+            # if it's the very first pixel on the image
+            if i == 0 and j == 0:
+                decode_matrix[i][j] = int(the_keys0[the_values0.index(encoded_matrix[i,j])])
+            # if it's on the boundary
+            elif i == 0 or i == decode_matrix.shape[0]-1 or j == 0 or j == decode_matrix.shape[1]-1:
+                decode_matrix[i][j] = int(the_keys0[the_values0.index(encoded_matrix[i,j])]) + decode_matrix[0][0]
+            # if not the boundary
+            else:
+                # predict the image with the known pixel value
+                z0 = decode_matrix[i-1][j-1]
+                z1 = decode_matrix[i-1][j]
+                z2 = decode_matrix[i-1][j+1]
+                z3 = decode_matrix[i][j-1]
+                y0 = int(-z0+z2-z3)
+                y1 = int(z0+z1+z2)
+                y2 = int(-z0-z1-z2-z3)
+                y = np.vstack((y0,y1,y2))
+                if use_diff:
+                    difference = max(z0,z1,z2,z3) - min(z0,z1,z2,z3)
+                else:
+                    
+                    f, difference, rank, s = la.lstsq(points, [z0,z1,z2,z3], rcond=None) 
+                    difference = difference.astype(int)
+                    
+                predict = np.round(np.round(np.linalg.solve(A,y)[-1][0],1))
+                
+                # add on the difference by searching the dictionary
+                # !!!!!WARNING!!!! has to change this part, eveytime you change the number of bins
+                if difference <= bins[0]:
+                    decode_matrix[i][j] = int(the_keys1[the_values1.index(encoded_matrix[i,j])]) + int(predict)
+                elif difference <= bins[1] and difference > bins[0]:
+                    decode_matrix[i][j] = int(the_keys2[the_values2.index(encoded_matrix[i,j])]) + int(predict)
+                elif difference <= bins[2] and difference > bins[1]:
+                    decode_matrix[i][j] = int(the_keys3[the_values3.index(encoded_matrix[i,j])]) + int(predict)
+                else:
+                    decode_matrix[i][j] = int(the_keys4[the_values4.index(encoded_matrix[i,j])]) + int(predict)
+                
+                
+    return decode_matrix.astype(int)
+
+# %%
+def compress_rate(image, new_error, diff, bound, list_dic, bins):
+    '''
+    This function is used to calculate the compression rate.
+    Input:
+    image      (512, 640): original image
+    new_error  (512, 640): error that includes the boundary
+    diff       (510, 638): difference of min and max of the 4 neighbors
+    bound      (2300,): the boundary values after subtracting the very first pixel value
+    list_dic   (num_dic + 1,): a list of huffman coding table 
+    bins       (num_bins - 1,): a list of threshold to cut the bins
+    
+    Return:
+    compression rate
+    '''
+    # the bits for the original image
+    o_len = 0
+    # the bits for the compressed image
+    c_len = 0
+    # initializing the varible 
+    im = np.reshape(image,(512, 640))
+    real_b = np.hstack((image[0,:],image[-1,:],image[1:-1,0],image[1:-1,-1]))
+    original = image[1:-1,1:-1].reshape(-1)
+    diff = diff.reshape(-1)
+    error = new_error[1:-1,1:-1].reshape(-1)
+    
+    # calculate the bit for boundary
+    for i in range(0,len(bound)):
+        o_len += len(bin(real_b[i])[2:])
+        c_len += len(list_dic[0][str(bound[i])])
+    
+    # calculate the bit for the pixels inside the boundary
+    for i in range(0,len(original)):
+
+        # for the original image
+        o_len += len(bin(original[i])[2:])
+        
+        # check the difference and find the coresponding huffman table
+        # !!!!!WARNING!!!! has to change this part, eveytime you change the number of bins
+        if diff[i] <= bins[0]:
+            c_len += len(list_dic[1][str(int(error[i]))])
+            
+        elif diff[i] <= bins[1] and diff[i] > bins[0]:
+            c_len += len(list_dic[2][str(int(error[i]))])
+            
+        elif diff[i] <= bins[2] and diff[i] > bins[1]:
+            c_len += len(list_dic[3][str(int(error[i]))])
+
+        else: 
+            c_len += len(list_dic[4][str(int(error[i]))])
+
+    return c_len/o_len
+
+# %%
+scenes = file_extractor()
+images = image_extractor(scenes)
+list_dic, image, new_error, diff, bound, predict, bins, A = huffman(images[0], 4, False)
+encoded_matrix = encoder(new_error, list_dic, diff, bound, bins)
+reconstruct_image = decoder(A, encoded_matrix, list_dic, bins, False)
+print(np.allclose(image, reconstruct_image))
+print(len(list_dic))
+
+# %%
+compress_rate(image, new_error, diff, bound, list_dic, bins)
+
+# %%
+print(sys.getsizeof(encoded_matrix))
+print(sys.getsizeof(reconstruct_image))
+
+# %%
+
+
+

testpyfiles/ReducedRuntimeEncoder.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "from matplotlib import pyplot as plt\n",
+    "import os\n",
+    "import sys\n",
+    "from PIL import Image\n",
+    "from collections import Counter\n",
+    "import numpy.linalg as la"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def file_extractor(dirname=\"images\"):\n",
+    "    files = os.listdir(dirname)\n",
+    "    scenes = []\n",
+    "    for file in files:\n",
+    "        if file == '.DS_Store':\n",
+    "            continue\n",
+    "        else:\n",
+    "            scenes.append(os.path.join(dirname, file))\n",
+    "    return scenes\n",
+    "\n",
+    "def image_extractor(scenes):\n",
+    "    image_folder = []\n",
+    "    for scene in scenes:\n",
+    "        files = os.listdir(scene)\n",
+    "        for file in files:\n",
+    "            if file[-5:] != \".tiff\" or file[-7:] == \"_6.tiff\":\n",
+    "                continue\n",
+    "            else:\n",
+    "                image_folder.append(os.path.join(scene, file))\n",
+    "    return image_folder #returns a list of file paths to .tiff files in the specified directory given in file_extractor"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def predict_pix(tiff_image_path, difference = True):\n",
+    "    \"\"\"\n",
+    "    This function predict the pixel values excluding the boundary.\n",
+    "    Using the 4 neighbor pixel values and MSE to predict the next pixel value\n",
+    "    (-1,1) (0,1) (1,1)  => relative position of the 4 other given values\n",
+    "    (-1,0) (0,0)        => (0,0) is the one we want to predict\n",
+    "    take the derivative of mean square error to solve for the system of equation \n",
+    "    A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]])\n",
+    "    A @ [a, b, c] = [-z0+z2-z3, z0+z1+z2, -z0-z1-z2-z3] where z0 = (-1,1), z1 = (0,1), z2 = (1,1), z3 = (-1,0)\n",
+    "    and the predicted pixel value is c.\n",
+    "    \n",
+    "    Input:\n",
+    "    tiff_image_path (string): path to the tiff file\n",
+    "    \n",
+    "    Return:\n",
+    "    image   ndarray(512 X 640): original image \n",
+    "    predict ndarray(325380,): predicted image excluding the boundary\n",
+    "    diff.   ndarray(325380,): IF difference = TRUE, difference between the min and max of four neighbors exclude the boundary\n",
+    "                            ELSE: the residuals of the four nearest pixels to a fitted hyperplane\n",
+    "    error   ndarray(325380,): difference between the original image and predicted image\n",
+    "    A       ndarray(3 X 3): system of equation\n",
+    "    \"\"\"\n",
+    "    image_obj = Image.open(tiff_image_path)    #Open the image and read it as an Image object\n",
+    "    image_array = np.array(image_obj)[1:,:].astype(int)    #Convert to an array, leaving out the first row because the first row is just housekeeping data\n",
+    "    # image_array = image_array.astype(int) \n",
+    "    A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation\n",
+    "    # where z0 = (-1,1), z1 = (0,1), z2 = (1,1), z3 = (-1,0)\n",
+    "    z0 = image_array[0:-2,0:-2]   # get all the first pixel for the entire image\n",
+    "    z1 = image_array[0:-2,1:-1]   # get all the second pixel for the entire image\n",
+    "    z2 = image_array[0:-2,2::]    # get all the third pixel for the entire image\n",
+    "    z3 = image_array[1:-1,0:-2]   # get all the forth pixel for the entire image\n",
+    "    \n",
+    "    # calculate the out put of the system of equation\n",
+    "    y0 = np.ravel(-z0+z2-z3)\n",
+    "    y1 = np.ravel(z0+z1+z2)\n",
+    "    y2 = np.ravel(-z0-z1-z2-z3)\n",
+    "    y = np.vstack((y0,y1,y2))\n",
+    "    \n",
+    "    # use numpy solver to solve the system of equations all at once\n",
+    "    #predict = np.floor(np.linalg.solve(A,y)[-1])\n",
+    "    predict = np.round(np.round((np.linalg.solve(A,y)[-1]),1))\n",
+    "    \n",
+    "    #Matrix system of points that will be used to solve the least squares fitting hyperplane\n",
+    "    points = np.array([[-1,-1,1], [-1,0,1], [-1,1,1], [0,-1,1]])\n",
+    "    \n",
+    "    # flatten the neighbor pixlels and stack them together\n",
+    "    z0 = np.ravel(z0)\n",
+    "    z1 = np.ravel(z1)\n",
+    "    z2 = np.ravel(z2)\n",
+    "    z3 = np.ravel(z3)\n",
+    "    neighbor = np.vstack((z0,z1,z2,z3)).T\n",
+    "    \n",
+    "    if difference:\n",
+    "        # calculate the difference\n",
+    "        diff = np.max(neighbor,axis = 1) - np.min(neighbor, axis=1)\n",
+    "    \n",
+    "    else:\n",
+    "        #Compute the best fitting hyperplane using least squares\n",
+    "        #The res is the residuals of the four points used to fit the hyperplane (summed distance of each of the \n",
+    "        #points to the hyperplane), it is a measure of gradient\n",
+    "        f, diff, rank, s = la.lstsq(points, neighbor.T, rcond=None)\n",
+    "        diff = diff.astype(int)\n",
+    "    \n",
+    "    # calculate the error\n",
+    "    error = np.ravel(image_array[1:-1,1:-1])-predict\n",
+    "    \n",
+    "    return image_array, predict, diff, error, A"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\"\"\"\n",
+    "this huffman encoding code is found online\n",
+    "https://favtutor.com/blogs/huffman-coding\n",
+    "\"\"\"\n",
+    "\n",
+    "class NodeTree(object):\n",
+    "    def __init__(self, left=None, right=None):\n",
+    "        self.left = left\n",
+    "        self.right = right\n",
+    "\n",
+    "    def children(self):\n",
+    "        return self.left, self.right\n",
+    "\n",
+    "    def __str__(self):\n",
+    "        return self.left, self.right\n",
+    "\n",
+    "\n",
+    "def huffman_code_tree(node, binString=''):\n",
+    "    '''\n",
+    "    Function to find Huffman Code\n",
+    "    '''\n",
+    "    if type(node) is str:\n",
+    "        return {node: binString}\n",
+    "    (l, r) = node.children()\n",
+    "    d = dict()\n",
+    "    d.update(huffman_code_tree(l, binString + '0'))\n",
+    "    d.update(huffman_code_tree(r, binString + '1'))\n",
+    "    return d\n",
+    "\n",
+    "\n",
+    "def make_tree(nodes):\n",
+    "    '''\n",
+    "    Function to make tree\n",
+    "    :param nodes: Nodes\n",
+    "    :return: Root of the tree\n",
+    "    '''\n",
+    "    while len(nodes) > 1:\n",
+    "        (key1, c1) = nodes[-1]\n",
+    "        (key2, c2) = nodes[-2]\n",
+    "        nodes = nodes[:-2]\n",
+    "        node = NodeTree(key1, key2)\n",
+    "        nodes.append((node, c1 + c2))\n",
+    "        #reverse True, decending order\n",
+    "\n",
+    "        #There is a huge memory leak here, no idea how or why\n",
+    "        nodes = sorted(nodes, key=lambda x: x[1], reverse=True)\n",
+    "    return nodes[0][0]\n",
+    "\n",
+    "def decode_string(huffman_string,node):\n",
+    "    if type(node) is str:\n",
+    "        return huffman_string, int(node)\n",
+    "    else:\n",
+    "        if (huffman_string[0] == '0'):\n",
+    "            return decode_string(huffman_string[1:],node.left)\n",
+    "        else:\n",
+    "            return decode_string(huffman_string[1:],node.right)\n",
+    "    \n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def huffman(tiff_image_path, num_bins=4, difference = True):\n",
+    "    \"\"\"\n",
+    "    This function is used to encode the error based on the difference\n",
+    "    and split the difference into different bins\n",
+    "    \n",
+    "    Input:\n",
+    "    tiff_image_path     (string): path to the tiff file\n",
+    "    num_bins            (int): number of bins\n",
+    "    \n",
+    "    Return:\n",
+    "    huffman_encoding_list  list    (num_bins + 1): a list of dictionary\n",
+    "    image_array            ndarray (512, 640): original image\n",
+    "    new_error              ndarray (512, 640): error that includes the boundary\n",
+    "    diff                   ndarray (510, 638): difference of min and max of the 4 neighbors\n",
+    "    boundary               ndarray (2300,): the boundary values after subtracting the very first pixel value\n",
+    "    predict                ndarray (325380,): the list of predicted values\n",
+    "    bins                   list    (num_bins - 1,): a list of threshold to cut the bins\n",
+    "    A                      ndarray (3 X 3): system of equation\n",
+    "    \n",
+    "    \"\"\"\n",
+    "    # get the image_array, etc\n",
+    "    image_array, predict, diff, error, A = predict_pix(tiff_image_path, difference)\n",
+    "    \n",
+    "    # calculate the number of points that will go in each bin\n",
+    "    data_points_per_bin = diff.size // num_bins\n",
+    "\n",
+    "    # sort the difference and create the bins\n",
+    "    sorted_diff = np.sort(diff.copy())\n",
+    "    bins = [sorted_diff[i*data_points_per_bin] for i in range(1,num_bins)]\n",
+    "    \n",
+    "    # get the boundary \n",
+    "    boundary = np.hstack((image_array[0,:],image_array[-1,:],image_array[1:-1,0],image_array[1:-1,-1]))\n",
+    "    \n",
+    "    # take the difference of the boundary with the very first pixel\n",
+    "    boundary = boundary - image_array[0,0]\n",
+    "    \n",
+    "    #boundary is 1dim, so boundary[0] is just the first element\n",
+    "    boundary[0] = image_array[0,0]\n",
+    "    \n",
+    "    # huffman encode the boundary\n",
+    "    bound_vals_as_string = [str(i) for i in boundary]\n",
+    "    freq = dict(Counter(bound_vals_as_string))\n",
+    "    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)\n",
+    "    node1 = make_tree(freq)\n",
+    "    huffman_encoding_dict = huffman_code_tree(node1)\n",
+    "    nodelist = [node1]\n",
+    "    # create a list of huffman table\n",
+    "    huffman_encoding_list = [huffman_encoding_dict]\n",
+    "    n = len(bins)\n",
+    "    \n",
+    "    # loop through different bins\n",
+    "    for i in range (0,n):\n",
+    "        # the first bin\n",
+    "        if i == 0 :\n",
+    "            # get the point within the bin and huffman huffman_encoding_dict\n",
+    "            mask = diff <= bins[i]\n",
+    "            line_as_string = [str(i) for i in error[mask].astype(int)]\n",
+    "            freq = dict(Counter(line_as_string))\n",
+    "            freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)\n",
+    "            node2 = make_tree(freq)\n",
+    "            huffman_encoding_dict = huffman_code_tree(node2)\n",
+    "            huffman_encoding_list.append(huffman_encoding_dict)\n",
+    "            nodelist.append(node2)\n",
+    "        # the middle bins\n",
+    "        else:\n",
+    "            # get the point within the bin and huffman huffman_encoding_dict\n",
+    "            mask = diff > bins[i-1]\n",
+    "            new_error = error[mask]\n",
+    "            mask2 = diff[mask] <= bins[i]\n",
+    "            line_as_string = [str(i) for i in new_error[mask2].astype(int)]\n",
+    "            freq = dict(Counter(line_as_string))\n",
+    "            freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)\n",
+    "            node3 = make_tree(freq)\n",
+    "            huffman_encoding_dict = huffman_code_tree(node3)\n",
+    "            huffman_encoding_list.append(huffman_encoding_dict)\n",
+    "            nodelist.append(node3)\n",
+    "    # the last bin       \n",
+    "    # get the point within the bin and huffman huffman_encoding_dict\n",
+    "    mask = diff > bins[-1]\n",
+    "    line_as_string = [str(i) for i in error[mask].astype(int)]\n",
+    "    freq = dict(Counter(line_as_string))\n",
+    "    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)\n",
+    "    node4 = make_tree(freq)\n",
+    "    huffman_encoding_dict = huffman_code_tree(node4)\n",
+    "    huffman_encoding_list.append(huffman_encoding_dict)\n",
+    "    nodelist.append(node4)\n",
+    "    # create a error matrix that includes the boundary (used in encoding matrix)\n",
+    "    new_error = np.copy(image_array)\n",
+    "    new_error[1:-1,1:-1] = np.reshape(error,(510, 638))\n",
+    "    keep = new_error[0,0]\n",
+    "    new_error[0,:] = new_error[0,:] - keep\n",
+    "    new_error[-1,:] = new_error[-1,:] - keep\n",
+    "    new_error[1:-1,0] = new_error[1:-1,0] - keep\n",
+    "    new_error[1:-1,-1] = new_error[1:-1,-1] - keep\n",
+    "    new_error[0,0] = keep\n",
+    "    # huffman_encoding_list = list(set(huffman_encoding_list))\n",
+    "    diff = np.reshape(diff,(510,638))\n",
+    "    # return the huffman dictionary\n",
+    "    return huffman_encoding_list, image_array, new_error, diff, boundary, predict, bins, A, nodelist\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def encoder(error, list_dic, diff, bound, bins):\n",
+    "    \"\"\"\n",
+    "    This function encode the matrix with huffman coding tables\n",
+    "    \n",
+    "    Input:\n",
+    "    error     (512, 640): a matrix with all the errors\n",
+    "    list_dic  (num_dic + 1,): a list of huffman coding table \n",
+    "    bound     (2300,): the boundary values after subtracting the very first pixel value\n",
+    "    bins       (num_bins - 1,): a list of threshold to cut the bins\n",
+    "    \n",
+    "    Return:\n",
+    "    encoded   (512, 640): encoded matrix\n",
+    "    \"\"\"\n",
+    "    returnable_encode = \"\"\n",
+    "    # copy the error matrix (including the boundary)\n",
+    "    encoded = np.copy(error).astype(int).astype(str).astype(object)\n",
+    "    #diff = np.reshape(diff,(510,638))\n",
+    "    # loop through all the pixel to encode\n",
+    "    for i in range(encoded.shape[0]):\n",
+    "        for j in range(encoded.shape[1]):\n",
+    "            if i == 0 or i == encoded.shape[0]-1 or j == 0 or j == encoded.shape[1]-1:\n",
+    "                returnable_encode += list_dic[0][encoded[i][j]]\n",
+    "            elif diff[i-1][j-1] <= bins[0]:\n",
+    "                returnable_encode += list_dic[1][encoded[i][j]]\n",
+    "            elif diff[i-1][j-1] <= bins[1] and diff[i-1][j-1] > bins[0]:\n",
+    "                returnable_encode +=list_dic[2][encoded[i][j]]\n",
+    "            elif diff[i-1][j-1] <= bins[2] and diff[i-1][j-1] > bins[1]:\n",
+    "                returnable_encode +=list_dic[3][encoded[i][j]]\n",
+    "            else:\n",
+    "                returnable_encode += list_dic[4][encoded[i][j]]\n",
+    "\n",
+    "    return returnable_encode"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def decoder(A, encoded_string, list_dic, bins, use_diff, nodes):\n",
+    "    \"\"\"\n",
+    "    This function decodes the encoded_matrix.\n",
+    "    Input:\n",
+    "    A               (3 X 3): system of equation\n",
+    "    list_dic        (num_dic + 1,): a list of huffman coding table \n",
+    "    encoded_matrix  (512, 640): encoded matrix\n",
+    "    bins            (num_bins - 1,): a list of threshold to cut the bins\n",
+    "    \n",
+    "    Return:\n",
+    "    decode_matrix   (512, 640): decoded matrix\n",
+    "    \"\"\"\n",
+    "    # change the dictionary back to list\n",
+    "    # !!!!!WARNING!!!! has to change this part, everytime you change the number of bins\n",
+    "    the_keys0 = list(list_dic[0].keys())\n",
+    "    the_values0 = list(list_dic[0].values())\n",
+    "    \n",
+    "    the_keys1 = list(list_dic[1].keys())\n",
+    "    the_values1 = list(list_dic[1].values())\n",
+    "    \n",
+    "    the_keys2 = list(list_dic[2].keys())\n",
+    "    the_values2 = list(list_dic[2].values())\n",
+    "    \n",
+    "    the_keys3 = list(list_dic[3].keys())\n",
+    "    the_values3 = list(list_dic[3].values())\n",
+    "    \n",
+    "    the_keys4 = list(list_dic[4].keys())\n",
+    "    the_values4 = list(list_dic[4].values())\n",
+    "    #Matrix system of points that will be used to solve the least squares fitting hyperplane\n",
+    "    points = np.array([[-1,-1,1], [-1,0,1], [-1,1,1], [0,-1,1]])\n",
+    "    \n",
+    "    decode_matrix = np.zeros((512,640))\n",
+    "    # loop through all the element in the matrix\n",
+    "    for i in range(decode_matrix.shape[0]):\n",
+    "        for j in range(decode_matrix.shape[1]):\n",
+    "            # if it's the very first pixel on the image\n",
+    "            if i == 0 and j == 0:\n",
+    "                encoded_string, colorvalue = decode_string(encoded_string, nodes[0])\n",
+    "                decode_matrix[i][j] = colorvalue\n",
+    "                \n",
+    "            # if it's on the boundary (any of the 4 edges)\n",
+    "            elif i == 0 or i == decode_matrix.shape[0]-1 or j == 0 or j == decode_matrix.shape[1]-1:\n",
+    "                encoded_string, colorvalue = decode_string(encoded_string, nodes[0])\n",
+    "                decode_matrix[i][j] = colorvalue + decode_matrix[0][0]\n",
+    "            # if not the boundary\n",
+    "            else:\n",
+    "                # predict the image with the known pixel value\n",
+    "                z0 = decode_matrix[i-1][j-1]\n",
+    "                z1 = decode_matrix[i-1][j]\n",
+    "                z2 = decode_matrix[i-1][j+1]\n",
+    "                z3 = decode_matrix[i][j-1]\n",
+    "                y0 = int(-z0+z2-z3)\n",
+    "                y1 = int(z0+z1+z2)\n",
+    "                y2 = int(-z0-z1-z2-z3)\n",
+    "                y = np.vstack((y0,y1,y2))\n",
+    "                if use_diff:\n",
+    "                    difference = max(z0,z1,z2,z3) - min(z0,z1,z2,z3)\n",
+    "                else:\n",
+    "                    \n",
+    "                    f, difference, rank, s = la.lstsq(points, [z0,z1,z2,z3], rcond=None) \n",
+    "                    difference = difference.astype(int)\n",
+    "                    \n",
+    "                predict = np.round(np.round(np.linalg.solve(A,y)[-1][0],1))\n",
+    "                \n",
+    "                # add on the difference by searching the dictionary\n",
+    "                # !!!!!WARNING!!!! has to change this part, eveytime you change the number of bins\n",
+    "                if difference <= bins[0]:\n",
+    "                    encoded_string, colorvalue = decode_string(encoded_string, nodes[1])\n",
+    "                    decode_matrix[i][j] = colorvalue + int(predict)\n",
+    "                elif difference <= bins[1] and difference > bins[0]:\n",
+    "                    encoded_string, colorvalue = decode_string(encoded_string, nodes[2])\n",
+    "                    decode_matrix[i][j] = colorvalue + int(predict)\n",
+    "                elif difference <= bins[2] and difference > bins[1]:\n",
+    "                    encoded_string, colorvalue = decode_string(encoded_string, nodes[3])\n",
+    "                    decode_matrix[i][j] = colorvalue + int(predict)\n",
+    "                else:\n",
+    "                    encoded_string, colorvalue = decode_string(encoded_string, nodes[4])\n",
+    "                    decode_matrix[i][j] = colorvalue + int(predict)\n",
+    "                \n",
+    "    return decode_matrix.astype(int)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 47,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def compress_rate(image_array, new_error, diff, bound, huffman_encoding_list, bins):\n",
+    "    '''\n",
+    "    This function is used to calculate the compression rate.\n",
+    "    Input:\n",
+    "    image_array      (512, 640): original_core image\n",
+    "    new_error  (512, 640): error that includes the boundary\n",
+    "    diff       (510, 638): difference of min and max of the 4 neighbors\n",
+    "    bound      (2300,): the boundary values after subtracting the very first pixel value\n",
+    "    huffman_encoding_list   (num_dic + 1,): a list of huffman coding table \n",
+    "    bins       (num_bins - 1,): a list of threshold to cut the bins\n",
+    "    \n",
+    "    Return:\n",
+    "    compression rate\n",
+    "    '''\n",
+    "    # the bits for the original image\n",
+    "    o_len = 0\n",
+    "    # the bits for the compressed image\n",
+    "    c_len = 0\n",
+    "    # initializing the varible \n",
+    "    \n",
+    "    #this was unused\n",
+    "    # im = np.reshape(image,(512, 640))\n",
+    "    \n",
+    "    real_boundary = np.hstack((image_array[0,:],image_array[-1,:],image_array[1:-1,0],image_array[1:-1,-1]))\n",
+    "    #Bryce's notes: Why are they all reshaped?\n",
+    "    original_core = image_array[1:-1,1:-1].reshape(-1)\n",
+    "    diff = diff.reshape(-1)\n",
+    "    error = new_error[1:-1,1:-1].reshape(-1)\n",
+    "    \n",
+    "    # calculate the bit for boundary\n",
+    "    for i in range(0,len(bound)):\n",
+    "        o_len += len(bin(real_boundary[i])[2:])\n",
+    "        c_len += len(huffman_encoding_list[0][str(bound[i])])\n",
+    "    \n",
+    "    # calculate the bit for the pixels inside the boundary\n",
+    "    for i in range(0,len(original_core)):\n",
+    "\n",
+    "        # for the original image\n",
+    "        o_len += len(bin(original_core[i])[2:])\n",
+    "        \n",
+    "        # check the difference and find the coresponding huffman table\n",
+    "        # !!!!!WARNING!!!! has to change this part, eveytime you change the number of bins\n",
+    "        if diff[i] <= bins[0]:\n",
+    "            c_len += len(huffman_encoding_list[1][str(int(error[i]))])\n",
+    "            \n",
+    "        elif diff[i] <= bins[1] and diff[i] > bins[0]:\n",
+    "            c_len += len(huffman_encoding_list[2][str(int(error[i]))])\n",
+    "            \n",
+    "        elif diff[i] <= bins[2] and diff[i] > bins[1]:\n",
+    "            c_len += len(huffman_encoding_list[3][str(int(error[i]))])\n",
+    "\n",
+    "        else: \n",
+    "            c_len += len(huffman_encoding_list[4][str(int(error[i]))])\n",
+    "            \n",
+    "    return c_len/o_len"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 48,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "True\n",
+      "5\n"
+     ]
+    }
+   ],
+   "source": [
+    "scenes = file_extractor()\n",
+    "images = image_extractor(scenes)\n",
+    "list_dic, image, new_error, diff, bound, predict, bins, A, nodes = huffman(images[1], 4, False)\n",
+    "encoded_string = encoder(new_error, list_dic, diff, bound, bins)\n",
+    "reconstruct_image = decoder(A, encoded_string, list_dic, bins, False,nodes)\n",
+    "print(np.allclose(image, reconstruct_image))\n",
+    "print(len(list_dic))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 49,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.4232928466796875"
+      ]
+     },
+     "execution_count": 49,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "compress_rate(image, new_error, diff, bound, list_dic, bins)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
+  },
+  "kernelspec": {
+   "display_name": "Python 3.8.10 64-bit",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.10"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

testpyfiles/SaveableEncoderDecoder.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "from matplotlib import pyplot as plt\n",
+    "from itertools import product\n",
+    "import os\n",
+    "import sys\n",
+    "from PIL import Image\n",
+    "from scipy.optimize import minimize,linprog\n",
+    "from sklearn.neighbors import KernelDensity\n",
+    "from collections import Counter\n",
+    "import numpy.linalg as la"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def file_extractor(dirname=\"images\"):\n",
+    "    files = os.listdir(dirname)\n",
+    "    scenes = []\n",
+    "    for file in files:\n",
+    "        if file == '.DS_Store':\n",
+    "            continue\n",
+    "        else:\n",
+    "            scenes.append(os.path.join(dirname, file))\n",
+    "    return scenes\n",
+    "\n",
+    "def image_extractor(scenes):\n",
+    "    image_folder = []\n",
+    "    for scene in scenes:\n",
+    "        files = os.listdir(scene)\n",
+    "        for file in files:\n",
+    "            if file[-5:] != \".tiff\" or file[-7:] == \"_6.tiff\":\n",
+    "                continue\n",
+    "            else:\n",
+    "                image_folder.append(os.path.join(scene, file))\n",
+    "    return image_folder #returns a list of file paths to .tiff files in the specified directory given in file_extractor\n",
+    "\n",
+    "def im_distribution(images, num):\n",
+    "    \"\"\"\n",
+    "    Function that extracts tiff files from specific cameras and returns a list of all\n",
+    "    the tiff files corresponding to that camera. i.e. all pictures labeled \"_7.tiff\" or otherwise\n",
+    "    specified camera numbers.\n",
+    "    \n",
+    "    Parameters:\n",
+    "        images (list): list of all tiff files, regardless of classification. This is NOT a list of directories but\n",
+    "        of specific tiff files that can be opened right away. This is the list that we iterate through and \n",
+    "        divide.\n",
+    "        \n",
+    "        num (str): a string designation for the camera number that we want to extract i.e. \"14\" for double digits\n",
+    "        of \"_1\" for single digits.\n",
+    "        \n",
+    "    Returns:\n",
+    "        tiff (list): A list of tiff files that have the specified designation from num. They are the files extracted\n",
+    "        from the 'images' list that correspond to the given num.\n",
+    "    \"\"\"\n",
+    "    tiff = []\n",
+    "    for im in images:\n",
+    "        if im[-7:-5] == num:\n",
+    "            tiff.append(im)\n",
+    "    return tiff"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def predict_pix(tiff_image_path, difference = True):\n",
+    "    \"\"\"\n",
+    "    This function predict the pixel values excluding the boundary.\n",
+    "    Using the 4 neighbor pixel values and MSE to predict the next pixel value\n",
+    "    (-1,1) (0,1) (1,1)  => relative position of the 4 other given values\n",
+    "    (-1,0) (0,0)        => (0,0) is the one we want to predict\n",
+    "    take the derivative of mean square error to solve for the system of equation \n",
+    "    A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]])\n",
+    "    A @ [a, b, c] = [-z0+z2-z3, z0+z1+z2, -z0-z1-z2-z3] where z0 = (-1,1), z1 = (0,1), z2 = (1,1), z3 = (-1,0)\n",
+    "    and the predicted pixel value is c.\n",
+    "    \n",
+    "    Input:\n",
+    "    tiff_image_path (string): path to the tiff file\n",
+    "    \n",
+    "    Return:\n",
+    "    image   ndarray(512 X 640): original image \n",
+    "    predict ndarray(325380,): predicted image excluding the boundary\n",
+    "    diff.   ndarray(325380,): IF difference = TRUE, difference between the min and max of four neighbors exclude the boundary\n",
+    "                            ELSE: the residuals of the four nearest pixels to a fitted hyperplane\n",
+    "    error   ndarray(325380,): difference between the original image and predicted image\n",
+    "    A       ndarray(3 X 3): system of equation\n",
+    "    \"\"\"\n",
+    "    image_obj = Image.open(tiff_image_path)    #Open the image and read it as an Image object\n",
+    "    image_array = np.array(image_obj)[1:,:].astype(int)    #Convert to an array, leaving out the first row because the first row is just housekeeping data\n",
+    "    # image_array = image_array.astype(int) \n",
+    "    A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation\n",
+    "    # where z0 = (-1,1), z1 = (0,1), z2 = (1,1), z3 = (-1,0)\n",
+    "    z0 = image_array[0:-2,0:-2]   # get all the first pixel for the entire image\n",
+    "    z1 = image_array[0:-2,1:-1]   # get all the second pixel for the entire image\n",
+    "    z2 = image_array[0:-2,2::]    # get all the third pixel for the entire image\n",
+    "    z3 = image_array[1:-1,0:-2]   # get all the forth pixel for the entire image\n",
+    "    \n",
+    "    # calculate the out put of the system of equation\n",
+    "    y0 = np.ravel(-z0+z2-z3)\n",
+    "    y1 = np.ravel(z0+z1+z2)\n",
+    "    y2 = np.ravel(-z0-z1-z2-z3)\n",
+    "    y = np.vstack((y0,y1,y2))\n",
+    "    \n",
+    "    # use numpy solver to solve the system of equations all at once\n",
+    "    #predict = np.floor(np.linalg.solve(A,y)[-1])\n",
+    "    predict = np.round(np.round((np.linalg.solve(A,y)[-1]),1))\n",
+    "    \n",
+    "    #Matrix system of points that will be used to solve the least squares fitting hyperplane\n",
+    "    points = np.array([[-1,-1,1], [-1,0,1], [-1,1,1], [0,-1,1]])\n",
+    "    \n",
+    "    # flatten the neighbor pixlels and stack them together\n",
+    "    z0 = np.ravel(z0)\n",
+    "    z1 = np.ravel(z1)\n",
+    "    z2 = np.ravel(z2)\n",
+    "    z3 = np.ravel(z3)\n",
+    "    neighbor = np.vstack((z0,z1,z2,z3)).T\n",
+    "    \n",
+    "    if difference:\n",
+    "        # calculate the difference\n",
+    "        diff = np.max(neighbor,axis = 1) - np.min(neighbor, axis=1)\n",
+    "    \n",
+    "    else:\n",
+    "        #Compute the best fitting hyperplane using least squares\n",
+    "        #The res is the residuals of the four points used to fit the hyperplane (summed distance of each of the \n",
+    "        #points to the hyperplane), it is a measure of gradient\n",
+    "        f, diff, rank, s = la.lstsq(points, neighbor.T, rcond=None)\n",
+    "        diff = diff.astype(int)\n",
+    "    \n",
+    "    # calculate the error\n",
+    "    error = np.ravel(image_array[1:-1,1:-1])-predict\n",
+    "    \n",
+    "    return image_array, predict, diff, error, A"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\"\"\"\n",
+    "this huffman encoding code is found online\n",
+    "https://favtutor.com/blogs/huffman-coding\n",
+    "\"\"\"\n",
+    "\n",
+    "class NodeTree(object):\n",
+    "    def __init__(self, left=None, right=None):\n",
+    "        self.left = left\n",
+    "        self.right = right\n",
+    "\n",
+    "    def children(self):\n",
+    "        return self.left, self.right\n",
+    "\n",
+    "    def __str__(self):\n",
+    "        return self.left, self.right\n",
+    "\n",
+    "\n",
+    "def huffman_code_tree(node, binString=''):\n",
+    "    '''\n",
+    "    Function to find Huffman Code\n",
+    "    '''\n",
+    "    if type(node) is str:\n",
+    "        return {node: binString}\n",
+    "    (l, r) = node.children()\n",
+    "    d = dict()\n",
+    "    d.update(huffman_code_tree(l, binString + '0'))\n",
+    "    d.update(huffman_code_tree(r, binString + '1'))\n",
+    "    return d\n",
+    "\n",
+    "\n",
+    "def make_tree(nodes):\n",
+    "    '''\n",
+    "    Function to make tree\n",
+    "    :param nodes: Nodes\n",
+    "    :return: Root of the tree\n",
+    "    '''\n",
+    "    while len(nodes) > 1:\n",
+    "        (key1, c1) = nodes[-1]\n",
+    "        (key2, c2) = nodes[-2]\n",
+    "        nodes = nodes[:-2]\n",
+    "        node = NodeTree(key1, key2)\n",
+    "        nodes.append((node, c1 + c2))\n",
+    "        #reverse True, decending order\n",
+    "\n",
+    "        #There is a huge memory leak here, no idea how or why\n",
+    "        nodes = sorted(nodes, key=lambda x: x[1], reverse=True)\n",
+    "    return nodes[0][0]\n",
+    "def decode_string(huffman_string, the_keys, the_values):\n",
+    "    for i in range(len(huffman_string)):\n",
+    "        try:\n",
+    "            return (int(the_keys[the_values.index(huffman_string[:i+1])]),huffman_string[i+1:])\n",
+    "        except:\n",
+    "            pass\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def huffman(tiff_image_path, num_bins=4, difference = True):\n",
+    "    \"\"\"\n",
+    "    This function is used to encode the error based on the difference\n",
+    "    and split the difference into different bins\n",
+    "    \n",
+    "    Input:\n",
+    "    tiff_image_path     (string): path to the tiff file\n",
+    "    num_bins            (int): number of bins\n",
+    "    \n",
+    "    Return:\n",
+    "    huffman_encoding_list  list    (num_bins + 1): a list of dictionary\n",
+    "    image_array            ndarray (512, 640): original image\n",
+    "    new_error              ndarray (512, 640): error that includes the boundary\n",
+    "    diff                   ndarray (510, 638): difference of min and max of the 4 neighbors\n",
+    "    boundary               ndarray (2300,): the boundary values after subtracting the very first pixel value\n",
+    "    predict                ndarray (325380,): the list of predicted values\n",
+    "    bins                   list    (num_bins - 1,): a list of threshold to cut the bins\n",
+    "    A                      ndarray (3 X 3): system of equation\n",
+    "    \n",
+    "    \"\"\"\n",
+    "    # get the image_array, etc\n",
+    "    image_array, predict, diff, error, A = predict_pix(tiff_image_path, difference)\n",
+    "    \n",
+    "    # calculate the number of points that will go in each bin\n",
+    "    data_points_per_bin = diff.size // num_bins\n",
+    "\n",
+    "    # sort the difference and create the bins\n",
+    "    sorted_diff = np.sort(diff.copy())\n",
+    "    bins = [sorted_diff[i*data_points_per_bin] for i in range(1,num_bins)]\n",
+    "    \n",
+    "    # get the boundary \n",
+    "    boundary = np.hstack((image_array[0,:],image_array[-1,:],image_array[1:-1,0],image_array[1:-1,-1]))\n",
+    "    \n",
+    "    # take the difference of the boundary with the very first pixel\n",
+    "    boundary = boundary - image_array[0,0]\n",
+    "    \n",
+    "    #boundary is 1dim, so boundary[0] is just the first element\n",
+    "    boundary[0] = image_array[0,0]\n",
+    "    \n",
+    "    # huffman encode the boundary\n",
+    "    bound_vals_as_string = [str(i) for i in boundary]\n",
+    "    freq = dict(Counter(bound_vals_as_string))\n",
+    "    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)\n",
+    "    node = make_tree(freq)\n",
+    "    huffman_encoding_dict = huffman_code_tree(node)\n",
+    "    \n",
+    "    # create a list of huffman table\n",
+    "    huffman_encoding_list = [huffman_encoding_dict]\n",
+    "    n = len(bins)\n",
+    "    \n",
+    "    # loop through different bins\n",
+    "    for i in range (0,n):\n",
+    "        # the first bin\n",
+    "        if i == 0 :\n",
+    "            # get the point within the bin and huffman huffman_encoding_dict\n",
+    "            mask = diff <= bins[i]\n",
+    "            line_as_string = [str(i) for i in error[mask].astype(int)]\n",
+    "            freq = dict(Counter(line_as_string))\n",
+    "            freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)\n",
+    "            node = make_tree(freq)\n",
+    "            huffman_encoding_dict = huffman_code_tree(node)\n",
+    "            huffman_encoding_list.append(huffman_encoding_dict)\n",
+    "            \n",
+    "        # the middle bins\n",
+    "        else:\n",
+    "            # get the point within the bin and huffman huffman_encoding_dict\n",
+    "            mask = diff > bins[i-1]\n",
+    "            new_error = error[mask]\n",
+    "            mask2 = diff[mask] <= bins[i]\n",
+    "            line_as_string = [str(i) for i in new_error[mask2].astype(int)]\n",
+    "            freq = dict(Counter(line_as_string))\n",
+    "            freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)\n",
+    "            node = make_tree(freq)\n",
+    "            huffman_encoding_dict = huffman_code_tree(node)\n",
+    "            huffman_encoding_list.append(huffman_encoding_dict)\n",
+    "            \n",
+    "    # the last bin       \n",
+    "    # get the point within the bin and huffman huffman_encoding_dict\n",
+    "    mask = diff > bins[-1]\n",
+    "    line_as_string = [str(i) for i in error[mask].astype(int)]\n",
+    "    freq = dict(Counter(line_as_string))\n",
+    "    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)\n",
+    "    node = make_tree(freq)\n",
+    "    huffman_encoding_dict = huffman_code_tree(node)\n",
+    "    huffman_encoding_list.append(huffman_encoding_dict)\n",
+    "\n",
+    "    # create a error matrix that includes the boundary (used in encoding matrix)\n",
+    "    new_error = np.copy(image_array)\n",
+    "    new_error[1:-1,1:-1] = np.reshape(error,(510, 638))\n",
+    "    keep = new_error[0,0]\n",
+    "    new_error[0,:] = new_error[0,:] - keep\n",
+    "    new_error[-1,:] = new_error[-1,:] - keep\n",
+    "    new_error[1:-1,0] = new_error[1:-1,0] - keep\n",
+    "    new_error[1:-1,-1] = new_error[1:-1,-1] - keep\n",
+    "    new_error[0,0] = keep\n",
+    "    # huffman_encoding_list = list(set(huffman_encoding_list))\n",
+    "    diff = np.reshape(diff,(510,638))\n",
+    "    # return the huffman dictionary\n",
+    "\n",
+    "    return huffman_encoding_list, image_array, new_error, diff, boundary, predict, bins, A\n",
+    " \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def encoder(error, list_dic, diff, bound, bins):\n",
+    "    \"\"\"\n",
+    "    This function encode the matrix with huffman coding tables\n",
+    "    \n",
+    "    Input:\n",
+    "    error     (512, 640): a matrix with all the errors\n",
+    "    list_dic  (num_dic + 1,): a list of huffman coding table \n",
+    "    bound     (2300,): the boundary values after subtracting the very first pixel value\n",
+    "    bins       (num_bins - 1,): a list of threshold to cut the bins\n",
+    "    \n",
+    "    Return:\n",
+    "    encoded   (512, 640): encoded matrix\n",
+    "    \"\"\"\n",
+    "    returnable_encode = \"\"\n",
+    "    # copy the error matrix (including the boundary)\n",
+    "    encoded = np.copy(error).astype(int).astype(str).astype(object)\n",
+    "    #diff = np.reshape(diff,(510,638))\n",
+    "    # loop through all the pixel to encode\n",
+    "    for i in range(encoded.shape[0]):\n",
+    "        for j in range(encoded.shape[1]):\n",
+    "            if i == 0 or i == encoded.shape[0]-1 or j == 0 or j == encoded.shape[1]-1:\n",
+    "                returnable_encode += list_dic[0][encoded[i][j]]\n",
+    "            elif diff[i-1][j-1] <= bins[0]:\n",
+    "                returnable_encode += list_dic[1][encoded[i][j]]\n",
+    "            elif diff[i-1][j-1] <= bins[1] and diff[i-1][j-1] > bins[0]:\n",
+    "                returnable_encode +=list_dic[2][encoded[i][j]]\n",
+    "            elif diff[i-1][j-1] <= bins[2] and diff[i-1][j-1] > bins[1]:\n",
+    "                returnable_encode +=list_dic[3][encoded[i][j]]\n",
+    "            else:\n",
+    "                returnable_encode += list_dic[4][encoded[i][j]]\n",
+    "\n",
+    "    return returnable_encode"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def decoder(A, encoded_string, list_dic, bins, use_diff):\n",
+    "    \"\"\"\n",
+    "    This function decodes the encoded_matrix.\n",
+    "    Input:\n",
+    "    A               (3 X 3): system of equation\n",
+    "    list_dic        (num_dic + 1,): a list of huffman coding table \n",
+    "    encoded_matrix  (512, 640): encoded matrix\n",
+    "    bins            (num_bins - 1,): a list of threshold to cut the bins\n",
+    "    \n",
+    "    Return:\n",
+    "    decode_matrix   (512, 640): decoded matrix\n",
+    "    \"\"\"\n",
+    "    # change the dictionary back to list\n",
+    "    # !!!!!WARNING!!!! has to change this part, everytime you change the number of bins\n",
+    "    the_keys0 = list(list_dic[0].keys())\n",
+    "    the_values0 = list(list_dic[0].values())\n",
+    "    \n",
+    "    the_keys1 = list(list_dic[1].keys())\n",
+    "    the_values1 = list(list_dic[1].values())\n",
+    "    \n",
+    "    the_keys2 = list(list_dic[2].keys())\n",
+    "    the_values2 = list(list_dic[2].values())\n",
+    "    \n",
+    "    the_keys3 = list(list_dic[3].keys())\n",
+    "    the_values3 = list(list_dic[3].values())\n",
+    "    \n",
+    "    the_keys4 = list(list_dic[4].keys())\n",
+    "    the_values4 = list(list_dic[4].values())\n",
+    "    \n",
+    "    #Matrix system of points that will be used to solve the least squares fitting hyperplane\n",
+    "    points = np.array([[-1,-1,1], [-1,0,1], [-1,1,1], [0,-1,1]])\n",
+    "    \n",
+    "    decode_matrix = np.zeros((512,640))\n",
+    "    # loop through all the element in the matrix\n",
+    "    for i in range(decode_matrix.shape[0]):\n",
+    "        for j in range(decode_matrix.shape[1]):\n",
+    "            # if it's the very first pixel on the image\n",
+    "            if i == 0 and j == 0:\n",
+    "                colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys0, the_values=the_values0)\n",
+    "                decode_matrix[i][j] = colorvalue\n",
+    "                \n",
+    "            # if it's on the boundary (any of the 4 edges)\n",
+    "            elif i == 0 or i == decode_matrix.shape[0]-1 or j == 0 or j == decode_matrix.shape[1]-1:\n",
+    "                colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys0, the_values=the_values0)\n",
+    "                decode_matrix[i][j] = colorvalue + decode_matrix[0][0]\n",
+    "            # if not the boundary\n",
+    "            else:\n",
+    "                # predict the image with the known pixel value\n",
+    "                z0 = decode_matrix[i-1][j-1]\n",
+    "                z1 = decode_matrix[i-1][j]\n",
+    "                z2 = decode_matrix[i-1][j+1]\n",
+    "                z3 = decode_matrix[i][j-1]\n",
+    "                y0 = int(-z0+z2-z3)\n",
+    "                y1 = int(z0+z1+z2)\n",
+    "                y2 = int(-z0-z1-z2-z3)\n",
+    "                y = np.vstack((y0,y1,y2))\n",
+    "                if use_diff:\n",
+    "                    difference = max(z0,z1,z2,z3) - min(z0,z1,z2,z3)\n",
+    "                else:\n",
+    "                    \n",
+    "                    f, difference, rank, s = la.lstsq(points, [z0,z1,z2,z3], rcond=None) \n",
+    "                    difference = difference.astype(int)\n",
+    "                    \n",
+    "                predict = np.round(np.round(np.linalg.solve(A,y)[-1][0],1))\n",
+    "                \n",
+    "                # add on the difference by searching the dictionary\n",
+    "                # !!!!!WARNING!!!! has to change this part, eveytime you change the number of bins\n",
+    "                if difference <= bins[0]:\n",
+    "                    colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys1, the_values=the_values1)\n",
+    "                    decode_matrix[i][j] = colorvalue + int(predict)\n",
+    "                elif difference <= bins[1] and difference > bins[0]:\n",
+    "                    colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys2, the_values=the_values2)\n",
+    "                    decode_matrix[i][j] = colorvalue + int(predict)\n",
+    "                elif difference <= bins[2] and difference > bins[1]:\n",
+    "                    colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys3, the_values=the_values3)\n",
+    "                    decode_matrix[i][j] = colorvalue + int(predict)\n",
+    "                else:\n",
+    "                    colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys4, the_values=the_values4)\n",
+    "                    decode_matrix[i][j] = colorvalue + int(predict)\n",
+    "\n",
+    "    return decode_matrix.astype(int)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def compress_rate(image_array, new_error, diff, bound, huffman_encoding_list, bins):\n",
+    "    '''\n",
+    "    This function is used to calculate the compression rate.\n",
+    "    Input:\n",
+    "    image_array      (512, 640): original_core image\n",
+    "    new_error  (512, 640): error that includes the boundary\n",
+    "    diff       (510, 638): difference of min and max of the 4 neighbors\n",
+    "    bound      (2300,): the boundary values after subtracting the very first pixel value\n",
+    "    huffman_encoding_list   (num_dic + 1,): a list of huffman coding table \n",
+    "    bins       (num_bins - 1,): a list of threshold to cut the bins\n",
+    "    \n",
+    "    Return:\n",
+    "    compression rate\n",
+    "    '''\n",
+    "    # the bits for the original image\n",
+    "    o_len = 0\n",
+    "    # the bits for the compressed image\n",
+    "    c_len = 0\n",
+    "    # initializing the varible \n",
+    "    \n",
+    "    #this was unused\n",
+    "    # im = np.reshape(image,(512, 640))\n",
+    "    \n",
+    "    real_boundary = np.hstack((image_array[0,:],image_array[-1,:],image_array[1:-1,0],image_array[1:-1,-1]))\n",
+    "    #Bryce's notes: Why are they all reshaped?\n",
+    "    original_core = image_array[1:-1,1:-1].reshape(-1)\n",
+    "    diff = diff.reshape(-1)\n",
+    "    error = new_error[1:-1,1:-1].reshape(-1)\n",
+    "    \n",
+    "    # calculate the bit for boundary\n",
+    "    for i in range(0,len(bound)):\n",
+    "        o_len += len(bin(real_boundary[i])[2:])\n",
+    "        c_len += len(huffman_encoding_list[0][str(bound[i])])\n",
+    "    \n",
+    "    # calculate the bit for the pixels inside the boundary\n",
+    "    for i in range(0,len(original_core)):\n",
+    "\n",
+    "        # for the original image\n",
+    "        o_len += len(bin(original_core[i])[2:])\n",
+    "        \n",
+    "        # check the difference and find the coresponding huffman table\n",
+    "        # !!!!!WARNING!!!! has to change this part, eveytime you change the number of bins\n",
+    "        if diff[i] <= bins[0]:\n",
+    "            c_len += len(huffman_encoding_list[1][str(int(error[i]))])\n",
+    "            \n",
+    "        elif diff[i] <= bins[1] and diff[i] > bins[0]:\n",
+    "            c_len += len(huffman_encoding_list[2][str(int(error[i]))])\n",
+    "            \n",
+    "        elif diff[i] <= bins[2] and diff[i] > bins[1]:\n",
+    "            c_len += len(huffman_encoding_list[3][str(int(error[i]))])\n",
+    "\n",
+    "        else: \n",
+    "            c_len += len(huffman_encoding_list[4][str(int(error[i]))])\n",
+    "            \n",
+    "    return c_len/o_len"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "True\n",
+      "5\n"
+     ]
+    }
+   ],
+   "source": [
+    "scenes = file_extractor()\n",
+    "images = image_extractor(scenes)\n",
+    "list_dic, image, new_error, diff, bound, predict, bins, A = huffman(images[2], 4, False)\n",
+    "encoded_string = encoder(new_error, list_dic, diff, bound, bins)\n",
+    "reconstruct_image = decoder(A, encoded_string, list_dic, bins, False)\n",
+    "print(np.allclose(image, reconstruct_image))\n",
+    "print(len(list_dic))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.4246848551432292"
+      ]
+     },
+     "execution_count": 30,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "compress_rate(image, new_error, diff, bound, list_dic, bins)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2087460\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(sys.getsizeof(encoded_string))\n",
+    "with open(\"MatrixNowString.txt\", 'wb') as f:\n",
+    "    f.write(bytearray(encoded_string, encoding = 'utf8'))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "657197\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(os.path.getsize('images/1626032610_393963/1626032610_393963_0.tiff'))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2087411\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(os.path.getsize('MatrixNowString.txt'))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def bitstring_to_bytes(s):\n",
+    "    v = int(s, 2)\n",
+    "    b = bytearray()\n",
+    "    while v:\n",
+    "        b.append(v & 0xff)\n",
+    "        v >>= 8\n",
+    "    return bytes(b[::-1])\n",
+    "\n",
+    "s = \"0110100001101001\"\n",
+    "inletters = bitstring_to_bytes(encoded_string)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2087460\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(sys.getsizeof(encoded_string))\n",
+    "with open(\"MatrixNowString.txt\", 'w') as f:\n",
+    "    f.write(inletters.decode(\"ISO-8859-1\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.607930346608399"
+      ]
+     },
+     "execution_count": 36,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "(os.path.getsize('MatrixNowString.txt'))/os.path.getsize('images/1626032610_393963/1626032610_393963_0.tiff')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "\u001b[1;32m/home/bryce/git/master/SaveableEncoderDecoder.ipynb Cell 18'\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      <a href='vscode-notebook-cell:/home/bryce/git/master/SaveableEncoderDecoder.ipynb#ch0000017?line=1'>2</a>\u001b[0m \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(\u001b[39m200\u001b[39m):\n\u001b[1;32m      <a href='vscode-notebook-cell:/home/bryce/git/master/SaveableEncoderDecoder.ipynb#ch0000017?line=2'>3</a>\u001b[0m     list_dic, image, new_error, diff, bound, predict, bins, A \u001b[39m=\u001b[39m huffman(images[i], \u001b[39m4\u001b[39m, \u001b[39mFalse\u001b[39;00m)\n\u001b[0;32m----> <a href='vscode-notebook-cell:/home/bryce/git/master/SaveableEncoderDecoder.ipynb#ch0000017?line=3'>4</a>\u001b[0m     encoded_string \u001b[39m=\u001b[39m encoder(new_error, list_dic, diff, bound, bins)\n\u001b[1;32m      <a href='vscode-notebook-cell:/home/bryce/git/master/SaveableEncoderDecoder.ipynb#ch0000017?line=4'>5</a>\u001b[0m     \u001b[39m# reconstruct_image = decoder(A, encoded_string, list_dic, bins, False)\u001b[39;00m\n\u001b[1;32m      <a href='vscode-notebook-cell:/home/bryce/git/master/SaveableEncoderDecoder.ipynb#ch0000017?line=5'>6</a>\u001b[0m     \u001b[39m# print(np.allclose(image, reconstruct_image))\u001b[39;00m\n\u001b[1;32m      <a href='vscode-notebook-cell:/home/bryce/git/master/SaveableEncoderDecoder.ipynb#ch0000017?line=6'>7</a>\u001b[0m     inletters \u001b[39m=\u001b[39m bitstring_to_bytes(encoded_string)\n",
+      "\u001b[1;32m/home/bryce/git/master/SaveableEncoderDecoder.ipynb Cell 6'\u001b[0m in \u001b[0;36mencoder\u001b[0;34m(error, list_dic, diff, bound, bins)\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/bryce/git/master/SaveableEncoderDecoder.ipynb#ch0000005?line=18'>19</a>\u001b[0m \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(encoded\u001b[39m.\u001b[39mshape[\u001b[39m0\u001b[39m]):\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/bryce/git/master/SaveableEncoderDecoder.ipynb#ch0000005?line=19'>20</a>\u001b[0m     \u001b[39mfor\u001b[39;00m j \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(encoded\u001b[39m.\u001b[39mshape[\u001b[39m1\u001b[39m]):\n\u001b[0;32m---> <a href='vscode-notebook-cell:/home/bryce/git/master/SaveableEncoderDecoder.ipynb#ch0000005?line=20'>21</a>\u001b[0m         \u001b[39mif\u001b[39;00m i \u001b[39m==\u001b[39;49m \u001b[39m0\u001b[39;49m \u001b[39mor\u001b[39;00m i \u001b[39m==\u001b[39m encoded\u001b[39m.\u001b[39mshape[\u001b[39m0\u001b[39m]\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m \u001b[39mor\u001b[39;00m j \u001b[39m==\u001b[39m \u001b[39m0\u001b[39m \u001b[39mor\u001b[39;00m j \u001b[39m==\u001b[39m encoded\u001b[39m.\u001b[39mshape[\u001b[39m1\u001b[39m]\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m:\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/bryce/git/master/SaveableEncoderDecoder.ipynb#ch0000005?line=21'>22</a>\u001b[0m             returnable_encode \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m list_dic[\u001b[39m0\u001b[39m][encoded[i][j]]\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/bryce/git/master/SaveableEncoderDecoder.ipynb#ch0000005?line=22'>23</a>\u001b[0m         \u001b[39melif\u001b[39;00m diff[i\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m][j\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m] \u001b[39m<\u001b[39m\u001b[39m=\u001b[39m bins[\u001b[39m0\u001b[39m]:\n",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
+     ]
+    }
+   ],
+   "source": [
+    "file_size_ratios = []\n",
+    "for i in range(200):\n",
+    "    list_dic, image, new_error, diff, bound, predict, bins, A = huffman(images[i], 4, False)\n",
+    "    encoded_string = encoder(new_error, list_dic, diff, bound, bins)\n",
+    "    # reconstruct_image = decoder(A, encoded_string, list_dic, bins, False)\n",
+    "    # print(np.allclose(image, reconstruct_image))\n",
+    "    inletters = bitstring_to_bytes(encoded_string)\n",
+    "    with open(\"MatrixNowString.txt\", 'w') as f:\n",
+    "        f.write(inletters.decode(\"ISO-8859-1\"))\n",
+    "    file_size_ratios.append((os.path.getsize('MatrixNowString.txt'))/os.path.getsize('images/1626032610_393963/1626032610_393963_0.tiff'))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0.5976777130753791\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(np.mean(file_size_ratios))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "247\n"
+     ]
+    }
+   ],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
+  },
+  "kernelspec": {
+   "display_name": "Python 3.8.10 64-bit",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.10"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

testpyfiles/compression_benchmark.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "from matplotlib import pyplot as plt\n",
+    "from itertools import product\n",
+    "import os\n",
+    "import sys\n",
+    "from PIL import Image\n",
+    "from scipy.optimize import minimize,linprog\n",
+    "import time\n",
+    "import seaborn as sns\n",
+    "from sklearn.neighbors import KernelDensity\n",
+    "import pandas as pd\n",
+    "from collections import Counter\n",
+    "import time\n",
+    "import numpy.linalg as la"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Round 1 : Using PIL to compress the tiff file.\n",
+    "\n",
+    "Conclusion : The best that I could get using the lzw compressor from PIL was 61.5%. This is worse than the other one at 40, which is likely because it is using multiple channels instead of just one."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def file_extractor(dirname=\"images\"):\n",
+    "    files = os.listdir(dirname)\n",
+    "    scenes = []\n",
+    "    for file in files:\n",
+    "        if file == '.DS_Store':\n",
+    "            continue\n",
+    "        else:\n",
+    "            scenes.append(os.path.join(dirname, file))\n",
+    "    return scenes\n",
+    "\n",
+    "def image_extractor(scenes):\n",
+    "    image_folder = []\n",
+    "    for scene in scenes:\n",
+    "        files = os.listdir(scene)\n",
+    "        for file in files:\n",
+    "            if file[-5:] != \".tiff\" or file[-7:] == \"_6.tiff\":\n",
+    "                continue\n",
+    "            else:\n",
+    "                image_folder.append(os.path.join(scene, file))\n",
+    "    return image_folder #returns a list of file paths to .tiff files in the specified directory given in file_extractor"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "scenes = file_extractor()\n",
+    "images = image_extractor(scenes)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(640, 513)\n"
+     ]
+    }
+   ],
+   "source": [
+    "first_source = images[0]\n",
+    "picture = Image.open(first_source)\n",
+    "print(picture.size)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "picture.save(\"Compressed_Test.tiff\",compression='tiff_lzw')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "compressed_location = \"Compressed_Test.tiff\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "compressed_image = Image.open(compressed_location)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(640, 513)\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(compressed_image.size)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "True\n"
+     ]
+    }
+   ],
+   "source": [
+    "from PIL import features\n",
+    "print(features.check('libtiff'))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "size = os.path.getsize(compressed_location)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "473404\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(size)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "657197\n"
+     ]
+    }
+   ],
+   "source": [
+    "oldsize = os.path.getsize(first_source)\n",
+    "print(oldsize)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0.7203380417135197\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(size/oldsize)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from PIL import TiffTags\n",
+    "TiffTags.LIBTIFF_CORE.add(318)\n",
+    "picture.save('Compressed_Round_2.tiff', compression='tiff_lzw', tiffinfo={317: 2})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "404176\n",
+      "True\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/bryce/.local/lib/python3.8/site-packages/PIL/TiffImagePlugin.py:845: UserWarning: Truncated File Read\n",
+      "  warnings.warn(str(msg))\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(os.path.getsize('Compressed_Round_2.tiff'))\n",
+    "potentially_compressed = Image.open('Compressed_Round_2.tiff')\n",
+    "print(np.allclose(picture,potentially_compressed))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0.6149997641498668\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(404176/oldsize)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Round 2: Trying the same thing but with tifffile and doing grayscale\n",
+    "\n",
+    "Conclusion: The documentation was terrible. It advertizes compression but doesn't say how it's done, and the specifics of it. Moving on to something else, but still worth a shot."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(640, 513)\n"
+     ]
+    }
+   ],
+   "source": [
+    "scenes = file_extractor()\n",
+    "images = image_extractor(scenes)\n",
+    "first_source = images[0]\n",
+    "picture = Image.open(first_source)\n",
+    "print(picture.size)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import tifffile"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[[    2     1 54668 ... 65535 65535 65535]\n",
+      " [22275 22292 22292 ... 22280 22212 22270]\n",
+      " [22303 22301 22298 ... 22254 22248 22262]\n",
+      " ...\n",
+      " [21832 21820 21844 ... 21892 21852 21845]\n",
+      " [21843 21821 21830 ... 21870 21865 21864]\n",
+      " [21836 21829 21840 ... 21858 21857 21860]]\n"
+     ]
+    }
+   ],
+   "source": [
+    "otherpic = tifffile.imread(first_source)\n",
+    "print(otherpic)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Round 3: Trying numcompress, something that compresses numbers. Should provide an ok benchmark, and is actually documented.\n",
+    "\n",
+    "Conclusion: The headline spoofed, it said I could get over 80%, I got under 30%. So that's worth looking into. Also this algorithm doesn't look into verticle changes, just horizontal, so I don't know why they advertized it to be as good as it is. No way it could be better than PNG, which doesn't advertize anything that high."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from numcompress import compress_ndarray, decompress_ndarray"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "True\n",
+      "452529\n",
+      "328320\n"
+     ]
+    }
+   ],
+   "source": [
+    "# print(list(otherpic).dtype)\n",
+    "otherpic = tifffile.imread(first_source)\n",
+    "\n",
+    "compressed = compress_ndarray(otherpic, precision = 0)\n",
+    "# print(compressed)\n",
+    "decompressed = decompress_ndarray(compressed)\n",
+    "print(np.allclose(otherpic,decompressed))\n",
+    "print(len(compressed))\n",
+    "print(otherpic.size)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open(\"ThirdTry.txt\", 'w') as f:\n",
+    "    f.write(compressed)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "thirdsize = os.path.getsize(\"ThirdTry.txt\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0.6885743544173208\n"
+     ]
+    }
+   ],
+   "source": [
+    "print((thirdsize)/oldsize)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "452578"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "sys.getsizeof(compressed)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
+  },
+  "kernelspec": {
+   "display_name": "Python 3.8.10 64-bit",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.10"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}