more compressed files

from sklearn.neighbors import KernelDensity

from sklearn.neighbors import KernelDensity

from collections import Counter

from collections import Counter

import numpy.linalg as la

import numpy.linalg as la

from time import time

def file_extractor(dirname="images"):

def file_extractor(dirname="images"):

    files = os.listdir(dirname)

    files = os.listdir(dirname)

    # calculate the error

    # calculate the error

    error = np.ravel(image_array[1:-1,1:-1])-predict

    error = np.ravel(image_array[1:-1,1:-1])-predict

    return image_array, predict, diff, error

    return image_array, diff, error

"""

"""

this huffman encoding code is found online

this huffman encoding code is found online

            return (int(the_keys[the_values.index(huffman_string[:i+1])]),huffman_string[i+1:])

            return (int(the_keys[the_values.index(huffman_string[:i+1])]),huffman_string[i+1:])

        except:

        except:

            pass

            pass

def make_dictionary(tiff_image_path_list, num_bins=4, difference = True):

    """

    This function is used to encode the error based on the difference

    and split the difference into different bins

    Input:

    tiff_image_path     (string): path to the tiff file

    num_bins            (int): number of bins

    Return:

    huffman_encoding_list  list    (num_bins + 1): a list of dictionary

    image_array            ndarray (512, 640): original image

    new_error              ndarray (512, 640): error that includes the boundary

    diff                   ndarray (510, 638): difference of min and max of the 4 neighbors

    boundary               ndarray (2300,): the boundary values after subtracting the very first pixel value

    predict                ndarray (325380,): the list of predicted values

    bins                   list    (num_bins - 1,): a list of threshold to cut the bins

    A                      ndarray (3 X 3): system of equation

    """

    list_of_all_vals = []

    huffman_encoding_list = []

    for _ in range(num_bins+1):

        list_of_all_vals.append([])

    for _, tiff_image_path in enumerate(tiff_image_path_list):

        # get the image_array, etc

        image_array, diff, error= predict_pix(tiff_image_path, difference)

        bins = [21,32,48]

        # get the boundary 

        boundary = np.hstack((image_array[0,:],image_array[-1,:],image_array[1:-1,0],image_array[1:-1,-1]))

        # take the difference of the boundary with the very first pixel

        boundary = boundary - image_array[0,0]

        #boundary is 1dim, so boundary[0] is just the first element

        boundary[0] = image_array[0,0]

        # huffman encode the boundary

        for j in boundary:

            list_of_all_vals[0].append(str(j))

        # create a list of huffman table

        n = len(bins)

        # loop through different bins

        for k in range (0,n):

            # the first bin

            if k == 0 :

                # get the point within the bin and huffman huffman_encoding_dict

                mask = diff <= bins[k]

                for j in error[mask].astype(int):

                    list_of_all_vals[k+1].append(str(j))

            # the middle bins

            else:

                # get the point within the bin and huffman huffman_encoding_dict

                mask = diff > bins[k-1]

                new_error = error[mask]

                mask2 = diff[mask] <= bins[k]

                for j in new_error[mask2].astype(int):

                    list_of_all_vals[k+1].append(str(j))

        # the last bin       

        # get the point within the bin and huffman huffman_encoding_dict

        mask = diff > bins[-1]

        for j in error[mask].astype(int):

            list_of_all_vals[-1].append(str(j))

    for item in list_of_all_vals:

        freq = dict(Counter(item))

        freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)

        node = make_tree(freq)

        huffman_encoding_list.append(huffman_code_tree(node))

        # create a error matrix that includes the boundary (used in encoding matrix)

    new_error = np.copy(image_array)

    new_error[1:-1,1:-1] = np.reshape(error,(510, 638))

    keep = new_error[0,0]

    new_error[0,:] = new_error[0,:] - keep

    new_error[-1,:] = new_error[-1,:] - keep

    new_error[1:-1,0] = new_error[1:-1,0] - keep

    new_error[1:-1,-1] = new_error[1:-1,-1] - keep

    new_error[0,0] = keep

        # huffman_encoding_list = list(set(huffman_encoding_list))

    diff = np.reshape(diff,(510,638))

        # return the huffman dictionary

    return huffman_encoding_list,bins

def huffman(tiff_image_path, num_bins=4, difference = True):

    """

    This function is used to encode the error based on the difference

    and split the difference into different bins

    Input:

    tiff_image_path     (string): path to the tiff file

    num_bins            (int): number of bins

    Return:

    huffman_encoding_list  list    (num_bins + 1): a list of dictionary

    image_as_array         ndarray (512, 640): original image

    new_error              ndarray (512, 640): error that includes the boundary

    diff                   ndarray (510, 638): difference of min and max of the 4 neighbors

    boundary               ndarray (2300,): the boundary values after subtracting the very first pixel value

    predict                ndarray (325380,): the list of predicted values

    bins                   list    (num_bins - 1,): a list of threshold to cut the bins

    A                      ndarray (3 X 3): system of equation

    """

    # get the image_as_array, etc

    image_as_array, diff, error= predict_pix(tiff_image_path, difference)

    # calculate the number of points that will go in each bin

    # sort the difference and create the bins

    bins = [21,32,48]

    # get the boundary 

    boundary = np.hstack((image_as_array[0,:],image_as_array[-1,:],image_as_array[1:-1,0],image_as_array[1:-1,-1]))

    # take the difference of the boundary with the very first pixel

    boundary = boundary - image_as_array[0,0]

    #boundary is 1dim, so boundary[0] is just the first element

    boundary[0] = image_as_array[0,0]

    # huffman encode the boundary

    bound_vals_as_string = [str(i) for i in boundary]

    freq = dict(Counter(bound_vals_as_string))

    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)

    node = make_tree(freq)

    huffman_encoding_dict = huffman_code_tree(node)

    # create a list of huffman table

    huffman_encoding_list = [huffman_encoding_dict]

    n = len(bins)

    # loop through different bins

    for i in range (0,n):

        # the first bin

        if i == 0 :

            # get the point within the bin and huffman huffman_encoding_dict

            mask = diff <= bins[i]

            line_as_string = [str(i) for i in error[mask].astype(int)]

            freq = dict(Counter(line_as_string))

            freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)

            node = make_tree(freq)

            huffman_encoding_dict = huffman_code_tree(node)

            huffman_encoding_list.append(huffman_encoding_dict)

        # the middle bins

        else:

            # get the point within the bin and huffman huffman_encoding_dict

            mask = diff > bins[i-1]

            new_error = error[mask]

            mask2 = diff[mask] <= bins[i]

            line_as_string = [str(i) for i in new_error[mask2].astype(int)]

            freq = dict(Counter(line_as_string))

            freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)

            node = make_tree(freq)

            huffman_encoding_dict = huffman_code_tree(node)

            huffman_encoding_list.append(huffman_encoding_dict)

    # the last bin       

    # get the point within the bin and huffman huffman_encoding_dict

    mask = diff > bins[-1]

    line_as_string = [str(i) for i in error[mask].astype(int)]

    freq = dict(Counter(line_as_string))

    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)

    node = make_tree(freq)

    huffman_encoding_dict = huffman_code_tree(node)

    huffman_encoding_list.append(huffman_encoding_dict)

    # create a error matrix that includes the boundary (used in encoding matrix)

    new_error = np.copy(image_as_array)

    new_error[1:-1,1:-1] = np.reshape(error,(510, 638))

    keep = new_error[0,0]

    new_error[0,:] = new_error[0,:] - keep

    new_error[-1,:] = new_error[-1,:] - keep

    new_error[1:-1,0] = new_error[1:-1,0] - keep

    new_error[1:-1,-1] = new_error[1:-1,-1] - keep

    new_error[0,0] = keep

    # huffman_encoding_list = list(set(huffman_encoding_list))

    diff = np.reshape(diff,(510,638))

    # return the huffman dictionary

    return image_as_array, new_error, diff

def encoder(error, list_dic, diff, bins):

    """

    This function encode the matrix with huffman coding tables

    Input:

    error     (512, 640): a matrix with all the errors

    list_dic  (num_dic + 1,): a list of huffman coding table 

    bins       (num_bins - 1,): a list of threshold to cut the bins

    Return:

    encoded   (512, 640): encoded matrix

    """

    returnable_encode = ""

    # copy the error matrix (including the boundary)

    encoded = np.copy(error).astype(int).astype(str).astype(object)

    #diff = np.reshape(diff,(510,638))

    # loop through all the pixel to encode

    for i in range(encoded.shape[0]):

        for j in range(encoded.shape[1]):

            if i == 0 or i == encoded.shape[0]-1 or j == 0 or j == encoded.shape[1]-1:

                returnable_encode += list_dic[0][encoded[i][j]]

            elif diff[i-1][j-1] <= bins[0]:

                returnable_encode += list_dic[1][encoded[i][j]]

            elif diff[i-1][j-1] <= bins[1] and diff[i-1][j-1] > bins[0]:

                returnable_encode +=list_dic[2][encoded[i][j]]

            elif diff[i-1][j-1] <= bins[2] and diff[i-1][j-1] > bins[1]:

                returnable_encode +=list_dic[3][encoded[i][j]]

            else:

                returnable_encode += list_dic[4][encoded[i][j]]

    return returnable_encode

def decoder(encoded_string, list_dic, bins, use_diff):

    """

    This function decodes the encoded_matrix.

    Input:

    A               (3 X 3): system of equation

    list_dic        (num_dic + 1,): a list of huffman coding table 

    encoded_matrix  (512, 640): encoded matrix

    bins            (num_bins - 1,): a list of threshold to cut the bins

    Return:

    decode_matrix   (512, 640): decoded matrix

    """

    A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation

    # change the dictionary back to list

    # !!!!!WARNING!!!! has to change this part, everytime you change the number of bins

    the_keys0 = list(list_dic[0].keys())

    the_values0 = list(list_dic[0].values())

    the_keys1 = list(list_dic[1].keys())

    the_values1 = list(list_dic[1].values())

    the_keys2 = list(list_dic[2].keys())

    the_values2 = list(list_dic[2].values())

    the_keys3 = list(list_dic[3].keys())

    the_values3 = list(list_dic[3].values())

    the_keys4 = list(list_dic[4].keys())

    the_values4 = list(list_dic[4].values())

    #Matrix system of points that will be used to solve the least squares fitting hyperplane

    points = np.array([[-1,-1,1], [-1,0,1], [-1,1,1], [0,-1,1]])

    decode_matrix = np.zeros((512,640))

    # loop through all the element in the matrix

    for i in range(decode_matrix.shape[0]):

        for j in range(decode_matrix.shape[1]):

            # if it's the very first pixel on the image

            if i == 0 and j == 0:

                colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys0, the_values=the_values0)

                decode_matrix[i][j] = colorvalue

            # if it's on the boundary (any of the 4 edges)

            elif i == 0 or i == decode_matrix.shape[0]-1 or j == 0 or j == decode_matrix.shape[1]-1:

                colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys0, the_values=the_values0)

                decode_matrix[i][j] = colorvalue + decode_matrix[0][0]

            # if not the boundary

            else:

                # predict the image with the known pixel value

                z0 = decode_matrix[i-1][j-1]

                z1 = decode_matrix[i-1][j]

                z2 = decode_matrix[i-1][j+1]

                z3 = decode_matrix[i][j-1]

                y0 = int(-z0+z2-z3)

                y1 = int(z0+z1+z2)

                y2 = int(-z0-z1-z2-z3)

                y = np.vstack((y0,y1,y2))

                if use_diff:

                    difference = max(z0,z1,z2,z3) - min(z0,z1,z2,z3)

                else:

                    f, difference, rank, s = la.lstsq(points, [z0,z1,z2,z3], rcond=None) 

                    difference = difference.astype(int)

                predict = np.round(np.round(np.linalg.solve(A,y)[-1][0],1))

                # add on the difference by searching the dictionary

                # !!!!!WARNING!!!! has to change this part, eveytime you change the number of bins

                if difference <= bins[0]:

                    colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys1, the_values=the_values1)

                    decode_matrix[i][j] = colorvalue + int(predict)

                elif difference <= bins[1] and difference > bins[0]:

                    colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys2, the_values=the_values2)

                    decode_matrix[i][j] = colorvalue + int(predict)

                elif difference <= bins[2] and difference > bins[1]:

                    colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys3, the_values=the_values3)

                    decode_matrix[i][j] = colorvalue + int(predict)

                else:

                    colorvalue, encoded_string = decode_string(encoded_string,the_keys=the_keys4, the_values=the_values4)

                    decode_matrix[i][j] = colorvalue + int(predict)

    return decode_matrix.astype(int)

def read_from_file(filename):

    with open(filename, 'rb') as file:

        return file.read()

def bitstring_to_bytes(input_string):

    int_array = []

    length_of_string = len(input_string)

    while length_of_string >= 8:

        int_array.append(int(input_string[:8],2))

        input_string = input_string[8:]

        length_of_string = len(input_string)

    if length_of_string > 0:

        zerobuffer = ""

        for _ in range(8-length_of_string):

            zerobuffer += "0"

        int_array.append(int(input_string+zerobuffer,2))

    return bytes(int_array)

def bytes_to_bitstring(input_bytearray):

    end_string = ""

    int_array = [i for i in input_bytearray]

    for i, item in enumerate(int_array):

        end_string += (bin(item)[2:].zfill(8))

    return end_string

starttime = time()

scenes = file_extractor()

newnamesforlater = []

images = image_extractor(scenes)

list_dic, ogbins = make_dictionary(images[0:10], 4, False)

file_size_ratios = []

np.save("first_dic.npy", list_dic)

for i in range(10):

    image, new_error, diff = huffman(images[i], 4, False)

    encoded_string1 = encoder(new_error, list_dic, diff, ogbins)

    # reconstruct_image = decoder(A, encoded_string, list_dic, bins, False)

    # print(np.allclose(image, reconstruct_image))

    inletters = bitstring_to_bytes(encoded_string1)

    if images[i][:-5] == ".tiff":

        newname = images[i][:-5]

    else:

        newname = images[i][:-4]

    newnamesforlater.append(newname + "_Compressed.txt")

    with open(newname + "_Compressed.txt", 'wb') as f:

        f.write(inletters)

list_dic = np.load("first_dic.npy", allow_pickle="TRUE")

ogbins = [21,32,48]

for i,item in enumerate(newnamesforlater[0:10]):

    image, new_error, diff = huffman(images[i], 4, False)

    encoded_string2 = bytes_to_bitstring(read_from_file(item))

    reconstruct_image = decoder(encoded_string2, list_dic, ogbins, False)

    print(np.allclose(image, reconstruct_image))

print(time() - starttime)

 No newline at end of file