kelly changes

%% Cell type:code id:8868bc30 tags:

%% Cell type:code id:8868bc30 tags:

``` python

``` python

import numpy as np

import numpy as np

from matplotlib import pyplot as plt

from matplotlib import pyplot as plt

from itertools import product

from itertools import product

import os

import os

import sys

import sys

from PIL import Image

from PIL import Image

from scipy.optimize import minimize,linprog

from scipy.optimize import minimize,linprog

import time

import time

import seaborn as sns

import seaborn as sns

from sklearn.neighbors import KernelDensity

from sklearn.neighbors import KernelDensity

import pandas as pd

import pandas as pd

from collections import Counter

from collections import Counter

import time

import time

```

```

%% Cell type:code id:76317b02 tags:

%% Cell type:code id:0f944705 tags:

``` python

``` python

def file_extractor(dirname="images"):

def file_extractor(dirname="images"):

    files = os.listdir(dirname)

    files = os.listdir(dirname)

    scenes = []

    scenes = []

    for file in files:

    for file in files:

        scenes.append(os.path.join(dirname, file))

        if file == '.DS_Store':

            continue

        else:

            scenes.append(os.path.join(dirname, file))

    return scenes

    return scenes

def image_extractor(scenes):

def image_extractor(scenes):

    image_folder = []

    image_folder = []

    for scene in scenes:

    for scene in scenes:

        files = os.listdir(scene)

        files = os.listdir(scene)

        for file in files:

        for file in files:

            image_folder.append(os.path.join(scene, file))

            #if file[-4:] == ".jp4" or file[-7:] == "_6.tiff":

            if file[-5:] != ".tiff" or file[-7:] == "_6.tiff":

                continue

            else:

                image_folder.append(os.path.join(scene, file))

    '''print(image_folder)

    images = []

    images = []

    for folder in image_folder:

    for folder in image_folder:

        ims = os.listdir(folder)

        ims = os.listdir(folder)

        for im in ims:

        for im in ims:

            if im[-4:] == ".jp4" or im[-7:] == "_6.tiff":

            if im[-4:] == ".jp4" or im[-7:] == "_6.tiff":

                continue

                continue

            else:

            else:

                images.append(os.path.join(folder, im))

                images.append(os.path.join(folder, im))'''

    return images #returns a list of file paths to .tiff files in the specified directory given in file_extractor

    return image_folder #returns a list of file paths to .tiff files in the specified directory given in file_extractor

def im_distribution(images, num):

def im_distribution(images, num):

    """

    """

    Function that extracts tiff files from specific cameras and returns a list of all

    Function that extracts tiff files from specific cameras and returns a list of all

    the tiff files corresponding to that camera. i.e. all pictures labeled "_7.tiff" or otherwise

    the tiff files corresponding to that camera. i.e. all pictures labeled "_7.tiff" or otherwise

    specified camera numbers.

    specified camera numbers.

    Parameters:

    Parameters:

        images (list): list of all tiff files, regardless of classification. This is NOT a list of directories but

        images (list): list of all tiff files, regardless of classification. This is NOT a list of directories but

        of specific tiff files that can be opened right away. This is the list that we iterate through and

        of specific tiff files that can be opened right away. This is the list that we iterate through and

        divide.

        divide.

        num (str): a string designation for the camera number that we want to extract i.e. "14" for double digits

        num (str): a string designation for the camera number that we want to extract i.e. "14" for double digits

        of "_1" for single digits.

        of "_1" for single digits.

    Returns:

    Returns:

        tiff (list): A list of tiff files that have the specified designation from num. They are the files extracted

        tiff (list): A list of tiff files that have the specified designation from num. They are the files extracted

        from the 'images' list that correspond to the given num.

        from the 'images' list that correspond to the given num.

    """

    """

    tiff = []

    tiff = []

    for im in images:

    for im in images:

        if im[-7:-5] == num:

        if im[-7:-5] == num:

            tiff.append(im)

            tiff.append(im)

    return tiff

    return tiff

```

```

%% Cell type:code id:be1ff8a1 tags:

%% Cell type:code id:b18d5e38 tags:

``` python

``` python

def plot_hist(tiff_list):

def plot_hist(tiff_list):

    """

    """

    This function is the leftovers from the first attempt to plot histograms.

    This function is the leftovers from the first attempt to plot histograms.

    As it stands it needs some work in order to function again. We will

    As it stands it needs some work in order to function again. We will

    fix this later. 1/25/22

    fix this later. 1/25/22

    """

    """

    image = tiff_list

    image = tiff_list

    image = Image.open(image)    #Open the image and read it as an Image object

    image = Image.open(image)    #Open the image and read it as an Image object

    image = np.array(image)[1:,:]    #Convert to an array, leaving out the first row because the first row is just housekeeping data

    image = np.array(image)[1:,:]    #Convert to an array, leaving out the first row because the first row is just housekeeping data

    image = image.astype(int)

    image = image.astype(int)

    A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation

    A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation

    z0 = image[0:-2,0:-2]   # get all the first pixel for the entire image

    z0 = image[0:-2,0:-2]   # get all the first pixel for the entire image

    z1 = image[0:-2,1:-1]   # get all the second pixel for the entire image

    z1 = image[0:-2,1:-1]   # get all the second pixel for the entire image

    z2 = image[0:-2,2::]    # get all the third pixel for the entire image

    z2 = image[0:-2,2::]    # get all the third pixel for the entire image

    z3 = image[1:-1,0:-2]   # get all the forth pixel for the entire image

    z3 = image[1:-1,0:-2]   # get all the forth pixel for the entire image

    # calculate the out put of the system of equation

    # calculate the out put of the system of equation

    y0 = np.ravel(-z0+z2-z3)

    y0 = np.ravel(-z0+z2-z3)

    y1 = np.ravel(z0+z1+z2)

    y1 = np.ravel(z0+z1+z2)

    y2 = np.ravel(-z0-z1-z2-z3)

    y2 = np.ravel(-z0-z1-z2-z3)

    y = np.vstack((y0,y1,y2))

    y = np.vstack((y0,y1,y2))

    # use numpy solver to solve the system of equations all at once

    # use numpy solver to solve the system of equations all at once

    #predict = np.floor(np.linalg.solve(A,y)[-1])

    #predict = np.floor(np.linalg.solve(A,y)[-1])

    predict = np.round(np.round((np.linalg.solve(A,y)[-1]),1))

    predict = np.round(np.round((np.linalg.solve(A,y)[-1]),1))

    # flatten the neighbor pixlels and stack them together

    # flatten the neighbor pixlels and stack them together

    z0 = np.ravel(z0)

    z0 = np.ravel(z0)

    z1 = np.ravel(z1)

    z1 = np.ravel(z1)

    z2 = np.ravel(z2)

    z2 = np.ravel(z2)

    z3 = np.ravel(z3)

    z3 = np.ravel(z3)

    neighbor = np.vstack((z0,z1,z2,z3)).T

    neighbor = np.vstack((z0,z1,z2,z3)).T

    # calculate the difference

    # calculate the difference

    diff = np.max(neighbor,axis = 1) - np.min(neighbor, axis=1)

    diff = np.max(neighbor,axis = 1) - np.min(neighbor, axis=1)

    # flatten the image to a vector

    # flatten the image to a vector

    image = np.ravel(image[1:-1,1:-1])

    image = np.ravel(image[1:-1,1:-1])

    error = image-predict

    error = image-predict

    return image, predict, diff, error, A

    return image, predict, diff, error, A

```

```

%% Cell type:code id:8483903e tags:

%% Cell type:code id:35d4f6a0 tags:

``` python

``` python

class NodeTree(object):

class NodeTree(object):

    def __init__(self, left=None, right=None):

    def __init__(self, left=None, right=None):

        self.left = left

        self.left = left

        self.right = right

        self.right = right

    def children(self):

    def children(self):

        return self.left, self.right

        return self.left, self.right

    def __str__(self):

    def __str__(self):

        return self.left, self.right

        return self.left, self.right

def huffman_code_tree(node, binString=''):

def huffman_code_tree(node, binString=''):

    '''

    '''

    Function to find Huffman Code

    Function to find Huffman Code

    '''

    '''

    if type(node) is str:

    if type(node) is str:

        return {node: binString}

        return {node: binString}

    (l, r) = node.children()

    (l, r) = node.children()

    d = dict()

    d = dict()

    d.update(huffman_code_tree(l, binString + '0'))

    d.update(huffman_code_tree(l, binString + '0'))

    d.update(huffman_code_tree(r, binString + '1'))

    d.update(huffman_code_tree(r, binString + '1'))

    return d

    return d

def make_tree(nodes):

def make_tree(nodes):

    '''

    '''

    Function to make tree

    Function to make tree

    :param nodes: Nodes

    :param nodes: Nodes

    :return: Root of the tree

    :return: Root of the tree

    '''

    '''

    while len(nodes) > 1:

    while len(nodes) > 1:

        (key1, c1) = nodes[-1]

        (key1, c1) = nodes[-1]

        (key2, c2) = nodes[-2]

        (key2, c2) = nodes[-2]

        nodes = nodes[:-2]

        nodes = nodes[:-2]

        node = NodeTree(key1, key2)

        node = NodeTree(key1, key2)

        nodes.append((node, c1 + c2))

        nodes.append((node, c1 + c2))

        nodes = sorted(nodes, key=lambda x: x[1], reverse=True)

        nodes = sorted(nodes, key=lambda x: x[1], reverse=True)

    return nodes[0][0]

    return nodes[0][0]

```

```

%% Cell type:markdown id:c7104fbf tags:

%% Cell type:markdown id:c7104fbf tags:

### Huffman without dividing into bins

### Huffman without dividing into bins

%% Cell type:code id:a43f3f1c tags:

%% Cell type:code id:c50169ed tags:

``` python

``` python

scenes = file_extractor()

images = image_extractor(scenes)

def huffman_nb(image):

def huffman_nb(image):

    origin, predict, diff, error, A = plot_hist(image)

    origin, predict, diff, error, A = plot_hist(image)

    image = Image.open(image)

    image = Image.open(image)

    image = np.array(image)[1:,:]    #Convert to an array, leaving out the first row because the first row is just housekeeping data

    image = np.array(image)[1:,:]    #Convert to an array, leaving out the first row because the first row is just housekeeping data

    image = image.astype(int)

    image = image.astype(int)

    new_error = np.copy(image)

    new_error = np.copy(image)

    new_error[1:-1,1:-1] = np.reshape(error,(510, 638))

    new_error[1:-1,1:-1] = np.reshape(error,(510, 638))

    keep = new_error[0,0]

    keep = new_error[0,0]

    new_error[0,:] = new_error[0,:] - keep

    new_error[0,:] = new_error[0,:] - keep

    new_error[-1,:] = new_error[-1,:] - keep

    new_error[-1,:] = new_error[-1,:] - keep

    new_error[1:-1,0] = new_error[1:-1,0] - keep

    new_error[1:-1,0] = new_error[1:-1,0] - keep

    new_error[1:-1,-1] = new_error[1:-1,-1] - keep

    new_error[1:-1,-1] = new_error[1:-1,-1] - keep

    new_error[0,0] = keep

    new_error[0,0] = keep

    new_error = np.ravel(new_error)

    new_error = np.ravel(new_error)

    #ab_error = np.abs(new_error)

    #ab_error = np.abs(new_error)

    #string = [str(i) for i in ab_error]

    #string = [str(i) for i in ab_error]

    string = [str(i) for i in new_error.astype(int)]

    string = [str(i) for i in new_error.astype(int)]

    freq = dict(Counter(string))

    freq = dict(Counter(string))

    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)

    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)

    node = make_tree(freq)

    node = make_tree(freq)

    encoding = huffman_code_tree(node)

    encoding = huffman_code_tree(node)

    #encoded = ["1"+encoding[str(-i)] if i < 0 else "0"+encoding[str(i)] for i in error]

    #encoded = ["1"+encoding[str(-i)] if i < 0 else "0"+encoding[str(i)] for i in error]

    # return the huffman dictionary

    # return the huffman dictionary

    return encoding, new_error, image.reshape(-1)

    return encoding, new_error, image.reshape(-1)

def compress_rate_nb(image, error, encoding):

def compress_rate_nb(image, error, encoding):

    #original = original.reshape(-1)

    original = image.reshape(-1)

    #error = error.reshape(-1)

    error = error.reshape(-1)

    o_len = 0

    o_len = 0

    c_len = 0

    c_len = 0

    for i in range(0, len(original)):

    for i in range(0, len(original)):

        o_len += len(bin(original[i])[2:])

        o_len += len(bin(original[i])[2:])

        c_len += len(encoding[str(int(error[i]))])

        c_len += len(encoding[str(int(error[i]))])

    return c_len/o_len

    return c_len/o_len

```

```

%% Cell type:markdown id:eac2f456 tags:

%% Cell type:markdown id:e34201fd tags:

### Huffman with dividing into non-uniform bins

%% Cell type:markdown id:3a3f06a5 tags:

### Huffman with dividing into uniform bins

%% Cell type:code id:14075c94 tags:

``` python

def huffman_u(image):

    origin, predict, diff, error, A = plot_hist(image)

    image = Image.open(image)

    image = np.array(image)[1:,:]    #Convert to an array, leaving out the first row because the first row is just housekeeping data

    image = image.astype(int)

    print(len(diff))

    boundary = np.hstack((image[0,:],image[-1,:],image[1:-1,0],image[1:-1,-1]))

    boundary = boundary - image[0,0]

    boundary[0] = image[0,0]

    string = [str(i) for i in boundary]

    freq = dict(Counter(string))

    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)

    node = make_tree(freq)

    encode1 = huffman_code_tree(node)

    mask = diff <= 100

    string = [str(i) for i in error[mask].astype(int)]

    freq = dict(Counter(string))

    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)

    node = make_tree(freq)

    encode2 = huffman_code_tree(node)

    mask = diff > 100

    #new_error = error[mask]

    #mask2 = diff[mask] <= 200

    #string = [str(i) for i in new_error[mask2].astype(int)]

    string = [str(i) for i in error[mask].astype(int)]

    freq = dict(Counter(string))

    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)

    node = make_tree(freq)

    encode3 = huffman_code_tree(node)

    '''mask = diff > 200

    new_error = error[mask]

    mask2 = diff[mask] <= 300

    string = [str(i) for i in new_error[mask2].astype(int)]

    freq = dict(Counter(string))

    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)

    node = make_tree(freq)

    encode4 = huffman_code_tree(node)

    mask = diff > 300

    string = [str(i) for i in error[mask].astype(int)]

    freq = dict(Counter(string))

    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)

    node = make_tree(freq)

    encode5 = huffman_code_tree(node)'''

    new_error = np.copy(image)

    new_error[1:-1,1:-1] = np.reshape(error,(510, 638))

    keep = new_error[0,0]

    new_error[0,:] = new_error[0,:] - keep

    new_error[-1,:] = new_error[-1,:] - keep

    new_error[1:-1,0] = new_error[1:-1,0] - keep

    new_error[1:-1,-1] = new_error[1:-1,-1] - keep

    new_error[0,0] = keep

    new_error = np.ravel(new_error)

    # return the huffman dictionary

    #return encode1, encode2, encode3, encode4, encode5, np.ravel(image), error, diff, boundary

    print(len(diff))

    return encode1, encode2, encode3, np.ravel(image), error, diff, boundary

#def compress_rate_u(image, error, diff, bound, encode1, encode2, encode3, encode4, encode5):

def compress_rate_u(image, error, diff, bound, encode1, encode2, encode3):

    #original = original.reshape(-1)

    #error = error.reshape(-1)

    o_len = 0

    c_len = 0

    im = np.reshape(image,(512, 640))

    real_b = np.hstack((im[0,:],im[-1,:],im[1:-1,0],im[1:-1,-1]))

    original = im[1:-1,1:-1].reshape(-1)

    for i in range(0,len(bound)):

        o_len += len(bin(real_b[i])[2:])

        c_len += len(encode1[str(bound[i])])

    for i in range(0, len(original)):

        o_len += len(bin(original[i])[2:])

        if diff[i] <= 100:

            c_len += len(encode2[str(int(error[i]))])

        if diff[i] > 100:

            c_len += len(encode3[str(int(error[i]))])

        '''if diff[i] <= 200 and diff[i] > 100:

            c_len += len(encode3[str(int(error[i]))])'''

        '''if diff[i] <= 300 and diff[i] > 200:

            c_len += len(encode4[str(int(error[i]))])

        if diff[i] > 300:

            c_len += len(encode5[str(int(error[i]))])'''

    return c_len/o_len

scenes = file_extractor()

images = image_extractor(scenes)

encode1, encode2, encode3, image, error, diff, boundary = huffman_u(images[0])

compress_rate_u(image, error, diff, boundary, encode1, encode2, encode3)

```

%% Output

    325380

    325380

    0.4432273356119792

### Huffman dividing into bins

%% Cell type:code id:207b0bd2 tags:

%% Cell type:code id:205c4731 tags:

``` python

``` python

def huffman(image):

def huffman(image):

    origin, predict, diff, error, A = plot_hist(image)

    origin, predict, diff, error, A = plot_hist(image)

    image = Image.open(image)

    image = Image.open(image)

    image = np.array(image)[1:,:]    #Convert to an array, leaving out the first row because the first row is just housekeeping data

    image = np.array(image)[1:,:]    #Convert to an array, leaving out the first row because the first row is just housekeeping data

    image = image.astype(int)

    image = image.astype(int)

    boundary = np.hstack((image[0,:],image[-1,:],image[1:-1,0],image[1:-1,-1]))

    boundary = np.hstack((image[0,:],image[-1,:],image[1:-1,0],image[1:-1,-1]))

    boundary = boundary - image[0,0]

    boundary = boundary - image[0,0]

    boundary[0] = image[0,0]

    boundary[0] = image[0,0]

    string = [str(i) for i in boundary]

    string = [str(i) for i in boundary]

    freq = dict(Counter(string))

    freq = dict(Counter(string))

    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)

    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)

    node = make_tree(freq)

    node = make_tree(freq)

    encode1 = huffman_code_tree(node)

    encode1 = huffman_code_tree(node)

    mask = diff <= 25

    mask = diff <= 25

    string = [str(i) for i in error[mask].astype(int)]

    string = [str(i) for i in error[mask].astype(int)]

    freq = dict(Counter(string))

    freq = dict(Counter(string))

    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)

    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)

    node = make_tree(freq)

    node = make_tree(freq)

    encode2 = huffman_code_tree(node)

    encode2 = huffman_code_tree(node)

    mask = diff > 25

    mask = diff > 25

    new_error = error[mask]

    new_error = error[mask]

    mask2 = diff[mask] <= 40

    mask2 = diff[mask] <= 40

    string = [str(i) for i in new_error[mask2].astype(int)]

    string = [str(i) for i in new_error[mask2].astype(int)]

    freq = dict(Counter(string))

    freq = dict(Counter(string))

    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)

    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)

    node = make_tree(freq)

    node = make_tree(freq)

    encode3 = huffman_code_tree(node)

    encode3 = huffman_code_tree(node)