new_changes

%% Cell type:code id:dbef8759 tags:

%% Cell type:code id:dbef8759 tags:

``` python

``` python

import numpy as np

import numpy as np

from prediction_MSE_Scout import file_extractor, image_extractor, im_distribution

from prediction_MSE_Scout import file_extractor, image_extractor, im_distribution

from matplotlib import pyplot as plt

from matplotlib import pyplot as plt

from itertools import product

from itertools import product

import os

import os

import sys

import sys

from PIL import Image

from PIL import Image

from scipy.optimize import minimize

from scipy.optimize import minimize

from time import time

from time import time

from numpy import linalg as la

from numpy import linalg as la

from scipy.stats import gaussian_kde

from scipy.stats import gaussian_kde

import seaborn as sns

import seaborn as sns

import pywt

import pywt

from collections import Counter

from collections import Counter

```

```

%% Output

%% Output

    Average Error: 19.44221679267325

    Average Error: 19.44221679267325

    Standard Deviaiton of Mean Errors: 0.17734010606906342

    Standard Deviaiton of Mean Errors: 0.17734010606906342

    Average Difference: 51.95430150900486

    Average Difference: 51.95430150900486

    Average Time per Image for First: 0.058679431676864624

    Average Time per Image for First: 0.058679431676864624

    Std Deviation of E:  26.627504708827136

    Std Deviation of E:  26.627504708827136

    Normal bits:  15

    Normal bits:  15

    Encoded Bits:  6.677845333316752

    Encoded Bits:  6.677845333316752

    (258, 322)

    (258, 322)

%% Cell type:code id:9ed20f84 tags:

%% Cell type:code id:9ed20f84 tags:

``` python

``` python

def plot_hist(tiff_list, i=0):

def plot_hist(tiff_list, i=0):

    """

    """

    This function is the leftovers from the first attempt to plot histograms.

    This function is the leftovers from the first attempt to plot histograms.

    As it stands it needs some work in order to function again. We will

    As it stands it needs some work in order to function again. We will

    fix this later. 1/25/22

    fix this later. 1/25/22

    """

    """

    image = tiff_list[i]

    image = tiff_list[i]

    image = Image.open(image)    #Open the image and read it as an Image object

    image = Image.open(image)    #Open the image and read it as an Image object

    image = np.array(image)[1:,:]    #Convert to an array, leaving out the first row because the first row is just housekeeping data

    image = np.array(image)[1:,:]    #Convert to an array, leaving out the first row because the first row is just housekeeping data

    image_int = image.astype(int)

    image_int = image.astype(int)

    A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation

    A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation

    z0 = image_int[0:-2,0:-2]   # get all the first pixel for the entire image

    z0 = image_int[0:-2,0:-2]   # get all the first pixel for the entire image

    z1 = image_int[0:-2,1:-1]   # get all the second pixel for the entire image

    z1 = image_int[0:-2,1:-1]   # get all the second pixel for the entire image

    z2 = image_int[0:-2,2::]    # get all the third pixel for the entire image

    z2 = image_int[0:-2,2::]    # get all the third pixel for the entire image

    z3 = image_int[1:-1,0:-2]   # get all the fourth pixel for the entire image

    z3 = image_int[1:-1,0:-2]   # get all the fourth pixel for the entire image

    # calculate the out put of the system of equation

    # calculate the out put of the system of equation

    y0 = np.ravel(-z0+z2-z3)

    y0 = np.ravel(-z0+z2-z3)

    y1 = np.ravel(z0+z1+z2)

    y1 = np.ravel(z0+z1+z2)

    y2 = np.ravel(-z0-z1-z2-z3)

    y2 = np.ravel(-z0-z1-z2-z3)

    y = np.vstack((y0,y1,y2))

    y = np.vstack((y0,y1,y2))

    # use numpy solver to solve the system of equations all at once

    # use numpy solver to solve the system of equations all at once

    #predict = np.linalg.solve(A,y)[-1]

    #predict = np.linalg.solve(A,y)[-1]

    predict = np.floor(np.linalg.solve(A,y)[-1]).astype(int)

    predict = np.floor(np.linalg.solve(A,y)[-1]).astype(int)

    #predict = []

    #predict = []

    # flatten the neighbor pixels and stack them together

    # flatten the neighbor pixels and stack them together

    z0 = np.ravel(z0)

    z0 = np.ravel(z0)

    z1 = np.ravel(z1)

    z1 = np.ravel(z1)

    z2 = np.ravel(z2)

    z2 = np.ravel(z2)

    z3 = np.ravel(z3)

    z3 = np.ravel(z3)

    neighbor = np.vstack((z0,z1,z2,z3)).T

    neighbor = np.vstack((z0,z1,z2,z3)).T

    # calculate the difference

    # calculate the difference

    diff = np.max(neighbor,axis = 1) - np.min(neighbor, axis=1)

    diff = np.max(neighbor,axis = 1) - np.min(neighbor, axis=1)

    diff = np.pad(diff.reshape(510,638), pad_width=1)

    diff = np.pad(diff.reshape(510,638), pad_width=1)

    # flatten the image to a vector

    # flatten the image to a vector

    small_image = image_int[1:-1,1:-1]

    small_image = image_int[1:-1,1:-1]

    #Reshape the predictions to be a 2D array

    #Reshape the predictions to be a 2D array

    predict = np.pad(predict.reshape(510,638), pad_width=1)

    predict = np.pad(predict.reshape(510,638), pad_width=1)

    """predict[0,:] = image[0,:]

    """predict[0,:] = image[0,:]

    predict[:,0] = image[:,0]

    predict[:,0] = image[:,0]

    predict[:,-1] = image[:,-1]

    predict[:,-1] = image[:,-1]

    predict[-1,:] = image[-1,:]"""

    predict[-1,:] = image[-1,:]"""

    #Calculate the error between the original image and our predictions

    #Calculate the error between the original image and our predictions

    #Note that we only predicted on the inside square of the original image, excluding

    #Note that we only predicted on the inside square of the original image, excluding

    #The first row, column and last row, column

    #The first row, column and last row, column

    #error = (image_int - predict).astype(int) #Experiment

    #error = (image_int - predict).astype(int) #Experiment

    #this one works

    #this one works

    error = image_int - predict

    error = image_int - predict

    return predict, diff, image_int, error, A

    return predict, diff, image_int, error, A

```

```

%% Cell type:code id:ba2881d9 tags:

%% Cell type:code id:ba2881d9 tags:

``` python

``` python

scenes = file_extractor()

scenes = file_extractor()

images = image_extractor(scenes)

images = image_extractor(scenes)

num_images = im_distribution(images, "11")

num_images = im_distribution(images, "11")

```

```

%% Cell type:code id:11e95c34 tags:

%% Cell type:code id:11e95c34 tags:

``` python

``` python

predict, diff, im, err, A = plot_hist(images, 2)

predict, diff, im, err, A = plot_hist(images, 2)

```

```

%% Cell type:code id:434e4d2f tags:

%% Cell type:code id:434e4d2f tags:

``` python

``` python

def reconstruct(error, A):

def reconstruct(error, A):

    """

    """

    Function that reconstructs the original image

    Function that reconstructs the original image

    from the error matrix and using the predictive

    from the error matrix and using the predictive

    algorithm developed in the encoding.

    algorithm developed in the encoding.

    Parameters:

    Parameters:

        error (array): matrix of errors computed in encoding. Same

        error (array): matrix of errors computed in encoding. Same

                       shape as the original image (512, 640) in this case

                       shape as the original image (512, 640) in this case

        A (array): Matrix used for the system of equations to create predictions

        A (array): Matrix used for the system of equations to create predictions

    Returns:

    Returns:

        image (array): The reconstructed image

        image (array): The reconstructed image

    """

    """

    new_e = error.copy()

    new_e = error.copy()

    rows, columns = new_e.shape

    rows, columns = new_e.shape

    for r in range(1, rows-1):

    for r in range(1, rows-1):

        for c in range(1, columns-1):

        for c in range(1, columns-1):

            z0, z1, z2, z3 = new_e[r-1][c-1], new_e[r-1][c], new_e[r-1][c+1], new_e[r][c-1]

            z0, z1, z2, z3 = new_e[r-1][c-1], new_e[r-1][c], new_e[r-1][c+1], new_e[r][c-1]

            y = np.vstack((-z0+z2-z3, z0+z1+z2, -z0-z1-z2-z3))

            y = np.vstack((-z0+z2-z3, z0+z1+z2, -z0-z1-z2-z3))

            new_e[r][c] = np.round(new_e[r][c] + np.linalg.solve(A,y)[-1], 1)

            new_e[r][c] = np.round(new_e[r][c] + np.linalg.solve(A,y)[-1], 1)

    return new_e.astype(int)

    return new_e.astype(int)

```

```

%% Cell type:code id:3cc609dc tags:

%% Cell type:code id:3cc609dc tags:

``` python

``` python

new_error = reconstruct(err, A)

new_error = reconstruct(err, A)

```

```

%% Cell type:code id:5d290a0c tags:

%% Cell type:code id:5d290a0c tags:

``` python

``` python

im == new_error

im == new_error

```

```

%% Output

%% Output

    array([[ True,  True,  True, ...,  True,  True,  True],

    array([[ True,  True,  True, ...,  True,  True,  True],

           [ True,  True,  True, ...,  True,  True,  True],

           [ True,  True,  True, ...,  True,  True,  True],

           [ True,  True,  True, ...,  True,  True,  True],

           [ True,  True,  True, ...,  True,  True,  True],

           ...,

           ...,

           [ True,  True,  True, ...,  True,  True,  True],

           [ True,  True,  True, ...,  True,  True,  True],

           [ True,  True,  True, ...,  True,  True,  True],

           [ True,  True,  True, ...,  True,  True,  True],

           [ True,  True,  True, ...,  True,  True,  True]])

           [ True,  True,  True, ...,  True,  True,  True]])

%% Cell type:code id:706f2816 tags:

%% Cell type:code id:706f2816 tags:

``` python

``` python

first = []

first = []

second = []

second = []

third = []

third = []

fourth = []

fourth = []

for i in range(1,diff.shape[0]-1):

for i in range(1,diff.shape[0]-1):

    for j in range(1,diff.shape[1]-1):

    for j in range(1,diff.shape[1]-1):

        if diff[i][j] <= 50:

        if diff[i][j] <= 50:

            first.append(np.abs(err[i][j]))

            first.append(np.abs(err[i][j]))

        elif diff[i][j] > 50 and diff[i][j] <= 100:

        elif diff[i][j] > 50 and diff[i][j] <= 100:

            second.append(np.abs(err[i][j]))

            second.append(np.abs(err[i][j]))

        elif diff[i][j] > 100 and diff[i][j] <= 200:

        elif diff[i][j] > 100 and diff[i][j] <= 200:

            third.append(np.abs(err[i][j]))

            third.append(np.abs(err[i][j]))

        else:

        else:

            fourth.append(np.abs(err[i][j]))

            fourth.append(np.abs(err[i][j]))

```

```

%% Cell type:code id:530d2cab tags:

%% Cell type:code id:530d2cab tags:

``` python

``` python

plt.hist(first)

plt.hist(first)

plt.show()

plt.show()

print(np.max(first))

print(np.max(first))

plt.hist(second)

plt.hist(second)

plt.show()

plt.show()

print(np.max(second))

print(np.max(second))

plt.hist(third)

plt.hist(third)

plt.show()

plt.show()

print(np.max(third))

print(np.max(third))

plt.hist(fourth)

plt.hist(fourth)

plt.show()

plt.show()

print(np.max(fourth))

print(np.max(fourth))

```

```

%% Output

%% Output

    142

    142

    154

    154

    217

    217

    176

    176

%% Cell type:code id:bb11dcd0 tags:

%% Cell type:code id:bb11dcd0 tags:

``` python

``` python

class NodeTree(object):

class NodeTree(object):

    def __init__(self, left=None, right=None):

    def __init__(self, left=None, right=None):

        self.left = left

        self.left = left

        self.right = right

        self.right = right

    def children(self):

    def children(self):

        return self.left, self.right

        return self.left, self.right

    def __str__(self):

    def __str__(self):

        return self.left, self.right

        return self.left, self.right

def huffman_code_tree(node, binString=''):

def huffman_code_tree(node, binString=''):

    '''

    '''

    Function to find Huffman Code

    Function to find Huffman Code

    '''

    '''

    if type(node) is str:

    if type(node) is str:

        return {node: binString}

        return {node: binString}

    (l, r) = node.children()

    (l, r) = node.children()

    d = dict()

    d = dict()

    d.update(huffman_code_tree(l, binString + '0'))

    d.update(huffman_code_tree(l, binString + '0'))

    d.update(huffman_code_tree(r, binString + '1'))

    d.update(huffman_code_tree(r, binString + '1'))

    return d

    return d

def make_tree(nodes):

def make_tree(nodes):

    '''

    '''

    Function to make tree

    Function to make tree

    :param nodes: Nodes

    :param nodes: Nodes

    :return: Root of the tree

    :return: Root of the tree

    '''

    '''

    while len(nodes) > 1:

    while len(nodes) > 1:

        (key1, c1) = nodes[-1]

        (key1, c1) = nodes[-1]

        (key2, c2) = nodes[-2]

        (key2, c2) = nodes[-2]

        nodes = nodes[:-2]

        nodes = nodes[:-2]

        node = NodeTree(key1, key2)

        node = NodeTree(key1, key2)

        nodes.append((node, c1 + c2))

        nodes.append((node, c1 + c2))

        nodes = sorted(nodes, key=lambda x: x[1], reverse=True)

        nodes = sorted(nodes, key=lambda x: x[1], reverse=True)

    return nodes[0][0]

    return nodes[0][0]

```

```

%% Cell type:code id:c01fda28 tags:

%% Cell type:code id:c01fda28 tags:

``` python

``` python

def enc_experiment(images, plot=True):

def enc_experiment(images, plot=True):

    origin, predict, diff, error, A = plot_hist(images, 2)

    origin, predict, diff, error, A = plot_hist(images, 2)

    image = Image.open(images[2])    #Open the image and read it as an Image object

    image = Image.open(images[2])    #Open the image and read it as an Image object

    image = np.array(image)[1:,:]    #Convert to an array, leaving out the first row because the first row is just housekeeping data

    image = np.array(image)[1:,:]    #Convert to an array, leaving out the first row because the first row is just housekeeping data

    image = image.astype(int)

    image = image.astype(int)

    new_error = np.copy(image)

    new_error = np.copy(image)

    #new_error[1:-1,1:-1] = np.reshape(error[1:-1,1:-1],(510, 638))

    #new_error[1:-1,1:-1] = np.reshape(error[1:-1,1:-1],(510, 638))

    new_error[1:-1, 1:-1] = error[1:-1, 1:-1]

    new_error[1:-1, 1:-1] = error[1:-1, 1:-1]

    keep = new_error[0,0]

    keep = new_error[0,0]

    new_error[0,:] = new_error[0,:] - keep

    new_error[0,:] = new_error[0,:] - keep

    new_error[-1,:] = new_error[-1,:] - keep

    new_error[-1,:] = new_error[-1,:] - keep

    new_error[1:-1,0] = new_error[1:-1,0] - keep

    new_error[1:-1,0] = new_error[1:-1,0] - keep

    new_error[1:-1,-1] = new_error[1:-1,-1] - keep

    new_error[1:-1,-1] = new_error[1:-1,-1] - keep

    new_error[0,0] = keep

    new_error[0,0] = keep

    new_error = np.ravel(new_error)

    new_error = np.ravel(new_error)

    if plot:

    if plot:

        plt.hist(new_error[1:],bins=100)

        plt.hist(new_error[1:],bins=100)

        plt.show()

        plt.show()

    #ab_error = np.abs(new_error)

    #ab_error = np.abs(new_error)

    #string = [str(i) for i in ab_error]

    #string = [str(i) for i in ab_error]

    string = [str(i) for i in new_error]

    string = [str(i) for i in new_error]

    #string = [str(i) for i in np.arange(0,5)] + [str(i) for i in np.arange(0,5)] + [str(i) for i in np.arange(0,2)]*2

    #string = [str(i) for i in np.arange(0,5)] + [str(i) for i in np.arange(0,5)] + [str(i) for i in np.arange(0,2)]*2

    freq = dict(Counter(string))

    freq = dict(Counter(string))

    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)

    freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)

    node = make_tree(freq)

    node = make_tree(freq)

    encoding_dict = huffman_code_tree(node)

    encoding_dict = huffman_code_tree(node)

    #encoded = ["1"+encoding[str(-i)] if i < 0 else "0"+encoding[str(i)] for i in error]

    #encoded = ["1"+encoding[str(-i)] if i < 0 else "0"+encoding[str(i)] for i in error]

    #print(time.time()-start)

    #print(time.time()-start)

    encoded = new_error.reshape((512,640)).copy().astype(str).astype(object)

    encoded = new_error.reshape((512,640)).copy().astype(str).astype(object)

    for i in range(encoded.shape[0]):

    for i in range(encoded.shape[0]):

        for j in range(encoded.shape[1]):

        for j in range(encoded.shape[1]):

            if i == 0 and j == 0:

            if i == 0 and j == 0:

                encoded[i][j] = encoded[i][j]

                encoded[i][j] = encoded[i][j]

            else:

            else:

                #print(encoding_dict[encoded[i][j]])

                #print(encoding_dict[encoded[i][j]])

                encoded[i][j] = encoding_dict[encoded[i][j]]

                encoded[i][j] = encoding_dict[encoded[i][j]]

                #print(encoded[i][j])

                #print(encoded[i][j])

    return encoding_dict, encoded, new_error.reshape((512,640)), image

    return encoding_dict, encoded, new_error.reshape((512,640)), image

    #print(encoding)

    #print(encoding)

```

```

%% Cell type:code id:ffa858e8 tags:

%% Cell type:code id:ffa858e8 tags:

``` python

``` python

encode_dict, encoding, error, orig_image = enc_experiment(images, plot=False)

encode_dict, encoding, error, orig_image = enc_experiment(images, plot=False)

```

```

%% Cell type:code id:8dfdedc6 tags:

%% Cell type:code id:8dfdedc6 tags:

``` python

``` python

error[1,6]

error[1,6]

```

```

%% Output

%% Output

    0

    0

%% Cell type:code id:825cc48c tags:

%% Cell type:code id:825cc48c tags:

``` python

``` python

def decoder(A, encoded_matrix, encoding_dict):

def decoder(A, encoded_matrix, encoding_dict):

    """

    """

    Function that accecpts the prediction matrix A for the linear system,

    Function that accecpts the prediction matrix A for the linear system,

    the encoded matrix of error values, and the encoding dicitonary.

    the encoded matrix of error values, and the encoding dicitonary.

    """

    """

    the_keys = list(encode_dict.keys())

    the_keys = list(encode_dict.keys())

    the_values = list(encode_dict.values())

    the_values = list(encode_dict.values())

    error_matrix = encoded_matrix.copy()

    error_matrix = encoded_matrix.copy()

    for i in range(error_matrix.shape[0]):

    for i in range(error_matrix.shape[0]):

        for j in range(error_matrix.shape[1]):

        for j in range(error_matrix.shape[1]):

            if i == 0 and j == 0:

            if i == 0 and j == 0:

                error_matrix[i][j] = int(encoded_matrix[i][j])

                error_matrix[i][j] = int(encoded_matrix[i][j])

            elif i == 0 or i == error_matrix.shape[0]-1 or j == 0 or j == error_matrix.shape[1]-1:

            elif i == 0 or i == error_matrix.shape[0]-1 or j == 0 or j == error_matrix.shape[1]-1:

                error_matrix[i][j] = int(the_keys[the_values.index(error_matrix[i,j])]) + error_matrix[0][0]

                error_matrix[i][j] = int(the_keys[the_values.index(error_matrix[i,j])]) + error_matrix[0][0]

            else:

            else:

                """z0, z1, z2, z3 = error_matrix[i-1][j-1], error_matrix[i-1][j], \

                """z0, z1, z2, z3 = error_matrix[i-1][j-1], error_matrix[i-1][j], \

                error_matrix[i-1][j+1], error_matrix[i][j-1]

                error_matrix[i-1][j+1], error_matrix[i][j-1]

                y = np.vstack((-z0+z2-z3, z0+z1+z2, -z0-z1-z2-z3))"""

                y = np.vstack((-z0+z2-z3, z0+z1+z2, -z0-z1-z2-z3))"""

                error_matrix[i][j] = int(the_keys[the_values.index(error_matrix[i,j])])

                error_matrix[i][j] = int(the_keys[the_values.index(error_matrix[i,j])])

    return error_matrix.astype(int)