update (ffadfc7a) · Commits · Elphel / image-compression

.ipynb_checkpoints/Error_to_Image-checkpoint.ipynb

+12 −2

Original line number	Original line	Diff line number	Diff line
	%% Cell type:code id:dbef8759 tags:		%% Cell type:code id:dbef8759 tags:

	``` python		``` python
	import numpy as np		import numpy as np
	from prediction_MSE_Scout import file_extractor, image_extractor, im_distribution		from prediction_MSE_Scout import file_extractor, image_extractor, im_distribution
	from matplotlib import pyplot as plt		from matplotlib import pyplot as plt
	from itertools import product		from itertools import product
	import os		import os
	import sys		import sys
	from PIL import Image		from PIL import Image
	from scipy.optimize import minimize		from scipy.optimize import minimize
	from time import time		from time import time
	from numpy import linalg as la		from numpy import linalg as la
	from scipy.stats import gaussian_kde		from scipy.stats import gaussian_kde
	import seaborn as sns		import seaborn as sns
	from collections import Counter		from collections import Counter
	import pandas as pd		import pandas as pd
	import scipy as sp		import scipy as sp
	```		```

	%% Cell type:code id:9ed20f84 tags:		%% Cell type:code id:9ed20f84 tags:

	``` python		``` python
	def predict(tiff_list, i=0):		def predict(tiff_list, i=0):
	"""		"""
	This function predicts the pixel values based on a linear combination		This function predicts the pixel values based on a linear combination
	of the MSE from the three pixels above it and the one to the left. It		of the MSE from the three pixels above it and the one to the left. It
	uses a system of equations to fit the plane ax + by + c and takes c		uses a system of equations to fit the plane ax + by + c and takes c
	as the prediction for the unknown pixel. It does this all at once		as the prediction for the unknown pixel. It does this all at once
	by constructing vectors and matrices of the surrounding pixels and solving each system simultaneously		by constructing vectors and matrices of the surrounding pixels and solving each system simultaneously
	so as not to iterate through each one.		so as not to iterate through each one.

	Parameters:		Parameters:
	tiff_list: list, list of names of image file paths to access. These should be strings		tiff_list: list, list of names of image file paths to access. These should be strings
	in the form of a path to the image		in the form of a path to the image

	i: int, which index in the tiff_list of images we want to predict on		i: int, which index in the tiff_list of images we want to predict on

	Returns:		Returns:
	prediction: matrix (ndarray), the matrix of predicted values		prediction: matrix (ndarray), the matrix of predicted values
	for the image using the previous four piexels		for the image using the previous four piexels

	diff: matrix (ndarray), the difference between the highest and lowest valued surrounding four pixels		diff: matrix (ndarray), the difference between the highest and lowest valued surrounding four pixels

	image_int: matrix (ndarray), the original image, changed into integers		image_int: matrix (ndarray), the original image, changed into integers

	error: matrix (ndarray), a matrix of errors, so each entry is the		error: matrix (ndarray), a matrix of errors, so each entry is the
	difference between the integer predicted value and the actual value. Should		difference between the integer predicted value and the actual value. Should
	be all integers		be all integers

	A: matrix (3,3 ndarray), the matrix used to solve the MSE system		A: matrix (3,3 ndarray), the matrix used to solve the MSE system
	"""		"""

	image = tiff_list[i]		image = tiff_list[i]
	image = Image.open(image) #Open the image and read it as an Image object		image = Image.open(image) #Open the image and read it as an Image object
	image = np.array(image)[1:,:] #Convert to an array, leaving out the first row because the first row is just housekeeping data		image = np.array(image)[1:,:] #Convert to an array, leaving out the first row because the first row is just housekeeping data
	image_int = image.astype(int)		image_int = image.astype(int)

	A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation		A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation

	z0 = image_int[0:-2,0:-2] # get all the first pixel for the entire image		z0 = image_int[0:-2,0:-2] # get all the first pixel for the entire image
	z1 = image_int[0:-2,1:-1] # get all the second pixel for the entire image		z1 = image_int[0:-2,1:-1] # get all the second pixel for the entire image
	z2 = image_int[0:-2,2::] # get all the third pixel for the entire image		z2 = image_int[0:-2,2::] # get all the third pixel for the entire image
	z3 = image_int[1:-1,0:-2] # get all the fourth pixel for the entire image		z3 = image_int[1:-1,0:-2] # get all the fourth pixel for the entire image

	# calculate the out put of the system of equation		# calculate the out put of the system of equation
	y0 = np.ravel(-z0+z2-z3)		y0 = np.ravel(-z0+z2-z3)
	y1 = np.ravel(z0+z1+z2)		y1 = np.ravel(z0+z1+z2)
	y2 = np.ravel(-z0-z1-z2-z3)		y2 = np.ravel(-z0-z1-z2-z3)
	y = np.vstack((y0,y1,y2))		y = np.vstack((y0,y1,y2))

	# use numpy solver to solve the system of equations all at once		# use numpy solver to solve the system of equations all at once
	#predict = np.linalg.solve(A,y)[-1]		#predict = np.linalg.solve(A,y)[-1]
	prediction = np.floor(np.linalg.solve(A,y)[-1]).astype(int)		prediction = np.floor(np.linalg.solve(A,y)[-1]).astype(int)
	#predict = []		#predict = []

	# flatten the neighbor pixels and stack them together		# flatten the neighbor pixels and stack them together
	z0 = np.ravel(z0)		z0 = np.ravel(z0)
	z1 = np.ravel(z1)		z1 = np.ravel(z1)
	z2 = np.ravel(z2)		z2 = np.ravel(z2)
	z3 = np.ravel(z3)		z3 = np.ravel(z3)
	neighbor = np.vstack((z0,z1,z2,z3)).T		neighbor = np.vstack((z0,z1,z2,z3)).T

	# calculate the difference		# calculate the difference
	diff = np.max(neighbor,axis = 1) - np.min(neighbor, axis=1)		diff = np.max(neighbor,axis = 1) - np.min(neighbor, axis=1)
	#diff = np.pad(diff.reshape(510,638), pad_width=1)		#diff = np.pad(diff.reshape(510,638), pad_width=1)

	# flatten the image to a vector		# flatten the image to a vector
	small_image = image_int[1:-1,1:-1]		small_image = image_int[1:-1,1:-1]

	#Reshape the predictions to be a 2D array		#Reshape the predictions to be a 2D array
	prediction = np.pad(prediction.reshape(510,638), pad_width=1)		prediction = np.pad(prediction.reshape(510,638), pad_width=1)


	#Calculate the error between the original image and our predictions		#Calculate the error between the original image and our predictions
	#Note that we only predicted on the inside square of the original image, excluding		#Note that we only predicted on the inside square of the original image, excluding
	#The first row, column and last row, column		#The first row, column and last row, column
	#error = (image_int - predict).astype(int) #Experiment		#error = (image_int - predict).astype(int) #Experiment

	#this one works		#this one works
	error = image_int - prediction		error = image_int - prediction


	return prediction, diff, image_int, error[1:-1,1:-1], A		return prediction, diff, image_int, error[1:-1,1:-1], A
	```		```

	%% Cell type:code id:ba2881d9 tags:		%% Cell type:code id:ba2881d9 tags:

	``` python		``` python
	scenes = file_extractor()		scenes = file_extractor()
	images = image_extractor(scenes)		images = image_extractor(scenes)
	num_images = im_distribution(images, "11")		num_images = im_distribution(images, "11")
	```		```

	%% Cell type:code id:11e95c34 tags:		%% Cell type:code id:11e95c34 tags:

	``` python		``` python
	prediction, diff, im, err, A = predict(images, 2)		prediction, diff, im, err, A = predict(images, 2)
	```		```

	%% Cell type:code id:434e4d2f tags:		%% Cell type:code id:434e4d2f tags:

	``` python		``` python
	def reconstruct(error, A):		def reconstruct(error, A):
	"""		"""
	Function that reconstructs the original image		Function that reconstructs the original image
	from the error matrix and using the predictive		from the error matrix and using the predictive
	algorithm developed in the encoding.		algorithm developed in the encoding.

	Parameters:		Parameters:
	error (array): matrix of errors computed in encoding. Same		error (array): matrix of errors computed in encoding. Same
	shape as the original image (512, 640) in this case		shape as the original image (512, 640) in this case
	A (array): Matrix used for the system of equations to create predictions		A (array): Matrix used for the system of equations to create predictions
	Returns:		Returns:
	image (array): The reconstructed image		image (array): The reconstructed image
	"""		"""
	new_e = error.copy()		new_e = error.copy()
	rows, columns = new_e.shape		rows, columns = new_e.shape

	for r in range(1, rows-1): #Iterate through the inside square of the error matrix		for r in range(1, rows-1): #Iterate through the inside square of the error matrix
	for c in range(1, columns-1):		for c in range(1, columns-1):
	z0, z1, z2, z3 = new_e[r-1][c-1], new_e[r-1][c], new_e[r-1][c+1], new_e[r][c-1] #Grab the four nearest pixels		z0, z1, z2, z3 = new_e[r-1][c-1], new_e[r-1][c], new_e[r-1][c+1], new_e[r][c-1] #Grab the four nearest pixels
	y = np.vstack((-z0+z2-z3, z0+z1+z2, -z0-z1-z2-z3)) #Create a vector of the linear combinations for the		y = np.vstack((-z0+z2-z3, z0+z1+z2, -z0-z1-z2-z3)) #Create a vector of the linear combinations for the
	#solution to be solved		#solution to be solved

	new_e[r][c] = np.round(new_e[r][c] + np.linalg.solve(A,y)[-1], 1) #Add the error to the solved system solution		new_e[r][c] = np.round(new_e[r][c] + np.linalg.solve(A,y)[-1], 1) #Add the error to the solved system solution
	#rounding the result because np.linalg.solve(A,y)		#rounding the result because np.linalg.solve(A,y)
	#can be a float. Since we did np.floor on it in		#can be a float. Since we did np.floor on it in
	#prediction, we round to the nearest integer here		#prediction, we round to the nearest integer here
	return new_e.astype(int)		return new_e.astype(int)

	```		```

	%% Cell type:code id:3cc609dc tags:		%% Cell type:code id:3cc609dc tags:

	``` python		``` python
	new_error = reconstruct(err, A)		new_error = reconstruct(err, A)
	```		```

	%% Cell type:code id:5d290a0c tags:		%% Cell type:code id:5d290a0c tags:

	``` python		``` python
	im == new_error		im == new_error
	```		```

	%% Output		%% Output

	C:\Users\calle\AppData\Local\Temp/ipykernel_23384/389333.py:1: DeprecationWarning: elementwise comparison failed; this will raise an error in the future.		C:\Users\calle\AppData\Local\Temp/ipykernel_23384/389333.py:1: DeprecationWarning: elementwise comparison failed; this will raise an error in the future.
	im == new_error		im == new_error

	False		False

	%% Cell type:code id:bb11dcd0 tags:		%% Cell type:code id:bb11dcd0 tags:

	``` python		``` python
	class NodeTree(object):		class NodeTree(object):
	def __init__(self, left=None, right=None):		def __init__(self, left=None, right=None):
	self.left = left		self.left = left
	self.right = right		self.right = right

	def children(self):		def children(self):
	return self.left, self.right		return self.left, self.right

	def __str__(self):		def __str__(self):
	return self.left, self.right		return self.left, self.right


	def huffman_code_tree(node, binString=''):		def huffman_code_tree(node, binString=''):
	'''		'''
	Function to find Huffman Code		Function to find Huffman Code
	'''		'''
	if type(node) is str:		if type(node) is str:
	return {node: binString}		return {node: binString}
	(l, r) = node.children()		(l, r) = node.children()
	d = dict()		d = dict()
	d.update(huffman_code_tree(l, binString + '0'))		d.update(huffman_code_tree(l, binString + '0'))
	d.update(huffman_code_tree(r, binString + '1'))		d.update(huffman_code_tree(r, binString + '1'))
	return d		return d


	def make_tree(nodes):		def make_tree(nodes):
	'''		'''
	Function to make tree		Function to make tree
	:param nodes: Nodes		:param nodes: Nodes
	:return: Root of the tree		:return: Root of the tree
	'''		'''
	while len(nodes) > 1:		while len(nodes) > 1:
	(key1, c1) = nodes[-1]		(key1, c1) = nodes[-1]
	(key2, c2) = nodes[-2]		(key2, c2) = nodes[-2]
	nodes = nodes[:-2]		nodes = nodes[:-2]
	node = NodeTree(key1, key2)		node = NodeTree(key1, key2)
	nodes.append((node, c1 + c2))		nodes.append((node, c1 + c2))
	nodes = sorted(nodes, key=lambda x: x[1], reverse=True)		nodes = sorted(nodes, key=lambda x: x[1], reverse=True)
	return nodes[0][0]		return nodes[0][0]
	```		```

	%% Cell type:code id:c01fda28 tags:		%% Cell type:code id:c01fda28 tags:

	``` python		``` python
	def encoder(images, i, plot=True):		def encoder(images, i, plot=True):
	"""		"""
	Function that creates Huffman encodings out of the error values		Function that creates Huffman encodings out of the error values
	for a given image. The encodings are more efficient ways to store		for a given image. The encodings are more efficient ways to store
	large integer values that the original image contains.		large integer values that the original image contains.

	Parameters:		Parameters:
	images (list): list of file paths to the images that		images (list): list of file paths to the images that
	will be encoded.		will be encoded.

	i (int): which index of the images list to grab and		i (int): which index of the images list to grab and
	then encode.		then encode.

	plot (bool): if true, this plots the error matrix to		plot (bool): if true, this plots the error matrix to
	show the distribution of values.		show the distribution of values.
	"""		"""

	prediction, diff, original, error, A = predict(images, i) #Predict the values and return the error for the specified image		prediction, diff, original, error, A = predict(images, i) #Predict the values and return the error for the specified image
	image = original		image = original
	new_error = np.copy(image) #Create a new matrix that is a copy of the original image, this is the matrix we will		new_error = np.copy(image) #Create a new matrix that is a copy of the original image, this is the matrix we will
	#update on throughout		#update on throughout
	#new_error[1:-1,1:-1] = np.reshape(error[1:-1,1:-1],(510, 638))		#new_error[1:-1,1:-1] = np.reshape(error[1:-1,1:-1],(510, 638))
	new_error[1:-1, 1:-1] = error[1:-1, 1:-1] #Set the inside of the updating matrix to be the same as the		new_error[1:-1, 1:-1] = error[1:-1, 1:-1] #Set the inside of the updating matrix to be the same as the
	#error matrix retreived from predicting		#error matrix retreived from predicting
	keep = new_error[0,0] #The top left entry stays the same		keep = new_error[0,0] #The top left entry stays the same
	new_error[0,:] = new_error[0,:] - keep #All edge pixels are set to be the difference between themselves and		new_error[0,:] = new_error[0,:] - keep #All edge pixels are set to be the difference between themselves and
	new_error[-1,:] = new_error[-1,:] - keep #the top left entry named "keep". This reduces their size to a more		new_error[-1,:] = new_error[-1,:] - keep #the top left entry named "keep". This reduces their size to a more
	new_error[1:-1,0] = new_error[1:-1,0] - keep #manageable integer and makes them encodeable with the other error values		new_error[1:-1,0] = new_error[1:-1,0] - keep #manageable integer and makes them encodeable with the other error values
	new_error[1:-1,-1] = new_error[1:-1,-1] - keep		new_error[1:-1,-1] = new_error[1:-1,-1] - keep
	new_error[0,0] = keep		new_error[0,0] = keep

	new_error = np.ravel(new_error) #Unravel it to plot it		new_error = np.ravel(new_error) #Unravel it to plot it
	if plot:		if plot:
	plt.hist(new_error[1:],bins=100)		plt.hist(new_error[1:],bins=100)
	plt.show()		plt.show()


	string = [str(i) for i in new_error] #Create strings out of the integers in the new_error matrix		string = [str(i) for i in new_error] #Create strings out of the integers in the new_error matrix

	freq = dict(Counter(string)) #Initialize a dictionary that maps integers to the string values		freq = dict(Counter(string)) #Initialize a dictionary that maps integers to the string values
	freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) #Create a frequency mapping of how often the string		freq = sorted(freq.items(), key=lambda x: x[1], reverse=True) #Create a frequency mapping of how often the string
	#values occur in the dictionary		#values occur in the dictionary
	node = make_tree(freq) #Use the Huffman code given above to make a Huffman tree		node = make_tree(freq) #Use the Huffman code given above to make a Huffman tree

	encoding_dict = huffman_code_tree(node) #Create the Huffman dictionary		encoding_dict = huffman_code_tree(node) #Create the Huffman dictionary

	#encoded = ["1"+encoding[str(-i)] if i < 0 else "0"+encoding[str(i)] for i in error]		#encoded = ["1"+encoding[str(-i)] if i < 0 else "0"+encoding[str(i)] for i in error]

	encoded = new_error.reshape((512,640)).copy().astype(str).astype(object) #Reshape the error matrix and make a copy that		encoded = new_error.reshape((512,640)).copy().astype(str).astype(object) #Reshape the error matrix and make a copy that
	#that is all strings so we can call the		#that is all strings so we can call the
	#dictionary on its entries		#dictionary on its entries
	for i in range(encoded.shape[0]): #Iterate through the string valued error dictionary		for i in range(encoded.shape[0]): #Iterate through the string valued error dictionary
	for j in range(encoded.shape[1]):		for j in range(encoded.shape[1]):
	if i == 0 and j == 0:		if i == 0 and j == 0:
	encoded[i][j] = encoded[i][j] #Replace each value in the dictionary with its encoding from the dictionary		encoded[i][j] = encoded[i][j] #Replace each value in the dictionary with its encoding from the dictionary
	else:		else:
	encoded[i][j] = encoding_dict[encoded[i][j]]		encoded[i][j] = encoding_dict[encoded[i][j]]

	return encoding_dict, encoded, new_error.reshape((512,640)), image		return encoding_dict, encoded, new_error.reshape((512,640)), image
	#print(encoding)		#print(encoding)
	```		```

	%% Cell type:code id:ffa858e8 tags:		%% Cell type:code id:ffa858e8 tags:

	``` python		``` python
	encode_dict, encoding, error, orig_image = encoder(images, 2, plot=False)		encode_dict, encoding, error, orig_image = encoder(images, 2, plot=False)
	```		```

	%% Output		%% Output

	---------------------------------------------------------------------------		---------------------------------------------------------------------------
	ValueError Traceback (most recent call last)		ValueError Traceback (most recent call last)
	~\AppData\Local\Temp/ipykernel_2620/384786850.py in <module>		~\AppData\Local\Temp/ipykernel_2620/384786850.py in <module>
	----> 1 encode_dict, encoding, error, orig_image = encoder(images, 2, plot=False)		----> 1 encode_dict, encoding, error, orig_image = encoder(images, 2, plot=False)

	~\AppData\Local\Temp/ipykernel_2620/3253315524.py in encoder(images, i, plot)		~\AppData\Local\Temp/ipykernel_2620/3253315524.py in encoder(images, i, plot)
	21 #update on throughout		21 #update on throughout
	22 #new_error[1:-1,1:-1] = np.reshape(error[1:-1,1:-1],(510, 638))		22 #new_error[1:-1,1:-1] = np.reshape(error[1:-1,1:-1],(510, 638))
	---> 23 new_error[1:-1, 1:-1] = error[1:-1, 1:-1] #Set the inside of the updating matrix to be the same as the		---> 23 new_error[1:-1, 1:-1] = error[1:-1, 1:-1] #Set the inside of the updating matrix to be the same as the
	24 #error matrix retreived from predicting		24 #error matrix retreived from predicting
	25 keep = new_error[0,0] #The top left entry stays the same		25 keep = new_error[0,0] #The top left entry stays the same
	ValueError: could not broadcast input array from shape (508,636) into shape (510,638)		ValueError: could not broadcast input array from shape (508,636) into shape (510,638)

	%% Cell type:code id:825cc48c tags:		%% Cell type:code id:825cc48c tags:

	``` python		``` python
	def decoder(A, encoded_matrix, encoding_dict):		def decoder(A, encoded_matrix, encoding_dict):
	"""		"""
	Function that accecpts the prediction matrix A for the linear system,		Function that accecpts the prediction matrix A for the linear system,
	the encoded matrix of error values, and the encoding dicitonary.		the encoded matrix of error values, and the encoding dicitonary.
	"""		"""
	the_keys = list(encoding_dict.keys())		the_keys = list(encoding_dict.keys())
	the_values = list(encoding_dict.values())		the_values = list(encoding_dict.values())
	error_matrix = encoded_matrix.copy()		error_matrix = encoded_matrix.copy()

	for i in range(error_matrix.shape[0]):		for i in range(error_matrix.shape[0]):
	for j in range(error_matrix.shape[1]):		for j in range(error_matrix.shape[1]):
	if i == 0 and j == 0:		if i == 0 and j == 0:
	error_matrix[i][j] = int(encoded_matrix[i][j])		error_matrix[i][j] = int(encoded_matrix[i][j])

	elif i == 0 or i == error_matrix.shape[0]-1 or j == 0 or j == error_matrix.shape[1]-1:		elif i == 0 or i == error_matrix.shape[0]-1 or j == 0 or j == error_matrix.shape[1]-1:
	error_matrix[i][j] = int(the_keys[the_values.index(error_matrix[i,j])]) + error_matrix[0][0]		error_matrix[i][j] = int(the_keys[the_values.index(error_matrix[i,j])]) + error_matrix[0][0]
	else:		else:
	"""z0, z1, z2, z3 = error_matrix[i-1][j-1], error_matrix[i-1][j], \		"""z0, z1, z2, z3 = error_matrix[i-1][j-1], error_matrix[i-1][j], \
	error_matrix[i-1][j+1], error_matrix[i][j-1]		error_matrix[i-1][j+1], error_matrix[i][j-1]
	y = np.vstack((-z0+z2-z3, z0+z1+z2, -z0-z1-z2-z3))"""		y = np.vstack((-z0+z2-z3, z0+z1+z2, -z0-z1-z2-z3))"""

	error_matrix[i][j] = int(the_keys[the_values.index(error_matrix[i,j])])		error_matrix[i][j] = int(the_keys[the_values.index(error_matrix[i,j])])

	return error_matrix.astype(int)		return error_matrix.astype(int)
	```		```

	%% Cell type:code id:ba1d2c2c tags:		%% Cell type:code id:ba1d2c2c tags:

	``` python		``` python
	em = decoder(A, encoding, encode_dict)		em = decoder(A, encoding, encode_dict)
	```		```

	%% Output		%% Output

	---------------------------------------------------------------------------		---------------------------------------------------------------------------
	NameError Traceback (most recent call last)		NameError Traceback (most recent call last)
	~\AppData\Local\Temp/ipykernel_23384/3979147550.py in <module>		~\AppData\Local\Temp/ipykernel_23384/3979147550.py in <module>
	----> 1 em = decoder(A, encoding, encode_dict)		----> 1 em = decoder(A, encoding, encode_dict)

	NameError: name 'encoding' is not defined		NameError: name 'encoding' is not defined

	%% Cell type:code id:b2cdce6d tags:		%% Cell type:code id:b2cdce6d tags:

	``` python		``` python
	hopefully = reconstruct(em, A)		hopefully = reconstruct(em, A)
	#22487 22483 22521 22464		#22487 22483 22521 22464
	```		```

	%% Output		%% Output

	---------------------------------------------------------------------------		---------------------------------------------------------------------------
	NameError Traceback (most recent call last)		NameError Traceback (most recent call last)
	~\AppData\Local\Temp/ipykernel_23384/2268978435.py in <module>		~\AppData\Local\Temp/ipykernel_23384/2268978435.py in <module>
	----> 1 hopefully = reconstruct(em, A)		----> 1 hopefully = reconstruct(em, A)
	2 #22487 22483 22521 22464		2 #22487 22483 22521 22464
	NameError: name 'em' is not defined		NameError: name 'em' is not defined

	%% Cell type:code id:285efcf0 tags:		%% Cell type:code id:285efcf0 tags:

	``` python		``` python
	def test_decoder():		def test_decoder():
	n = len(images)//12		n = len(images)//12
	fails = 0		fails = 0
	for i in range(n):		for i in range(n):
	encode_dict1, encoding1, error1, orig_image1 = encoder(images, i, plot=False)		encode_dict1, encoding1, error1, orig_image1 = encoder(images, i, plot=False)
	new_error = decoder(A, encoding1, encode_dict1)		new_error = decoder(A, encoding1, encode_dict1)
	reconstructed_image = reconstruct(new_error, A)		reconstructed_image = reconstruct(new_error, A)
	if False in np.ravel(reconstructed_image == orig_image):		if False in np.ravel(reconstructed_image == orig_image):
	fails += 0		fails += 0
	return fails/n		return fails/n
	f = test_decoder()		f = test_decoder()
	```		```

	%% Cell type:code id:30b1c87e tags:		%% Cell type:code id:30b1c87e tags:

	``` python		``` python
	def entropy_func(images):		def entropy_func(images):
	"""		"""
	Computes the entropy for all pictures (tiff files) in the images list.		Computes the entropy for all pictures (tiff files) in the images list.
	This gives an idea of how many bits it would take on average to encode the		This gives an idea of how many bits it would take on average to encode the
	given image. The output is a list of entropies, one per image.		given image. The output is a list of entropies, one per image.
	"""		"""
	entr = []		entr = []
	for i in range(len(images)):		for i in range(len(images)):
	prediction, diff, im, err, A = predict(images, i)		prediction, diff, im, err, A = predict(images, i)
	panda_im = pd.Series(np.ravel(im))		panda_im = pd.Series(np.ravel(im))
	counts = panda_im.value_counts()		counts = panda_im.value_counts()
	entr.append(sp.stats.entropy(counts))		entr.append(sp.stats.entropy(counts))
	return entr		return entr

	e = entropy_func(images)		e = entropy_func(images)
	print(np.mean(e))		print(np.mean(e))
	```		```

	%% Output		%% Output

	6.7830123821108295		6.7830123821108295

	%% Cell type:code id:4c268907 tags:		%% Cell type:code id:4c268907 tags:

	``` python		``` python
	def huffman(image):		def huffman(image):
	origin, predicty, diff, error, A = predict(image,0)		origin, predicty, diff, error, A = predict(image,0)

	image = Image.open(image[0])		image = Image.open(image[0])
	image = np.array(image)[1:,:] #Convert to an array, leaving out the first row because the first row is just housekeeping data		image = np.array(image)[1:,:] #Convert to an array, leaving out the first row because the first row is just housekeeping data
	image = image.astype(int)		image = image.astype(int)

	boundary = np.hstack((image[0,:],image[-1,:],image[1:-1,0],image[1:-1,-1]))		boundary = np.hstack((image[0,:],image[-1,:],image[1:-1,0],image[1:-1,-1]))
	boundary = boundary - image[0,0]		boundary = boundary - image[0,0]
	boundary[0] = image[0,0]		boundary[0] = image[0,0]

	string = [str(i) for i in boundary]		string = [str(i) for i in boundary]
	freq = dict(Counter(string))		freq = dict(Counter(string))
	freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)		freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)
	node = make_tree(freq)		node = make_tree(freq)
	encode1 = huffman_code_tree(node)		encode1 = huffman_code_tree(node)


	mask = diff <= 25		mask = diff <= 25
	string = [str(i) for i in error[mask].astype(int)]		string = [str(i) for i in error[mask].astype(int)]
	freq = dict(Counter(string))		freq = dict(Counter(string))
	freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)		freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)
	node = make_tree(freq)		node = make_tree(freq)
	encode2 = huffman_code_tree(node)		encode2 = huffman_code_tree(node)


	mask = diff > 25		mask = diff > 25
	new_error = error[mask]		new_error = error[mask]
	mask2 = diff[mask] <= 40		mask2 = diff[mask] <= 40
	string = [str(i) for i in new_error[mask2].astype(int)]		string = [str(i) for i in new_error[mask2].astype(int)]
	freq = dict(Counter(string))		freq = dict(Counter(string))
	freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)		freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)
	node = make_tree(freq)		node = make_tree(freq)
	encode3 = huffman_code_tree(node)		encode3 = huffman_code_tree(node)


	mask = diff > 40		mask = diff > 40
	new_error = error[mask]		new_error = error[mask]
	mask2 = diff[mask] <= 70		mask2 = diff[mask] <= 70
	string = [str(i) for i in new_error[mask2].astype(int)]		string = [str(i) for i in new_error[mask2].astype(int)]
	freq = dict(Counter(string))		freq = dict(Counter(string))
	freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)		freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)
	node = make_tree(freq)		node = make_tree(freq)
	encode4 = huffman_code_tree(node)		encode4 = huffman_code_tree(node)


	mask = diff > 70		mask = diff > 70
	string = [str(i) for i in error[mask].astype(int)]		string = [str(i) for i in error[mask].astype(int)]
	freq = dict(Counter(string))		freq = dict(Counter(string))
	freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)		freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)
	node = make_tree(freq)		node = make_tree(freq)
	encode5 = huffman_code_tree(node)		encode5 = huffman_code_tree(node)


	new_error = np.copy(image)		new_error = np.copy(image)
	new_error[1:-1,1:-1] = np.reshape(error,(510, 638))		new_error[1:-1,1:-1] = np.reshape(error,(510, 638))
	keep = new_error[0,0]		keep = new_error[0,0]
	new_error[0,:] = new_error[0,:] - keep		new_error[0,:] = new_error[0,:] - keep
	new_error[-1,:] = new_error[-1,:] - keep		new_error[-1,:] = new_error[-1,:] - keep
	new_error[1:-1,0] = new_error[1:-1,0] - keep		new_error[1:-1,0] = new_error[1:-1,0] - keep
	new_error[1:-1,-1] = new_error[1:-1,-1] - keep		new_error[1:-1,-1] = new_error[1:-1,-1] - keep
	new_error[0,0] = keep		new_error[0,0] = keep


	#new_error = np.ravel(new_error)		#new_error = np.ravel(new_error)

	bins = [25,40,70]		bins = [25,40,70]

	# return the huffman dictionary		# return the huffman dictionary
	return encode1, encode2, encode3, encode4, encode5, np.ravel(image), error, new_error, diff, boundary, bins		return encode1, encode2, encode3, encode4, encode5, np.ravel(image), error, new_error, diff, boundary, bins

	scenes = file_extractor()		scenes = file_extractor()
	images = image_extractor(scenes)		images = image_extractor(scenes)
	encode1, encode2, encode3, encode4, encode5, image, error, new_error, diff, boundary, bins = huffman(images)		encode1, encode2, encode3, encode4, encode5, image, error, new_error, diff, boundary, bins = huffman(images)
	```		```

	%% Output		%% Output

	---------------------------------------------------------------------------		---------------------------------------------------------------------------
	IndexError Traceback (most recent call last)		IndexError Traceback (most recent call last)
	~\AppData\Local\Temp/ipykernel_2620/1618652474.py in <module>		~\AppData\Local\Temp/ipykernel_2620/1618652474.py in <module>
	72 scenes = file_extractor()		72 scenes = file_extractor()
	73 images = image_extractor(scenes)		73 images = image_extractor(scenes)
	---> 74 encode1, encode2, encode3, encode4, encode5, image, error, new_error, diff, boundary, bins = huffman(images)		---> 74 encode1, encode2, encode3, encode4, encode5, image, error, new_error, diff, boundary, bins = huffman(images)

	~\AppData\Local\Temp/ipykernel_2620/1618652474.py in huffman(image)		~\AppData\Local\Temp/ipykernel_2620/1618652474.py in huffman(image)
	18		18
	19 mask = diff <= 25		19 mask = diff <= 25
	---> 20 string = [str(i) for i in error[mask].astype(int)]		---> 20 string = [str(i) for i in error[mask].astype(int)]
	21 freq = dict(Counter(string))		21 freq = dict(Counter(string))
	22 freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)		22 freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)
	IndexError: boolean index did not match indexed array along dimension 0; dimension is 510 but corresponding boolean dimension is 512		IndexError: boolean index did not match indexed array along dimension 0; dimension is 510 but corresponding boolean dimension is 512

	%% Cell type:code id:e98fc3cf tags:		%% Cell type:code id:e98fc3cf tags:

	``` python		``` python
	print(boundary)		print(boundary)
	```		```

	%% Output		%% Output

	[22541 -10 14 ... 62 151 208]		[22541 -10 14 ... 62 151 208]

	%% Cell type:code id:f5e71acc tags:		%% Cell type:code id:f5e71acc tags:

	``` python		``` python
	```		```

	%% Cell type:code id:642b95a3 tags:		%% Cell type:code id:642b95a3 tags:

	``` python		``` python
	def compress_rate(image, error, diff, bound, encode1, encode2, encode3, encode4, encode5):		def compress_rate(image, error, diff, bound, encode1, encode2, encode3, encode4, encode5):
	#original = original.reshape(-1)		#original = original.reshape(-1)
	#error = error.reshape(-1)		#error = error.reshape(-1)
	o_len = 0		o_len = 0
	c_len = 0		c_len = 0
	im = np.reshape(image,(512, 640))		im = np.reshape(image,(512, 640))
	real_b = np.hstack((im[0,:],im[-1,:],im[1:-1,0],im[1:-1,-1]))		real_b = np.hstack((im[0,:],im[-1,:],im[1:-1,0],im[1:-1,-1]))
	original = im[1:-1,1:-1].reshape(-1)		original = im[1:-1,1:-1].reshape(-1)

	for i in range(0,len(bound)):		for i in range(0,len(bound)):
	o_len += len(bin(real_b[i])[2:])		o_len += len(bin(real_b[i])[2:])
	c_len += len(encode1[str(bound[i])])		c_len += len(encode1[str(bound[i])])

	for i in range(0, len(original)):		for i in range(0, len(original)):
	o_len += len(bin(original[i])[2:])		o_len += len(bin(original[i])[2:])
	if diff[i] <= 10:		if diff[i] <= 10:
	c_len += len(encode2[str(int(error[i]))])		c_len += len(encode2[str(int(error[i]))])

	if diff[i] <= 25 and diff[i] > 10:		if diff[i] <= 25 and diff[i] > 10:
	c_len += len(encode3[str(int(error[i]))])		c_len += len(encode3[str(int(error[i]))])

	if diff[i] <= 45 and diff[i] > 25:		if diff[i] <= 45 and diff[i] > 25:
	c_len += len(encode4[str(int(error[i]))])		c_len += len(encode4[str(int(error[i]))])

	if diff[i] > 45:		if diff[i] > 45:
	c_len += len(encode5[str(int(error[i]))])		c_len += len(encode5[str(int(error[i]))])

	return c_len/o_len		return c_len/o_len
	compress_rate(origin, error, diff, boundary, encode1, encode2, encode3, encode4, encode5)		compress_rate(origin, error, diff, boundary, encode1, encode2, encode3, encode4, encode5)
	```		```

	%% Output		%% Output

	0.4427516682942708		0.4427516682942708

	%% Cell type:code id:7d507cfb tags:		%% Cell type:code id:7d507cfb tags:

	``` python		``` python
	def encode_multiple(error, diff, bound, encode1, encode2, encode3, encode4, encode5):		def encode_multiple(error, diff, bound, encode1, encode2, encode3, encode4, encode5):
	#original = original.reshape(-1)		#original = original.reshape(-1)
	#error = error.reshape(-1)		#error = error.reshape(-1)
	original = len(np.ravel(error))		original = len(np.ravel(error))
	error = np.ravel(error)		error = np.ravel(error)
	encode_error = error.astype(str).astype(object).copy()		encode_error = error.astype(str).astype(object).copy()
	bound_error = bound.astype(str).astype(object).copy()		bound_error = bound.astype(str).astype(object).copy()


	for i in range(0,len(bound_error)):		for i in range(0,len(bound_error)):
	bound_error[i] = encode1[bound_error[i]]		bound_error[i] = encode1[bound_error[i]]

	for i in range(0, original):		for i in range(0, original):
	if diff[i] <= 10:		if diff[i] <= 10:
	encode_error[i] = encode2[encode_error[i]]		encode_error[i] = encode2[encode_error[i]]

	if diff[i] <= 25 and diff[i] > 10:		if diff[i] <= 25 and diff[i] > 10:
	encode_error[i] = encode3[encode_error[i]]		encode_error[i] = encode3[encode_error[i]]

	if diff[i] <= 45 and diff[i] > 25:		if diff[i] <= 45 and diff[i] > 25:
	encode_error[i] = encode4[encode_error[i]]		encode_error[i] = encode4[encode_error[i]]

	if diff[i] > 45:		if diff[i] > 45:
	encode_error[i] = encode5[encode_error[i]]		encode_error[i] = encode5[encode_error[i]]

	encode_error = np.pad(encode_error.reshape(510,638), pad_width=1)		encode_error = np.pad(encode_error.reshape(510,638), pad_width=1)
	encode_error[0] = bound_error[:640]		encode_error[0] = bound_error[:640]
	encode_error[-1] = bound_error[640:640*2]		encode_error[-1] = bound_error[640:640*2]
	encode_error[1:-1,0] = bound_error[6402:(6402)+510]		encode_error[1:-1,0] = bound_error[6402:(6402)+510]
	encode_error[1:-1,-1] = bound_error[(640*2)+510:]		encode_error[1:-1,-1] = bound_error[(640*2)+510:]

	return encode_error, bound_error		return encode_error, bound_error
	enc_mat, bound_e = encode_multiple(error, diff, boundary, encode1, encode2, encode3, encode4, encode5)		enc_mat, bound_e = encode_multiple(error, diff, boundary, encode1, encode2, encode3, encode4, encode5)
	```		```

	%% Cell type:code id:2faf5cd9 tags:		%% Cell type:code id:2faf5cd9 tags:

	``` python		``` python
	def decode_multi(A, encoded_matrix, encode1, encode2, encode3, encode4, encode5, diff):		def decode_multi(A, encoded_matrix, encode1, encode2, encode3, encode4, encode5, diff):
	"""		"""
	Function that accecpts the prediction matrix A for the linear system,		Function that accecpts the prediction matrix A for the linear system,
	the encoded matrix of error values, and the encoding dicitonary.		the encoded matrix of error values, and the encoding dicitonary.
	"""		"""


	the_keys1 = list(encode1.keys())		the_keys1 = list(encode1.keys())
	the_values1 = list(encode1.values())		the_values1 = list(encode1.values())
	the_keys2 = list(encode2.keys())		the_keys2 = list(encode2.keys())
	the_values2 = list(encode2.values())		the_values2 = list(encode2.values())
	the_keys3 = list(encode3.keys())		the_keys3 = list(encode3.keys())
	the_values3 = list(encode3.values())		the_values3 = list(encode3.values())
	the_keys4 = list(encode4.keys())		the_keys4 = list(encode4.keys())
	the_values4 = list(encode4.values())		the_values4 = list(encode4.values())
	the_keys5 = list(encode5.keys())		the_keys5 = list(encode5.keys())
	the_values5 = list(encode5.values())		the_values5 = list(encode5.values())

	error_matrix = encoded_matrix.copy()		error_matrix = encoded_matrix.copy()

	for i in range(error_matrix.shape[0]):		for i in range(error_matrix.shape[0]):
	for j in range(error_matrix.shape[1]):		for j in range(error_matrix.shape[1]):
	if i == 0 and j == 0:		if i == 0 and j == 0:
	error_matrix[i][j] = int(the_keys1[the_values1.index(encoded_matrix[i,j])])		error_matrix[i][j] = int(the_keys1[the_values1.index(encoded_matrix[i,j])])


	elif i == 0 or i == error_matrix.shape[0]-1 or j == 0 or j == error_matrix.shape[1]-1:		elif i == 0 or i == error_matrix.shape[0]-1 or j == 0 or j == error_matrix.shape[1]-1:
	error_matrix[i][j] = int(the_keys1[the_values1.index(error_matrix[i,j])]) + int(error_matrix[0][0])		error_matrix[i][j] = int(the_keys1[the_values1.index(error_matrix[i,j])]) + int(error_matrix[0][0])
	else:		else:
	if diff[i*640 + j] <= 10:		if diff[i*640 + j] <= 10:
	error_matrix[i][j] = int(the_keys2[the_values2.index(error_matrix[i,j])])		error_matrix[i][j] = int(the_keys2[the_values2.index(error_matrix[i,j])])
	elif diff[i640 + j] > 10 and diff[i640 + j] <= 25:		elif diff[i640 + j] > 10 and diff[i640 + j] <= 25:
	error_matrix[i,j] = int(the_keys3[the_values3.index(error_matrix[i,j])])		error_matrix[i,j] = int(the_keys3[the_values3.index(error_matrix[i,j])])
	elif diff[i640 + j] > 25 and diff[i640 + j] <= 45:		elif diff[i640 + j] > 25 and diff[i640 + j] <= 45:
	if error_matrix[i,j] == '101011':		if error_matrix[i,j] == '101011':
	print(i,j)		print(i,j)

	error_matrix[i,j] = int(the_keys4[the_values4.index(error_matrix[i,j])])		error_matrix[i,j] = int(the_keys4[the_values4.index(error_matrix[i,j])])
	elif diff[i*640 + j] > 45:		elif diff[i*640 + j] > 45:
	error_matrix[i,j] = int(the_keys5[the_values5.index(error_matrix[i,j])])		error_matrix[i,j] = int(the_keys5[the_values5.index(error_matrix[i,j])])


	return error_matrix.astype(int)		return error_matrix.astype(int)

	dec = decode_multi(A, enc_mat, encode1, encode2, encode3, encode4, encode5, diff)		dec = decode_multi(A, enc_mat, encode1, encode2, encode3, encode4, encode5, diff)
	```		```

	%% Output		%% Output

	1 1		1 1

	---------------------------------------------------------------------------		---------------------------------------------------------------------------
	ValueError Traceback (most recent call last)		ValueError Traceback (most recent call last)
	~\AppData\Local\Temp/ipykernel_1700/1235154671.py in <module>		~\AppData\Local\Temp/ipykernel_1700/1235154671.py in <module>
	42 return error_matrix.astype(int)		42 return error_matrix.astype(int)
	43		43
	---> 44 dec = decode_multi(A, enc_mat, encode1, encode2, encode3, encode4, encode5, diff)		---> 44 dec = decode_multi(A, enc_mat, encode1, encode2, encode3, encode4, encode5, diff)

	~\AppData\Local\Temp/ipykernel_1700/1235154671.py in decode_multi(A, encoded_matrix, encode1, encode2, encode3, encode4, encode5, diff)		~\AppData\Local\Temp/ipykernel_1700/1235154671.py in decode_multi(A, encoded_matrix, encode1, encode2, encode3, encode4, encode5, diff)
	35 if error_matrix[i,j] == '101011':		35 if error_matrix[i,j] == '101011':
	36 print(i,j)		36 print(i,j)
	---> 37 error_matrix[i,j] = int(the_keys4[the_values4.index(error_matrix[i,j])])		---> 37 error_matrix[i,j] = int(the_keys4[the_values4.index(error_matrix[i,j])])
	38 elif diff[i*640 + j] > 45:		38 elif diff[i*640 + j] > 45:
	39 error_matrix[i,j] = int(the_keys5[the_values5.index(error_matrix[i,j])])		39 error_matrix[i,j] = int(the_keys5[the_values5.index(error_matrix[i,j])])
	ValueError: '101011' is not in list		ValueError: '101011' is not in list

	%% Cell type:code id:64832ca7 tags:		%% Cell type:code id:64832ca7 tags:

	``` python		``` python

	"""plt.hexbin(x,y,cmap="rocket")		"""plt.hexbin(x,y,cmap="rocket")
	plt.colorbar()		plt.colorbar()
	plt.xlim(0,50)		plt.xlim(0,50)
	plt.ylim(0,100)"""		plt.ylim(0,100)"""

	def rel_freq(x):		def rel_freq(x):
	freqs = [x.count(value) / len(x) for value in set(x)]		freqs = [x.count(value) / len(x) for value in set(x)]
	return freqs		return freqs
	print(sp.stats.entropy(rel_freq(list(np.ravel(o)))))		print(sp.stats.entropy(rel_freq(list(np.ravel(image)))))

	def entropy_check(x, y):		def entropy_check(x, y):
	#freq = rel_freq(list(np.ravel(o)))		#freq = rel_freq(list(np.ravel(o)))
	means = []		means = []
	for i in range(len(images)):		for i in range(len(images)):
	p, d, o, e, A = predict(images,0)		p, d, o, e, A = predict(images,0)
	d = d.reshape((510,638))		d = d.reshape((510,638))
	x = np.abs(np.ravel(e))		x = np.abs(np.ravel(e))
	y = np.ravel(d)		y = np.ravel(d)

	mask1 = y <= 25		mask1 = y <= 25
	x_masked1 = x[mask1]		x_masked1 = x[mask1]

	mask2 = y > 25		mask2 = y > 25
	x_masked2 = x[mask2]		x_masked2 = x[mask2]
	mask2 = y[mask2] <= 40		mask2 = y[mask2] <= 40
	x_masked2 = x_masked2[mask2]		x_masked2 = x_masked2[mask2]

	mask3 = y > 40		mask3 = y > 40
	x_masked3 = x[mask3]		x_masked3 = x[mask3]
	mask3 = y[mask3] <= 75		mask3 = y[mask3] <= 75
	x_masked3 = x_masked3[mask3]		x_masked3 = x_masked3[mask3]

	mask4 = y > 75		mask4 = y > 75
	x_masked4 = x[mask4]		x_masked4 = x[mask4]


	e_m1 = sp.stats.entropy(rel_freq(list(x_masked1)))		e_m1 = sp.stats.entropy(rel_freq(list(x_masked1)))
	e_m2 = sp.stats.entropy(rel_freq(list(x_masked2)))		e_m2 = sp.stats.entropy(rel_freq(list(x_masked2)))
	e_m3 = sp.stats.entropy(rel_freq(list(x_masked3)))		e_m3 = sp.stats.entropy(rel_freq(list(x_masked3)))
	e_m4 = sp.stats.entropy(rel_freq(list(x_masked4)))		e_m4 = sp.stats.entropy(rel_freq(list(x_masked4)))
	means.append([e_m1, e_m2, e_m3, e_m4])		means.append([e_m1, e_m2, e_m3, e_m4])
	return np.mean(np.array(means).reshape(len(images),4), axis=0)		return np.mean(np.array(means).reshape(len(images),4), axis=0)

	#print(entropy_check(x, y))		#print(entropy_check(x, y))
	```		```

	%% Output		%% Output

	6.736892561802416		[58 38 13 ... 65 97 32]

			---------------------------------------------------------------------------
			NameError Traceback (most recent call last)
			~\AppData\Local\Temp/ipykernel_2620/2795330121.py in <module>
			7 freqs = [x.count(value) / len(x) for value in set(x)]
			8 return freqs
			----> 9 print(sp.stats.entropy(rel_freq(list(np.ravel(o)))))
			10
			11 def entropy_check(x, y):
			NameError: name 'o' is not defined

.ipynb_checkpoints/Scout_Clean_Code_with_EncDec-checkpoint.ipynb

+9 −9

Original line number	Original line	Diff line number	Diff line
	%% Cell type:code id:0d67d099 tags:		%% Cell type:code id:5bb42c2c tags:

	``` python		``` python
	import numpy as np		import numpy as np
	from prediction_MSE_Scout import file_extractor, image_extractor, im_distribution		from prediction_MSE_Scout import file_extractor, image_extractor, im_distribution
	from matplotlib import pyplot as plt		from matplotlib import pyplot as plt
	from itertools import product		from itertools import product
	import os		import os
	import sys		import sys
	from PIL import Image		from PIL import Image
	from scipy.optimize import minimize		from scipy.optimize import minimize
	from time import time		from time import time
	from numpy import linalg as la		from numpy import linalg as la
	from scipy.stats import gaussian_kde		from scipy.stats import gaussian_kde
	import seaborn as sns		import seaborn as sns
	from collections import Counter		from collections import Counter
	import pandas as pd		import pandas as pd
	import scipy as sp		import scipy as sp
	```		```

	%% Cell type:code id:fc76b964 tags:		%% Cell type:code id:ec24fcba tags:

	``` python		``` python
	def plot_hist(tiff_list):		def plot_hist(tiff_list):
	"""		"""
	This function is the leftovers from the first attempt to plot histograms.		This function is the leftovers from the first attempt to plot histograms.
	As it stands it needs some work in order to function again. We will		As it stands it needs some work in order to function again. We will
	fix this later. 1/25/22		fix this later. 1/25/22
	"""		"""

	image = tiff_list		image = tiff_list
	image = Image.open(image) #Open the image and read it as an Image object		image = Image.open(image) #Open the image and read it as an Image object
	image = np.array(image)[1:,:] #Convert to an array, leaving out the first row because the first row is just housekeeping data		image = np.array(image)[1:,:] #Convert to an array, leaving out the first row because the first row is just housekeeping data
	image = image.astype(int)		image = image.astype(int)
	A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation		A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation
	z0 = image[0:-2,0:-2] # get all the first pixel for the entire image		z0 = image[0:-2,0:-2] # get all the first pixel for the entire image
	z1 = image[0:-2,1:-1] # get all the second pixel for the entire image		z1 = image[0:-2,1:-1] # get all the second pixel for the entire image
	z2 = image[0:-2,2::] # get all the third pixel for the entire image		z2 = image[0:-2,2::] # get all the third pixel for the entire image
	z3 = image[1:-1,0:-2] # get all the forth pixel for the entire image		z3 = image[1:-1,0:-2] # get all the forth pixel for the entire image
	# calculate the out put of the system of equation		# calculate the out put of the system of equation
	y0 = np.ravel(-z0+z2-z3)		y0 = np.ravel(-z0+z2-z3)
	y1 = np.ravel(z0+z1+z2)		y1 = np.ravel(z0+z1+z2)
	y2 = np.ravel(-z0-z1-z2-z3)		y2 = np.ravel(-z0-z1-z2-z3)
	y = np.vstack((y0,y1,y2))		y = np.vstack((y0,y1,y2))
	# use numpy solver to solve the system of equations all at once		# use numpy solver to solve the system of equations all at once
	#predict = np.floor(np.linalg.solve(A,y)[-1])		#predict = np.floor(np.linalg.solve(A,y)[-1])
	predict = np.round(np.round((np.linalg.solve(A,y)[-1]),1))		predict = np.round(np.round((np.linalg.solve(A,y)[-1]),1))
	# flatten the neighbor pixlels and stack them together		# flatten the neighbor pixlels and stack them together
	z0 = np.ravel(z0)		z0 = np.ravel(z0)
	z1 = np.ravel(z1)		z1 = np.ravel(z1)
	z2 = np.ravel(z2)		z2 = np.ravel(z2)
	z3 = np.ravel(z3)		z3 = np.ravel(z3)
	neighbor = np.vstack((z0,z1,z2,z3)).T		neighbor = np.vstack((z0,z1,z2,z3)).T
	# calculate the difference		# calculate the difference
	diff = np.max(neighbor,axis = 1) - np.min(neighbor, axis=1)		diff = np.max(neighbor,axis = 1) - np.min(neighbor, axis=1)

	# flatten the image to a vector		# flatten the image to a vector
	image = np.ravel(image[1:-1,1:-1])		image = np.ravel(image[1:-1,1:-1])
	error = image-predict		error = image-predict

	return image, predict, diff, error, A		return image, predict, diff, error, A
	```		```

	%% Cell type:code id:b781115b tags:		%% Cell type:code id:c2430512 tags:

	``` python		``` python
	class NodeTree(object):		class NodeTree(object):
	def __init__(self, left=None, right=None):		def __init__(self, left=None, right=None):
	self.left = left		self.left = left
	self.right = right		self.right = right

	def children(self):		def children(self):
	return self.left, self.right		return self.left, self.right

	def __str__(self):		def __str__(self):
	return self.left, self.right		return self.left, self.right


	def huffman_code_tree(node, binString=''):		def huffman_code_tree(node, binString=''):
	'''		'''
	Function to find Huffman Code		Function to find Huffman Code
	'''		'''
	if type(node) is str:		if type(node) is str:
	return {node: binString}		return {node: binString}
	(l, r) = node.children()		(l, r) = node.children()
	d = dict()		d = dict()
	d.update(huffman_code_tree(l, binString + '0'))		d.update(huffman_code_tree(l, binString + '0'))
	d.update(huffman_code_tree(r, binString + '1'))		d.update(huffman_code_tree(r, binString + '1'))
	return d		return d


	def make_tree(nodes):		def make_tree(nodes):
	'''		'''
	Function to make tree		Function to make tree
	:param nodes: Nodes		:param nodes: Nodes
	:return: Root of the tree		:return: Root of the tree
	'''		'''
	while len(nodes) > 1:		while len(nodes) > 1:
	(key1, c1) = nodes[-1]		(key1, c1) = nodes[-1]
	(key2, c2) = nodes[-2]		(key2, c2) = nodes[-2]
	nodes = nodes[:-2]		nodes = nodes[:-2]
	node = NodeTree(key1, key2)		node = NodeTree(key1, key2)
	nodes.append((node, c1 + c2))		nodes.append((node, c1 + c2))
	nodes = sorted(nodes, key=lambda x: x[1], reverse=True)		nodes = sorted(nodes, key=lambda x: x[1], reverse=True)
	return nodes[0][0]		return nodes[0][0]
	```		```

	%% Cell type:code id:fe145ec0 tags:		%% Cell type:code id:b973ed91 tags:

	``` python		``` python
	def huffman(image):		def huffman(image):
	origin, predict, diff, error, A = plot_hist(image)		origin, predict, diff, error, A = plot_hist(image)

	image = Image.open(image)		image = Image.open(image)
	image = np.array(image)[1:,:] #Convert to an array, leaving out the first row because the first row is just housekeeping data		image = np.array(image)[1:,:] #Convert to an array, leaving out the first row because the first row is just housekeeping data
	image = image.astype(int)		image = image.astype(int)

	boundary = np.hstack((image[0,:],image[-1,:],image[1:-1,0],image[1:-1,-1]))		boundary = np.hstack((image[0,:],image[-1,:],image[1:-1,0],image[1:-1,-1]))
	boundary = boundary - image[0,0]		boundary = boundary - image[0,0]
	boundary[0] = image[0,0]		boundary[0] = image[0,0]

	string = [str(i) for i in boundary]		string = [str(i) for i in boundary]
	freq = dict(Counter(string))		freq = dict(Counter(string))
	freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)		freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)
	node = make_tree(freq)		node = make_tree(freq)
	encode1 = huffman_code_tree(node)		encode1 = huffman_code_tree(node)


	mask = diff <= 25		mask = diff <= 25
	string = [str(i) for i in error[mask].astype(int)]		string = [str(i) for i in error[mask].astype(int)]
	freq = dict(Counter(string))		freq = dict(Counter(string))
	freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)		freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)
	node = make_tree(freq)		node = make_tree(freq)
	encode2 = huffman_code_tree(node)		encode2 = huffman_code_tree(node)


	mask = diff > 25		mask = diff > 25
	new_error = error[mask]		new_error = error[mask]
	mask2 = diff[mask] <= 40		mask2 = diff[mask] <= 40
	string = [str(i) for i in new_error[mask2].astype(int)]		string = [str(i) for i in new_error[mask2].astype(int)]
	freq = dict(Counter(string))		freq = dict(Counter(string))
	freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)		freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)
	node = make_tree(freq)		node = make_tree(freq)
	encode3 = huffman_code_tree(node)		encode3 = huffman_code_tree(node)


	mask = diff > 40		mask = diff > 40
	new_error = error[mask]		new_error = error[mask]
	mask2 = diff[mask] <= 70		mask2 = diff[mask] <= 70
	string = [str(i) for i in new_error[mask2].astype(int)]		string = [str(i) for i in new_error[mask2].astype(int)]
	freq = dict(Counter(string))		freq = dict(Counter(string))
	freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)		freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)
	node = make_tree(freq)		node = make_tree(freq)
	encode4 = huffman_code_tree(node)		encode4 = huffman_code_tree(node)


	mask = diff > 70		mask = diff > 70
	string = [str(i) for i in error[mask].astype(int)]		string = [str(i) for i in error[mask].astype(int)]
	freq = dict(Counter(string))		freq = dict(Counter(string))
	freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)		freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)
	node = make_tree(freq)		node = make_tree(freq)
	encode5 = huffman_code_tree(node)		encode5 = huffman_code_tree(node)


	new_error = np.copy(image)		new_error = np.copy(image)
	new_error[1:-1,1:-1] = np.reshape(error,(510, 638))		new_error[1:-1,1:-1] = np.reshape(error,(510, 638))
	keep = new_error[0,0]		keep = new_error[0,0]
	new_error[0,:] = new_error[0,:] - keep		new_error[0,:] = new_error[0,:] - keep
	new_error[-1,:] = new_error[-1,:] - keep		new_error[-1,:] = new_error[-1,:] - keep
	new_error[1:-1,0] = new_error[1:-1,0] - keep		new_error[1:-1,0] = new_error[1:-1,0] - keep
	new_error[1:-1,-1] = new_error[1:-1,-1] - keep		new_error[1:-1,-1] = new_error[1:-1,-1] - keep
	new_error[0,0] = keep		new_error[0,0] = keep


	#new_error = np.ravel(new_error)		#new_error = np.ravel(new_error)

	bins = [25,40,70]		bins = [25,40,70]

	# return the huffman dictionary		# return the huffman dictionary
	return encode1, encode2, encode3, encode4, encode5, np.ravel(image), error, new_error, diff, boundary, bins, predict, A		return encode1, encode2, encode3, encode4, encode5, np.ravel(image), error, new_error, diff, boundary, bins, predict, A

	```		```

	%% Cell type:code id:4fb8f5d0 tags:		%% Cell type:code id:4f6a5a0d tags:

	``` python		``` python
	def encoder(error, list_dic, diff, bound, bins):		def encoder(error, list_dic, diff, bound, bins):
	encoded = np.copy(error).astype(int).astype(str).astype(object)		encoded = np.copy(error).astype(int).astype(str).astype(object)

	diff = np.reshape(diff,(510,638))		diff = np.reshape(diff,(510,638))

	for i in range(encoded.shape[0]):		for i in range(encoded.shape[0]):
	for j in range(encoded.shape[1]):		for j in range(encoded.shape[1]):
	if i == 0 or i == encoded.shape[0]-1 or j == 0 or j == encoded.shape[1]-1:		if i == 0 or i == encoded.shape[0]-1 or j == 0 or j == encoded.shape[1]-1:
	encoded[i][j] = list_dic[0][encoded[i][j]]		encoded[i][j] = list_dic[0][encoded[i][j]]
	elif diff[i-1][j-1] <= bins[0]:		elif diff[i-1][j-1] <= bins[0]:
	encoded[i][j] = list_dic[1][encoded[i][j]]		encoded[i][j] = list_dic[1][encoded[i][j]]
	elif diff[i-1][j-1] <= bins[1] and diff[i-1][j-1] > bins[0]:		elif diff[i-1][j-1] <= bins[1] and diff[i-1][j-1] > bins[0]:
	encoded[i][j] = list_dic[2][encoded[i][j]]		encoded[i][j] = list_dic[2][encoded[i][j]]
	elif diff[i-1][j-1] <= bins[2] and diff[i-1][j-1] > bins[1]:		elif diff[i-1][j-1] <= bins[2] and diff[i-1][j-1] > bins[1]:
	encoded[i][j] = list_dic[3][encoded[i][j]]		encoded[i][j] = list_dic[3][encoded[i][j]]
	else:		else:
	encoded[i][j] = list_dic[4][encoded[i][j]]		encoded[i][j] = list_dic[4][encoded[i][j]]


	return encoded		return encoded
	```		```

	%% Cell type:code id:9a39a65b tags:		%% Cell type:code id:4b65c7e9 tags:

	``` python		``` python
	def decoder(A, encoded_matrix, list_dic, bins):		def decoder(A, encoded_matrix, list_dic, bins):
	"""		"""
	Function that accecpts the prediction matrix A for the linear system,		Function that accecpts the prediction matrix A for the linear system,
	the encoded matrix of error values, and the encoding dicitonary.		the encoded matrix of error values, and the encoding dicitonary.
	"""		"""

	the_keys0 = list(list_dic[0].keys())		the_keys0 = list(list_dic[0].keys())
	the_values0 = list(list_dic[0].values())		the_values0 = list(list_dic[0].values())

	the_keys1 = list(list_dic[1].keys())		the_keys1 = list(list_dic[1].keys())
	the_values1 = list(list_dic[1].values())		the_values1 = list(list_dic[1].values())

	the_keys2 = list(list_dic[2].keys())		the_keys2 = list(list_dic[2].keys())
	the_values2 = list(list_dic[2].values())		the_values2 = list(list_dic[2].values())

	the_keys3 = list(list_dic[3].keys())		the_keys3 = list(list_dic[3].keys())
	the_values3 = list(list_dic[3].values())		the_values3 = list(list_dic[3].values())

	the_keys4 = list(list_dic[4].keys())		the_keys4 = list(list_dic[4].keys())
	the_values4 = list(list_dic[4].values())		the_values4 = list(list_dic[4].values())

	error_matrix = np.zeros((512,640))		error_matrix = np.zeros((512,640))

	for i in range(error_matrix.shape[0]):		for i in range(error_matrix.shape[0]):
	for j in range(error_matrix.shape[1]):		for j in range(error_matrix.shape[1]):
	if i == 0 and j == 0:		if i == 0 and j == 0:
	error_matrix[i][j] = int(the_keys0[the_values0.index(encoded_matrix[i,j])])		error_matrix[i][j] = int(the_keys0[the_values0.index(encoded_matrix[i,j])])

	elif i == 0 or i == error_matrix.shape[0]-1 or j == 0 or j == error_matrix.shape[1]-1:		elif i == 0 or i == error_matrix.shape[0]-1 or j == 0 or j == error_matrix.shape[1]-1:
	error_matrix[i][j] = int(the_keys0[the_values0.index(encoded_matrix[i,j])]) + error_matrix[0][0]		error_matrix[i][j] = int(the_keys0[the_values0.index(encoded_matrix[i,j])]) + error_matrix[0][0]
	else:		else:
	z0 = error_matrix[i-1][j-1]		z0 = error_matrix[i-1][j-1]
	z1 = error_matrix[i-1][j]		z1 = error_matrix[i-1][j]
	z2 = error_matrix[i-1][j+1]		z2 = error_matrix[i-1][j+1]
	z3 = error_matrix[i][j-1]		z3 = error_matrix[i][j-1]
	y0 = int(-z0+z2-z3)		y0 = int(-z0+z2-z3)
	y1 = int(z0+z1+z2)		y1 = int(z0+z1+z2)
	y2 = int(-z0-z1-z2-z3)		y2 = int(-z0-z1-z2-z3)
	y = np.vstack((y0,y1,y2))		y = np.vstack((y0,y1,y2))
	difference = max(z0,z1,z2,z3) - min(z0,z1,z2,z3)		difference = max(z0,z1,z2,z3) - min(z0,z1,z2,z3)
	predict = np.round(np.round(np.linalg.solve(A,y)[-1][0],1))		predict = np.round(np.round(np.linalg.solve(A,y)[-1][0],1))

	if difference <= bins[0]:		if difference <= bins[0]:
	error_matrix[i][j] = int(the_keys1[the_values1.index(encoded_matrix[i,j])]) + int(predict)		error_matrix[i][j] = int(the_keys1[the_values1.index(encoded_matrix[i,j])]) + int(predict)
	elif difference <= bins[1] and difference > bins[0]:		elif difference <= bins[1] and difference > bins[0]:
	error_matrix[i][j] = int(the_keys2[the_values2.index(encoded_matrix[i,j])]) + int(predict)		error_matrix[i][j] = int(the_keys2[the_values2.index(encoded_matrix[i,j])]) + int(predict)
	elif difference <= bins[2] and difference > bins[1]:		elif difference <= bins[2] and difference > bins[1]:
	error_matrix[i][j] = int(the_keys3[the_values3.index(encoded_matrix[i,j])]) + int(predict)		error_matrix[i][j] = int(the_keys3[the_values3.index(encoded_matrix[i,j])]) + int(predict)
	else:		else:
	error_matrix[i][j] = int(the_keys4[the_values4.index(encoded_matrix[i,j])]) + int(predict)		error_matrix[i][j] = int(the_keys4[the_values4.index(encoded_matrix[i,j])]) + int(predict)


	return error_matrix.astype(int)		return error_matrix.astype(int)
	```		```

	%% Cell type:code id:fc4d80bd tags:		%% Cell type:code id:280aafd3 tags:

	``` python		``` python
	scenes = file_extractor()		scenes = file_extractor()
	images = image_extractor(scenes)		images = image_extractor(scenes)
	encode1, encode2, encode3, encode4, encode5, image, error, new_error, diff, bound, bins, predict, A = huffman(images[0])		encode1, encode2, encode3, encode4, encode5, image, error, new_error, diff, bound, bins, predict, A = huffman(images[0])
	encoded_matrix = encoder(np.reshape(new_error,(512,640)), [encode1, encode2, encode3, encode4, encode5], diff, bound, bins)		encoded_matrix = encoder(np.reshape(new_error,(512,640)), [encode1, encode2, encode3, encode4, encode5], diff, bound, bins)
	list_dic = [encode1, encode2, encode3, encode4, encode5]		list_dic = [encode1, encode2, encode3, encode4, encode5]
	```		```

	%% Cell type:code id:2d82e61a tags:		%% Cell type:code id:c4242b52 tags:

	``` python		``` python
	reconstruct_image = decoder(A, encoded_matrix, list_dic, bins)		reconstruct_image = decoder(A, encoded_matrix, list_dic, bins)
	np.allclose(image.reshape(512,640), reconstruct_image)		np.allclose(image.reshape(512,640), reconstruct_image)
	```		```

	%% Output		%% Output

	True		True

	%% Cell type:code id:e35be607 tags:		%% Cell type:code id:a9502e22 tags:

	``` python		``` python

	"""plt.hexbin(x,y,cmap="rocket")		"""plt.hexbin(x,y,cmap="rocket")
	plt.colorbar()		plt.colorbar()
	plt.xlim(0,50)		plt.xlim(0,50)
	plt.ylim(0,100)"""		plt.ylim(0,100)"""

	def rel_freq(x):		def rel_freq(x):
	freqs = [x.count(value) / len(x) for value in set(x)]		freqs = [x.count(value) / len(x) for value in set(x)]
	return freqs		return freqs
	print(sp.stats.entropy(rel_freq(list(diff))))		print(sp.stats.entropy(rel_freq(list(diff))))

	def entropy_check(x, y):		def entropy_check(x, y):
	#freq = rel_freq(list(np.ravel(o)))		#freq = rel_freq(list(np.ravel(o)))
	means = []		means = []
	for i in range(len(images)):		for i in range(len(images)):
	p, d, o, e, A = predict(images,0)		p, d, o, e, A = predict(images,0)
	d = d.reshape((510,638))		d = d.reshape((510,638))
	x = np.abs(np.ravel(e))		x = np.abs(np.ravel(e))
	y = np.ravel(d)		y = np.ravel(d)

	mask1 = y <= 25		mask1 = y <= 25
	x_masked1 = x[mask1]		x_masked1 = x[mask1]

	mask2 = y > 25		mask2 = y > 25
	x_masked2 = x[mask2]		x_masked2 = x[mask2]
	mask2 = y[mask2] <= 40		mask2 = y[mask2] <= 40
	x_masked2 = x_masked2[mask2]		x_masked2 = x_masked2[mask2]

	mask3 = y > 40		mask3 = y > 40
	x_masked3 = x[mask3]		x_masked3 = x[mask3]
	mask3 = y[mask3] <= 75		mask3 = y[mask3] <= 75
	x_masked3 = x_masked3[mask3]		x_masked3 = x_masked3[mask3]

	mask4 = y > 75		mask4 = y > 75
	x_masked4 = x[mask4]		x_masked4 = x[mask4]


	e_m1 = sp.stats.entropy(rel_freq(list(x_masked1)))		e_m1 = sp.stats.entropy(rel_freq(list(x_masked1)))
	e_m2 = sp.stats.entropy(rel_freq(list(x_masked2)))		e_m2 = sp.stats.entropy(rel_freq(list(x_masked2)))
	e_m3 = sp.stats.entropy(rel_freq(list(x_masked3)))		e_m3 = sp.stats.entropy(rel_freq(list(x_masked3)))
	e_m4 = sp.stats.entropy(rel_freq(list(x_masked4)))		e_m4 = sp.stats.entropy(rel_freq(list(x_masked4)))
	means.append([e_m1, e_m2, e_m3, e_m4])		means.append([e_m1, e_m2, e_m3, e_m4])
	return np.mean(np.array(means).reshape(len(images),4), axis=0)		return np.mean(np.array(means).reshape(len(images),4), axis=0)

	#print(entropy_check(x, y))		#print(entropy_check(x, y))
	```		```

	%% Output		%% Output

	4.720647237500972		4.720647237500972

Error_to_Image.ipynb

+2 −2

Scout_Clean_Code_with_EncDec.ipynb

+9 −9

Original line number	Original line	Diff line number	Diff line
	%% Cell type:code id:0d67d099 tags:		%% Cell type:code id:5bb42c2c tags:

	``` python		``` python
	import numpy as np		import numpy as np
	from prediction_MSE_Scout import file_extractor, image_extractor, im_distribution		from prediction_MSE_Scout import file_extractor, image_extractor, im_distribution
	from matplotlib import pyplot as plt		from matplotlib import pyplot as plt
	from itertools import product		from itertools import product
	import os		import os
	import sys		import sys
	from PIL import Image		from PIL import Image
	from scipy.optimize import minimize		from scipy.optimize import minimize
	from time import time		from time import time
	from numpy import linalg as la		from numpy import linalg as la
	from scipy.stats import gaussian_kde		from scipy.stats import gaussian_kde
	import seaborn as sns		import seaborn as sns
	from collections import Counter		from collections import Counter
	import pandas as pd		import pandas as pd
	import scipy as sp		import scipy as sp
	```		```

	%% Cell type:code id:fc76b964 tags:		%% Cell type:code id:ec24fcba tags:

	``` python		``` python
	def plot_hist(tiff_list):		def plot_hist(tiff_list):
	"""		"""
	This function is the leftovers from the first attempt to plot histograms.		This function is the leftovers from the first attempt to plot histograms.
	As it stands it needs some work in order to function again. We will		As it stands it needs some work in order to function again. We will
	fix this later. 1/25/22		fix this later. 1/25/22
	"""		"""

	image = tiff_list		image = tiff_list
	image = Image.open(image) #Open the image and read it as an Image object		image = Image.open(image) #Open the image and read it as an Image object
	image = np.array(image)[1:,:] #Convert to an array, leaving out the first row because the first row is just housekeeping data		image = np.array(image)[1:,:] #Convert to an array, leaving out the first row because the first row is just housekeeping data
	image = image.astype(int)		image = image.astype(int)
	A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation		A = np.array([[3,0,-1],[0,3,3],[1,-3,-4]]) # the matrix for system of equation
	z0 = image[0:-2,0:-2] # get all the first pixel for the entire image		z0 = image[0:-2,0:-2] # get all the first pixel for the entire image
	z1 = image[0:-2,1:-1] # get all the second pixel for the entire image		z1 = image[0:-2,1:-1] # get all the second pixel for the entire image
	z2 = image[0:-2,2::] # get all the third pixel for the entire image		z2 = image[0:-2,2::] # get all the third pixel for the entire image
	z3 = image[1:-1,0:-2] # get all the forth pixel for the entire image		z3 = image[1:-1,0:-2] # get all the forth pixel for the entire image
	# calculate the out put of the system of equation		# calculate the out put of the system of equation
	y0 = np.ravel(-z0+z2-z3)		y0 = np.ravel(-z0+z2-z3)
	y1 = np.ravel(z0+z1+z2)		y1 = np.ravel(z0+z1+z2)
	y2 = np.ravel(-z0-z1-z2-z3)		y2 = np.ravel(-z0-z1-z2-z3)
	y = np.vstack((y0,y1,y2))		y = np.vstack((y0,y1,y2))
	# use numpy solver to solve the system of equations all at once		# use numpy solver to solve the system of equations all at once
	#predict = np.floor(np.linalg.solve(A,y)[-1])		#predict = np.floor(np.linalg.solve(A,y)[-1])
	predict = np.round(np.round((np.linalg.solve(A,y)[-1]),1))		predict = np.round(np.round((np.linalg.solve(A,y)[-1]),1))
	# flatten the neighbor pixlels and stack them together		# flatten the neighbor pixlels and stack them together
	z0 = np.ravel(z0)		z0 = np.ravel(z0)
	z1 = np.ravel(z1)		z1 = np.ravel(z1)
	z2 = np.ravel(z2)		z2 = np.ravel(z2)
	z3 = np.ravel(z3)		z3 = np.ravel(z3)
	neighbor = np.vstack((z0,z1,z2,z3)).T		neighbor = np.vstack((z0,z1,z2,z3)).T
	# calculate the difference		# calculate the difference
	diff = np.max(neighbor,axis = 1) - np.min(neighbor, axis=1)		diff = np.max(neighbor,axis = 1) - np.min(neighbor, axis=1)

	# flatten the image to a vector		# flatten the image to a vector
	image = np.ravel(image[1:-1,1:-1])		image = np.ravel(image[1:-1,1:-1])
	error = image-predict		error = image-predict

	return image, predict, diff, error, A		return image, predict, diff, error, A
	```		```

	%% Cell type:code id:b781115b tags:		%% Cell type:code id:c2430512 tags:

	``` python		``` python
	class NodeTree(object):		class NodeTree(object):
	def __init__(self, left=None, right=None):		def __init__(self, left=None, right=None):
	self.left = left		self.left = left
	self.right = right		self.right = right

	def children(self):		def children(self):
	return self.left, self.right		return self.left, self.right

	def __str__(self):		def __str__(self):
	return self.left, self.right		return self.left, self.right


	def huffman_code_tree(node, binString=''):		def huffman_code_tree(node, binString=''):
	'''		'''
	Function to find Huffman Code		Function to find Huffman Code
	'''		'''
	if type(node) is str:		if type(node) is str:
	return {node: binString}		return {node: binString}
	(l, r) = node.children()		(l, r) = node.children()
	d = dict()		d = dict()
	d.update(huffman_code_tree(l, binString + '0'))		d.update(huffman_code_tree(l, binString + '0'))
	d.update(huffman_code_tree(r, binString + '1'))		d.update(huffman_code_tree(r, binString + '1'))
	return d		return d


	def make_tree(nodes):		def make_tree(nodes):
	'''		'''
	Function to make tree		Function to make tree
	:param nodes: Nodes		:param nodes: Nodes
	:return: Root of the tree		:return: Root of the tree
	'''		'''
	while len(nodes) > 1:		while len(nodes) > 1:
	(key1, c1) = nodes[-1]		(key1, c1) = nodes[-1]
	(key2, c2) = nodes[-2]		(key2, c2) = nodes[-2]
	nodes = nodes[:-2]		nodes = nodes[:-2]
	node = NodeTree(key1, key2)		node = NodeTree(key1, key2)
	nodes.append((node, c1 + c2))		nodes.append((node, c1 + c2))
	nodes = sorted(nodes, key=lambda x: x[1], reverse=True)		nodes = sorted(nodes, key=lambda x: x[1], reverse=True)
	return nodes[0][0]		return nodes[0][0]
	```		```

	%% Cell type:code id:fe145ec0 tags:		%% Cell type:code id:b973ed91 tags:

	``` python		``` python
	def huffman(image):		def huffman(image):
	origin, predict, diff, error, A = plot_hist(image)		origin, predict, diff, error, A = plot_hist(image)

	image = Image.open(image)		image = Image.open(image)
	image = np.array(image)[1:,:] #Convert to an array, leaving out the first row because the first row is just housekeeping data		image = np.array(image)[1:,:] #Convert to an array, leaving out the first row because the first row is just housekeeping data
	image = image.astype(int)		image = image.astype(int)

	boundary = np.hstack((image[0,:],image[-1,:],image[1:-1,0],image[1:-1,-1]))		boundary = np.hstack((image[0,:],image[-1,:],image[1:-1,0],image[1:-1,-1]))
	boundary = boundary - image[0,0]		boundary = boundary - image[0,0]
	boundary[0] = image[0,0]		boundary[0] = image[0,0]

	string = [str(i) for i in boundary]		string = [str(i) for i in boundary]
	freq = dict(Counter(string))		freq = dict(Counter(string))
	freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)		freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)
	node = make_tree(freq)		node = make_tree(freq)
	encode1 = huffman_code_tree(node)		encode1 = huffman_code_tree(node)


	mask = diff <= 25		mask = diff <= 25
	string = [str(i) for i in error[mask].astype(int)]		string = [str(i) for i in error[mask].astype(int)]
	freq = dict(Counter(string))		freq = dict(Counter(string))
	freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)		freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)
	node = make_tree(freq)		node = make_tree(freq)
	encode2 = huffman_code_tree(node)		encode2 = huffman_code_tree(node)


	mask = diff > 25		mask = diff > 25
	new_error = error[mask]		new_error = error[mask]
	mask2 = diff[mask] <= 40		mask2 = diff[mask] <= 40
	string = [str(i) for i in new_error[mask2].astype(int)]		string = [str(i) for i in new_error[mask2].astype(int)]
	freq = dict(Counter(string))		freq = dict(Counter(string))
	freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)		freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)
	node = make_tree(freq)		node = make_tree(freq)
	encode3 = huffman_code_tree(node)		encode3 = huffman_code_tree(node)


	mask = diff > 40		mask = diff > 40
	new_error = error[mask]		new_error = error[mask]
	mask2 = diff[mask] <= 70		mask2 = diff[mask] <= 70
	string = [str(i) for i in new_error[mask2].astype(int)]		string = [str(i) for i in new_error[mask2].astype(int)]
	freq = dict(Counter(string))		freq = dict(Counter(string))
	freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)		freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)
	node = make_tree(freq)		node = make_tree(freq)
	encode4 = huffman_code_tree(node)		encode4 = huffman_code_tree(node)


	mask = diff > 70		mask = diff > 70
	string = [str(i) for i in error[mask].astype(int)]		string = [str(i) for i in error[mask].astype(int)]
	freq = dict(Counter(string))		freq = dict(Counter(string))
	freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)		freq = sorted(freq.items(), key=lambda x: x[1], reverse=True)
	node = make_tree(freq)		node = make_tree(freq)
	encode5 = huffman_code_tree(node)		encode5 = huffman_code_tree(node)


	new_error = np.copy(image)		new_error = np.copy(image)
	new_error[1:-1,1:-1] = np.reshape(error,(510, 638))		new_error[1:-1,1:-1] = np.reshape(error,(510, 638))
	keep = new_error[0,0]		keep = new_error[0,0]
	new_error[0,:] = new_error[0,:] - keep		new_error[0,:] = new_error[0,:] - keep
	new_error[-1,:] = new_error[-1,:] - keep		new_error[-1,:] = new_error[-1,:] - keep
	new_error[1:-1,0] = new_error[1:-1,0] - keep		new_error[1:-1,0] = new_error[1:-1,0] - keep
	new_error[1:-1,-1] = new_error[1:-1,-1] - keep		new_error[1:-1,-1] = new_error[1:-1,-1] - keep
	new_error[0,0] = keep		new_error[0,0] = keep


	#new_error = np.ravel(new_error)		#new_error = np.ravel(new_error)

	bins = [25,40,70]		bins = [25,40,70]

	# return the huffman dictionary		# return the huffman dictionary
	return encode1, encode2, encode3, encode4, encode5, np.ravel(image), error, new_error, diff, boundary, bins, predict, A		return encode1, encode2, encode3, encode4, encode5, np.ravel(image), error, new_error, diff, boundary, bins, predict, A

	```		```

	%% Cell type:code id:4fb8f5d0 tags:		%% Cell type:code id:4f6a5a0d tags:

	``` python		``` python
	def encoder(error, list_dic, diff, bound, bins):		def encoder(error, list_dic, diff, bound, bins):
	encoded = np.copy(error).astype(int).astype(str).astype(object)		encoded = np.copy(error).astype(int).astype(str).astype(object)

	diff = np.reshape(diff,(510,638))		diff = np.reshape(diff,(510,638))

	for i in range(encoded.shape[0]):		for i in range(encoded.shape[0]):
	for j in range(encoded.shape[1]):		for j in range(encoded.shape[1]):
	if i == 0 or i == encoded.shape[0]-1 or j == 0 or j == encoded.shape[1]-1:		if i == 0 or i == encoded.shape[0]-1 or j == 0 or j == encoded.shape[1]-1:
	encoded[i][j] = list_dic[0][encoded[i][j]]		encoded[i][j] = list_dic[0][encoded[i][j]]
	elif diff[i-1][j-1] <= bins[0]:		elif diff[i-1][j-1] <= bins[0]:
	encoded[i][j] = list_dic[1][encoded[i][j]]		encoded[i][j] = list_dic[1][encoded[i][j]]
	elif diff[i-1][j-1] <= bins[1] and diff[i-1][j-1] > bins[0]:		elif diff[i-1][j-1] <= bins[1] and diff[i-1][j-1] > bins[0]:
	encoded[i][j] = list_dic[2][encoded[i][j]]		encoded[i][j] = list_dic[2][encoded[i][j]]
	elif diff[i-1][j-1] <= bins[2] and diff[i-1][j-1] > bins[1]:		elif diff[i-1][j-1] <= bins[2] and diff[i-1][j-1] > bins[1]:
	encoded[i][j] = list_dic[3][encoded[i][j]]		encoded[i][j] = list_dic[3][encoded[i][j]]
	else:		else:
	encoded[i][j] = list_dic[4][encoded[i][j]]		encoded[i][j] = list_dic[4][encoded[i][j]]


	return encoded		return encoded
	```		```

	%% Cell type:code id:9a39a65b tags:		%% Cell type:code id:4b65c7e9 tags:

	``` python		``` python
	def decoder(A, encoded_matrix, list_dic, bins):		def decoder(A, encoded_matrix, list_dic, bins):
	"""		"""
	Function that accecpts the prediction matrix A for the linear system,		Function that accecpts the prediction matrix A for the linear system,
	the encoded matrix of error values, and the encoding dicitonary.		the encoded matrix of error values, and the encoding dicitonary.
	"""		"""

	the_keys0 = list(list_dic[0].keys())		the_keys0 = list(list_dic[0].keys())
	the_values0 = list(list_dic[0].values())		the_values0 = list(list_dic[0].values())

	the_keys1 = list(list_dic[1].keys())		the_keys1 = list(list_dic[1].keys())
	the_values1 = list(list_dic[1].values())		the_values1 = list(list_dic[1].values())

	the_keys2 = list(list_dic[2].keys())		the_keys2 = list(list_dic[2].keys())
	the_values2 = list(list_dic[2].values())		the_values2 = list(list_dic[2].values())

	the_keys3 = list(list_dic[3].keys())		the_keys3 = list(list_dic[3].keys())
	the_values3 = list(list_dic[3].values())		the_values3 = list(list_dic[3].values())

	the_keys4 = list(list_dic[4].keys())		the_keys4 = list(list_dic[4].keys())
	the_values4 = list(list_dic[4].values())		the_values4 = list(list_dic[4].values())

	error_matrix = np.zeros((512,640))		error_matrix = np.zeros((512,640))

	for i in range(error_matrix.shape[0]):		for i in range(error_matrix.shape[0]):
	for j in range(error_matrix.shape[1]):		for j in range(error_matrix.shape[1]):
	if i == 0 and j == 0:		if i == 0 and j == 0:
	error_matrix[i][j] = int(the_keys0[the_values0.index(encoded_matrix[i,j])])		error_matrix[i][j] = int(the_keys0[the_values0.index(encoded_matrix[i,j])])

	elif i == 0 or i == error_matrix.shape[0]-1 or j == 0 or j == error_matrix.shape[1]-1:		elif i == 0 or i == error_matrix.shape[0]-1 or j == 0 or j == error_matrix.shape[1]-1:
	error_matrix[i][j] = int(the_keys0[the_values0.index(encoded_matrix[i,j])]) + error_matrix[0][0]		error_matrix[i][j] = int(the_keys0[the_values0.index(encoded_matrix[i,j])]) + error_matrix[0][0]
	else:		else:
	z0 = error_matrix[i-1][j-1]		z0 = error_matrix[i-1][j-1]
	z1 = error_matrix[i-1][j]		z1 = error_matrix[i-1][j]
	z2 = error_matrix[i-1][j+1]		z2 = error_matrix[i-1][j+1]
	z3 = error_matrix[i][j-1]		z3 = error_matrix[i][j-1]
	y0 = int(-z0+z2-z3)		y0 = int(-z0+z2-z3)
	y1 = int(z0+z1+z2)		y1 = int(z0+z1+z2)
	y2 = int(-z0-z1-z2-z3)		y2 = int(-z0-z1-z2-z3)
	y = np.vstack((y0,y1,y2))		y = np.vstack((y0,y1,y2))
	difference = max(z0,z1,z2,z3) - min(z0,z1,z2,z3)		difference = max(z0,z1,z2,z3) - min(z0,z1,z2,z3)
	predict = np.round(np.round(np.linalg.solve(A,y)[-1][0],1))		predict = np.round(np.round(np.linalg.solve(A,y)[-1][0],1))

	if difference <= bins[0]:		if difference <= bins[0]:
	error_matrix[i][j] = int(the_keys1[the_values1.index(encoded_matrix[i,j])]) + int(predict)		error_matrix[i][j] = int(the_keys1[the_values1.index(encoded_matrix[i,j])]) + int(predict)
	elif difference <= bins[1] and difference > bins[0]:		elif difference <= bins[1] and difference > bins[0]:
	error_matrix[i][j] = int(the_keys2[the_values2.index(encoded_matrix[i,j])]) + int(predict)		error_matrix[i][j] = int(the_keys2[the_values2.index(encoded_matrix[i,j])]) + int(predict)
	elif difference <= bins[2] and difference > bins[1]:		elif difference <= bins[2] and difference > bins[1]:
	error_matrix[i][j] = int(the_keys3[the_values3.index(encoded_matrix[i,j])]) + int(predict)		error_matrix[i][j] = int(the_keys3[the_values3.index(encoded_matrix[i,j])]) + int(predict)
	else:		else:
	error_matrix[i][j] = int(the_keys4[the_values4.index(encoded_matrix[i,j])]) + int(predict)		error_matrix[i][j] = int(the_keys4[the_values4.index(encoded_matrix[i,j])]) + int(predict)


	return error_matrix.astype(int)		return error_matrix.astype(int)
	```		```

	%% Cell type:code id:fc4d80bd tags:		%% Cell type:code id:280aafd3 tags:

	``` python		``` python
	scenes = file_extractor()		scenes = file_extractor()
	images = image_extractor(scenes)		images = image_extractor(scenes)
	encode1, encode2, encode3, encode4, encode5, image, error, new_error, diff, bound, bins, predict, A = huffman(images[0])		encode1, encode2, encode3, encode4, encode5, image, error, new_error, diff, bound, bins, predict, A = huffman(images[0])
	encoded_matrix = encoder(np.reshape(new_error,(512,640)), [encode1, encode2, encode3, encode4, encode5], diff, bound, bins)		encoded_matrix = encoder(np.reshape(new_error,(512,640)), [encode1, encode2, encode3, encode4, encode5], diff, bound, bins)
	list_dic = [encode1, encode2, encode3, encode4, encode5]		list_dic = [encode1, encode2, encode3, encode4, encode5]
	```		```

	%% Cell type:code id:2d82e61a tags:		%% Cell type:code id:c4242b52 tags:

	``` python		``` python
	reconstruct_image = decoder(A, encoded_matrix, list_dic, bins)		reconstruct_image = decoder(A, encoded_matrix, list_dic, bins)
	np.allclose(image.reshape(512,640), reconstruct_image)		np.allclose(image.reshape(512,640), reconstruct_image)
	```		```

	%% Output		%% Output

	True		True

	%% Cell type:code id:e35be607 tags:		%% Cell type:code id:a9502e22 tags:

	``` python		``` python

	"""plt.hexbin(x,y,cmap="rocket")		"""plt.hexbin(x,y,cmap="rocket")
	plt.colorbar()		plt.colorbar()
	plt.xlim(0,50)		plt.xlim(0,50)
	plt.ylim(0,100)"""		plt.ylim(0,100)"""

	def rel_freq(x):		def rel_freq(x):
	freqs = [x.count(value) / len(x) for value in set(x)]		freqs = [x.count(value) / len(x) for value in set(x)]
	return freqs		return freqs
	print(sp.stats.entropy(rel_freq(list(diff))))		print(sp.stats.entropy(rel_freq(list(diff))))

	def entropy_check(x, y):		def entropy_check(x, y):
	#freq = rel_freq(list(np.ravel(o)))		#freq = rel_freq(list(np.ravel(o)))
	means = []		means = []
	for i in range(len(images)):		for i in range(len(images)):
	p, d, o, e, A = predict(images,0)		p, d, o, e, A = predict(images,0)
	d = d.reshape((510,638))		d = d.reshape((510,638))
	x = np.abs(np.ravel(e))		x = np.abs(np.ravel(e))
	y = np.ravel(d)		y = np.ravel(d)

	mask1 = y <= 25		mask1 = y <= 25
	x_masked1 = x[mask1]		x_masked1 = x[mask1]

	mask2 = y > 25		mask2 = y > 25
	x_masked2 = x[mask2]		x_masked2 = x[mask2]
	mask2 = y[mask2] <= 40		mask2 = y[mask2] <= 40
	x_masked2 = x_masked2[mask2]		x_masked2 = x_masked2[mask2]

	mask3 = y > 40		mask3 = y > 40
	x_masked3 = x[mask3]		x_masked3 = x[mask3]
	mask3 = y[mask3] <= 75		mask3 = y[mask3] <= 75
	x_masked3 = x_masked3[mask3]		x_masked3 = x_masked3[mask3]

	mask4 = y > 75		mask4 = y > 75
	x_masked4 = x[mask4]		x_masked4 = x[mask4]


	e_m1 = sp.stats.entropy(rel_freq(list(x_masked1)))		e_m1 = sp.stats.entropy(rel_freq(list(x_masked1)))
	e_m2 = sp.stats.entropy(rel_freq(list(x_masked2)))		e_m2 = sp.stats.entropy(rel_freq(list(x_masked2)))
	e_m3 = sp.stats.entropy(rel_freq(list(x_masked3)))		e_m3 = sp.stats.entropy(rel_freq(list(x_masked3)))
	e_m4 = sp.stats.entropy(rel_freq(list(x_masked4)))		e_m4 = sp.stats.entropy(rel_freq(list(x_masked4)))
	means.append([e_m1, e_m2, e_m3, e_m4])		means.append([e_m1, e_m2, e_m3, e_m4])
	return np.mean(np.array(means).reshape(len(images),4), axis=0)		return np.mean(np.array(means).reshape(len(images),4), axis=0)

	#print(entropy_check(x, y))		#print(entropy_check(x, y))
	```		```

	%% Output		%% Output

	4.720647237500972		4.720647237500972

Admin message