Made some changes, have to rerun everything (625e9d15) · Commits · Elphel / image-compression

FullTester.ipynb

+39 −9

Original line number	Original line	Diff line number	Diff line
	%% Cell type:code id: tags:		%% Cell type:code id: tags:

	``` python		``` python
	from WorkingPyDemo import *		from WorkingPyDemo import *
	from time import time		from time import time
	from scipy import misc		from scipy import misc
	import imageio		import imageio
	import png		import png
	```		```

	%% Cell type:code id: tags:		%% Cell type:code id: tags:

	``` python		``` python
	scenes = file_extractor(folder_name)		scenes = file_extractor(folder_name)
	images = image_extractor(scenes)		images = image_extractor(scenes)
	print(len(images))		print(len(images))
	newnamesforlater = []		newnamesforlater = []
	file_sizes_new = []		file_sizes_new = []
	file_sizes_old = []		file_sizes_old = []
	list_dic = np.load("first_dic.npy", allow_pickle="TRUE")		list_dic = np.load("first_dic.npy", allow_pickle="TRUE")
	bins = [21,32,48]		bins = [21,32,48]
	for i in range(len(images)):		for i in range(len(images)):

	if images[i][-5:] == ".tiff":		if images[i][-5:] == ".tiff":
	newname = images[i][:-5]		newname = images[i][:-5]
	else:		else:
	newname = images[i][:-4]		newname = images[i][:-4]
	if images[i][-18:] != "Compressedlzw.tiff":		if images[i][-18:] != "Compressedlzw.tiff":
	newnamesforlater.append(newname + "_Compressed.txt")		newnamesforlater.append(newname + "_Compressed.txt")
	# with open(newname + "_Compressed.txt", 'wb') as f:		# with open(newname + "_Compressed.txt", 'wb') as f:
	# f.write(inletters)		# f.write(inletters)
	file_sizes_new.append((os.path.getsize(newname + "_Compressed.txt")))		file_sizes_new.append((os.path.getsize(newname + "_Compressed.txt")))
	file_sizes_old.append((os.path.getsize(images[i])))		file_sizes_old.append((os.path.getsize(images[i])))
	file_sizes_new.append(os.path.getsize("first_dic.npy"))		file_sizes_new.append(os.path.getsize("first_dic.npy"))
	print(len(newnamesforlater))		print(len(newnamesforlater))
	print(np.sum(file_sizes_new)/np.sum(file_sizes_old))		print(np.sum(file_sizes_new)/np.sum(file_sizes_old))
	```		```

	%% Output		%% Output

	262		262
	262		262
	0.3968581030135283		0.3968581030135283

	%% Cell type:code id: tags:		%% Cell type:code id: tags:

	``` python		``` python
			print(file_sizes_new)
			```

			%% Output

			[0, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 260942, 231302]

			%% Cell type:code id: tags:

			``` python
	pngsizes = []		pngsizes = []
	for i, filename in enumerate(images):		for i, filename in enumerate(images):
	newimage = Image.open(filename)		# newimage = Image.open(filename)
	newimage = np.array(newimage)		# newimage = np.array(newimage)
	newimage = newimage[1:]		# newimage = newimage[1:]
	with open(newnamesforlater[i][:-4] + ".png", 'wb') as f:		# with open(newnamesforlater[i][:-4] + ".png", 'wb') as f:
	writer = png.Writer(newimage.shape[1], newimage.shape[0], greyscale=True, bitdepth=16)		# writer = png.Writer(newimage.shape[1], newimage.shape[0], greyscale=True, bitdepth=16)
	writer.write(f, newimage)		# writer.write(f, newimage)
	# imageio.imwrite(newnamesforlater[i][:-4] + ".png", newimage)		# imageio.imwrite(newnamesforlater[i][:-4] + ".png", newimage)
	# newimage.close()		# newimage.close()
	pngsizes.append(os.path.getsize(newnamesforlater[i][:-4] + ".png"))		pngsizes.append(os.path.getsize(newnamesforlater[i][:-4] + ".png"))
	print(np.sum(pngsizes)/np.sum(file_sizes_old))		print(np.sum(pngsizes)/np.sum(file_sizes_old))
	```		```

	%% Output		%% Output

	0.6104417990437042		0.6104417990437042

	%% Cell type:code id: tags:		%% Cell type:code id: tags:

	``` python		``` python
	print(os.path.getsize("1626032610_393963_0.xcf")/os.path.getsize("images/1626032610_393963/1626032610_393963_0.tiff"))		print(os.path.getsize("1626032610_393963_0.xcf")/os.path.getsize("images/1626032610_393963/1626032610_393963_0.tiff"))
	```		```

	%% Output		%% Output

	0.5676821409714287		0.5676821409714287

	%% Cell type:code id: tags:		%% Cell type:code id: tags:

	``` python		``` python
	print(os.path.getsize("1626032610_393963_0.png")/os.path.getsize("images/1626032610_393963/1626032610_393963_0.tiff"))		print(os.path.getsize("1626032610_393963_0.png")/os.path.getsize("images/1626032610_393963/1626032610_393963_0.tiff"))
	```		```

	%% Output		%% Output

	0.4855910784741866		0.4855910784741866

	%% Cell type:code id: tags:		%% Cell type:code id: tags:

	``` python		``` python
	for i, filename in enumerate(images):		for i, filename in enumerate(images):
	original = Image.open(filename)		original = Image.open(filename)
	compressed = Image.open(newnamesforlater[i][:-4] + ".png")		compressed = Image.open(newnamesforlater[i][:-4] + ".png")
	original = np.array(original)[1:]		original = np.array(original)[1:]
	compressed = np.array(compressed)		compressed = np.array(compressed)
	if not np.allclose(original, compressed):		if not np.allclose(original, compressed):
	print(i)		print(i)
	```		```

	%% Cell type:code id: tags:		%% Cell type:code id: tags:

	``` python		``` python
	print((np.sum(pngsizes))/np.sum(file_sizes_old))		print((np.sum(pngsizes))/np.sum(file_sizes_old))
	```		```

	%% Output		%% Output

	0.6104417990437042		0.6104417990437042

	%% Cell type:code id: tags:		%% Cell type:code id: tags:

	``` python		``` python
	```		```

	%% Cell type:code id: tags:		%% Cell type:code id: tags:

	``` python		``` python
	from PIL import TiffTags		from PIL import TiffTags
	TiffTags.LIBTIFF_CORE.add(318)		TiffTags.LIBTIFF_CORE.add(318)
	```		```

	%% Cell type:code id: tags:		%% Cell type:code id: tags:

	``` python		``` python
	lwz_sizes = []		lwz_sizes = []
	for i, filename in enumerate(images):		for i, filename in enumerate(images):
	newimage = Image.open(filename)		newimage = Image.open(filename)
			newimage = np.array(newimage)
			newimage = newimage[1:]
			newimage = Image.fromarray(newimage)
	newimage.save(newnamesforlater[i][:-4]+ "lzw" + ".tiff", compression='tiff_lzw', tiffinfo={317: 2})		newimage.save(newnamesforlater[i][:-4]+ "lzw" + ".tiff", compression='tiff_lzw', tiffinfo={317: 2})

	lwz_sizes.append(os.path.getsize(newnamesforlater[i][:-4]+ "lzw" + ".tiff"))		lwz_sizes.append(os.path.getsize(newnamesforlater[i][:-4]+ "lzw" + ".tiff"))
	print(np.sum(lwz_sizes)/np.sum(file_sizes_old))		print(np.sum(lwz_sizes)/np.sum(file_sizes_old))
	```		```

	%% Output		%% Output

	0.6185872808807661		0.6177078151903203

			%% Cell type:code id: tags:

			``` python
			print((np.sum(lwz_sizes) - np.sum(file_sizes_new))/np.sum(lwz_sizes))
			```

			%% Output

			0.3575310312509913

	%% Cell type:code id: tags:		%% Cell type:code id: tags:

	``` python		``` python
	print(len(images))		print(len(images))
	for i, item in enumerate(images):		for i, item in enumerate(images):
	# print(item[-18:])		# print(item[-18:])
	if item[-18:] == "Compressedlzw.tiff":		if item[-18:] == "Compressedlzw.tiff":
	os.remove(item)		os.remove(item)
	scenes = file_extractor(folder_name)		scenes = file_extractor(folder_name)
	images = image_extractor(scenes)		images = image_extractor(scenes)
	print(len(images))		print(len(images))
	```		```

	%% Output		%% Output

	524		524
	262		262

	%% Cell type:code id: tags:		%% Cell type:code id: tags:

	``` python		``` python
	def all_image_extractor(scenes):		def all_image_extractor(scenes):
	"""		"""
	This function gives the list of all of the valid images		This function gives the list of all of the valid images
	Parameters:		Parameters:
	scenes (list): a list of all the files in the directory		scenes (list): a list of all the files in the directory
	Returns:		Returns:
	images (list): a list of all the images in the directory		images (list): a list of all the images in the directory
	"""		"""
	image_folder = []		image_folder = []
	for scene in scenes:		for scene in scenes:
	files = os.listdir(scene)		files = os.listdir(scene)
	for file in files:		for file in files:
	if file[-7:] == "_6.tiff":		if file[-7:] == "_6.tiff":
	continue		continue
	else:		else:
	image_folder.append(os.path.join(scene, file))		image_folder.append(os.path.join(scene, file))
	return image_folder #returns a list of file paths to .tiff files in the specified directory given in file_extractor		return image_folder #returns a list of file paths to .tiff files in the specified directory given in file_extractor
	```		```

	%% Cell type:code id: tags:		%% Cell type:code id: tags:

	``` python		``` python
	all_files = all_image_extractor(scenes)		all_files = all_image_extractor(scenes)
	print(len(all_files))		print(len(all_files))
	for i, item in enumerate(all_files):		for i, item in enumerate(all_files):
	# print(item[-18:])		# print(item[-18:])
			print(item[-4:])
	if item[-4:] == "..png":		if item[-4:] == "..png":
	os.remove(item)		os.remove(item)
	scenes = file_extractor(folder_name)		scenes = file_extractor(folder_name)
	all_files = all_image_extractor(scenes)		all_files = all_image_extractor(scenes)
	print(len(all_files))		print(len(all_files))
	```		```

	%% Output		%% Output

	1178		---------------------------------------------------------------------------
	1178		NameError Traceback (most recent call last)
			/home/bryce/git/master/FullTester.ipynb Cell 13' in <cell line: 1>()
			----> <a href='vscode-notebook-cell:/home/bryce/git/master/FullTester.ipynb#ch0000012?line=0'>1</a> all_files = all_image_extractor(scenes)
			<a href='vscode-notebook-cell:/home/bryce/git/master/FullTester.ipynb#ch0000012?line=1'>2</a> print(len(all_files))
			<a href='vscode-notebook-cell:/home/bryce/git/master/FullTester.ipynb#ch0000012?line=2'>3</a> for i, item in enumerate(all_files):
			<a href='vscode-notebook-cell:/home/bryce/git/master/FullTester.ipynb#ch0000012?line=3'>4</a> # print(item[-18:])
			NameError: name 'all_image_extractor' is not defined

	%% Cell type:code id: tags:		%% Cell type:code id: tags:

	``` python		``` python
	```		```

WorkingPyDemo.py

+18 −10

Original line number	Original line	Diff line number	Diff line
	@@ -9,6 +9,7 @@ from sklearn.neighbors import KernelDensity
	from collections import Counter		from collections import Counter
	import numpy.linalg as la		import numpy.linalg as la
	from time import time		from time import time
			from time import sleep
	import tifffile as tiff		import tifffile as tiff
	folder_name = "images"		folder_name = "images"
	outputlocation = ""		outputlocation = ""
	@@ -506,11 +507,12 @@ if __name__ == "__main__":
	scenes = file_extractor(folder_name)		scenes = file_extractor(folder_name)
	images = image_extractor(scenes)		images = image_extractor(scenes)
	newnamesforlater = []		newnamesforlater = []
	# list_dic, bins = make_dictionary(images, 4, False)		list_dic, bins = make_dictionary(images, 4, False)
	file_sizes_new = []		file_sizes_new = []
	file_sizes_old = []		file_sizes_old = []
			# list_dic = np.load("first_dic.npy", allow_pickle="TRUE")
	# np.save("first_dic.npy", list_dic)		bins = [21,32,48]
			np.save("first_dic.npy", list_dic)
	for i in range(len(images)):		for i in range(len(images)):
	image, new_error, diff = huffman(images[i], 4, False)		image, new_error, diff = huffman(images[i], 4, False)
	encoded_string = encoder(new_error, list_dic, diff, bins)		encoded_string = encoder(new_error, list_dic, diff, bins)
	@@ -525,16 +527,22 @@ if __name__ == "__main__":
	f.write(inletters)		f.write(inletters)
	file_sizes_new.append((os.path.getsize(newname + "_Compressed.txt")))		file_sizes_new.append((os.path.getsize(newname + "_Compressed.txt")))
	file_sizes_old.append((os.path.getsize(images[i])))		file_sizes_old.append((os.path.getsize(images[i])))
			sleep(5)
			if i % 50 == 0:
			print(i)
			sleep(20)

	file_sizes_new.append(os.path.getsize("first_dic.npy"))		file_sizes_new.append(os.path.getsize("first_dic.npy"))
	print(np.sum(file_sizes_new)/np.sum(file_sizes_old))		print(np.sum(file_sizes_new)/np.sum(file_sizes_old))
	# list_dic = np.load("first_dic.npy", allow_pickle="TRUE")		list_dic = np.load("first_dic.npy", allow_pickle="TRUE")
	# bins = [21,32,48]		bins = [21,32,48]
	# starttime = time()		# starttime = time()
	# for i,item in enumerate(newnamesforlater[0:6]):		for i,item in enumerate(newnamesforlater[2:5]):
	# image, new_error, diff = huffman(images[i], 4, False)		print(item)
	# encoded_string2 = bytes_to_bitstring(read_from_file(item))		image, new_error, diff = huffman(images[i], 4, False)
	# reconstruct_image = decoder(encoded_string2, list_dic, bins, False)		encoded_string2 = bytes_to_bitstring(read_from_file(item))
	# print(np.allclose(image, reconstruct_image))		reconstruct_image = decoder(encoded_string2, list_dic, bins, False)
			print(np.allclose(image, reconstruct_image))
	# text_to_tiff("images/1626033496_437803/1626033496_437803_3._Compressed.txt", list_dic, bins)		# text_to_tiff("images/1626033496_437803/1626033496_437803_3._Compressed.txt", list_dic, bins)
	# original_image = Image.open("images/1626033496_437803/1626033496_437803_3.tiff")		# original_image = Image.open("images/1626033496_437803/1626033496_437803_3.tiff")
	# original_image = np.array(original_image)[1:]		# original_image = np.array(original_image)[1:]

first_dic.npy

−54 B (226 KiB)

File changed.

No diff preview for this file type.

View original file

View changed file