trying to find a good metric

ErrorImageCreator.py

@@ -90,7 +90,7 @@ def color_adjust(visual_array):
 if __name__ == "__main__":
     scenes = file_extractor("averaged_images(11)")
     images = image_extractor(scenes)
-    list_dic = np.load("second_dic.npy", allow_pickle="TRUE")
+    list_dic = np.load("second_dict.npy", allow_pickle="TRUE")
     bins = [21,32,48]
     print(images)
     encoded_string2 = bytes_to_bitstring(read_from_file(images[0][:-5] + "_Compressed.txt"))

FPNprojstart.py

+from functools import singledispatchmethod
+from cv2 import mean
+import numpy as np
+from WorkingPyDemo import *
+from Remove_Noise import *
+import pandas as pd
+import statsmodels.api as sm
+from matplotlib import pyplot as plt
+from sklearn.metrics import mean_squared_error
+# scenes = file_extractor("images")
+# images = image_extractor(scenes)
+# print(images)
+
+scenes = remote_file_extractor("/media/elphel/NVME/lwir16-proc/te0607/scenes/")
+images = remote_image_extractor(scenes)
+# images = find_only_in_channel(images, "11")
+sftp_client = setup_remote_sftpclient()
+image_array = np.array(Image.open(sftp_client.open(images[250])))
+# image_array = np.array(Image.open(images[250])).astype(np.uint16)
+# print(image_array[0])
+# print(images[0:5])
+# print(image_array[0,0].dtype)
+thebitstring = bytes_to_bitstring(image_array[0])
+frame = thebitstring[84*8:88*8]
+# print(int(frame,2))
+otherframe = thebitstring[88*8:92*8]
+# print(int(otherframe,2))
+# print(image_array[0].astype(np.uint16))
+# print(image_array[0,42])
+# print(bin(image_array[0,0])[2:].zfill(16))
+# for i in range(len(images)):
+#     image_array = np.array(Image.open(images[i]))
+#     print(image_array[0,91] - image_array[0,90])
+# print(np.array(Image.open(images[100]))[0])
+def uint16_array_to_bitstring(array):
+    bitstring = ""
+    for i in range(len(array)):
+        bitstring += bin(array[i])[2:].zfill(16)
+    return bitstring
+def repopulate_array_with_bitstring(bitstring):
+    array = []
+    for i in range(0,len(bitstring),8):
+        array.append(int(bitstring[i:i+8],2))
+    return np.array(array)
+def intarray_to_uint32(int_array):
+    bitstring = ""
+    # print(int_array)
+    for i in range(len(int_array)):
+        bitstring += bin(int_array[i])[2:].zfill(8)
+    # print(bitstring)
+    return int(bitstring,2)
+# print(repopulate_array_with_bitstring(uint16_array_to_bitstring(image_array[0]))[0:6])
+frame_spacings = []
+signaltonoise = []
+information_array = np.zeros((len(images),3))
+def celcius_to_kelvin(celcius):
+    return celcius + 273.15
+
+for i in range(len(images)):
+    image_array = np.array(Image.open(sftp_client.open(images[i]))).astype(np.uint16)
+    current_frame_count = intarray_to_uint32(repopulate_array_with_bitstring(uint16_array_to_bitstring(image_array[0]))[84:88])
+    frame_count_at_FFC = intarray_to_uint32(repopulate_array_with_bitstring(uint16_array_to_bitstring(image_array[0]))[88:92])
+    # signaltonoise.append(1/np.std(image_array[1:]))
+    information_array[i,0] = current_frame_count - frame_count_at_FFC
+    # information_array[i,-1] = 1/np.std(image_array[1:])
+    information_array[i,-1] = mean_squared_error(image_array[1:],gaussian_filter(image_array[1:],sigma=.3))
+    # print(repopulate_array_with_bitstring(uint16_array_to_bitstring(image_array[0]))[76:77])
+    # print("start")
+    information_array[i,1] = intarray_to_uint32(repopulate_array_with_bitstring(uint16_array_to_bitstring(image_array[0]))[94:96]) - \
+    celcius_to_kelvin(intarray_to_uint32(repopulate_array_with_bitstring(uint16_array_to_bitstring(image_array[0]))[162:166]))
+    # information_array[i,2] = intarray_to_uint32(repopulate_array_with_bitstring(uint16_array_to_bitstring(image_array[0]))[96:98])
+    # information_array[i,2] = celcius_to_kelvin(intarray_to_uint32(repopulate_array_with_bitstring(uint16_array_to_bitstring(image_array[0]))[162:166]))
+    # information_array[i,4] = information_array[i,1] - information_array[i,2]
+    # information_array[i,5] = information_array[i,1] - information_array[i,3]
+    # information_array[i,4] = intarray_to_uint32(repopulate_array_with_bitstring(uint16_array_to_bitstring(image_array[0]))[170:174])
+    # information_array[i,5] = intarray_to_uint32(repopulate_array_with_bitstring(uint16_array_to_bitstring(image_array[0]))[174:178])
+    # frame_spacings.append(current_frame_count - frame_count_at_FFC)
+# print(np.argmin(signaltonoise))
+# print(np.argmax(signaltonoise))
+# print(np.min(abs(np.array(frame_spacings))))
+# print("middle")
+
+
+mask = information_array[:,0] > 0
+information_df = pd.DataFrame(information_array[mask],columns=["Frame_Spacing","Curr_Temp_Diff","Signal_to_Noise"])
+information_df.to_csv("information_weird_array.csv")
+# information_df = pd.read_csv("information_array.csv")
+information_array = information_df.values
+plt.scatter(information_array[:,0],information_array[:,-1],s=1)
+plt.legend()
+plt.xlabel("Frame_Spacing")
+plt.ylabel("Signal_to_Noise")
+plt.show()
+print(np.max(information_array[:,0]))
+print(information_array[-5:-1,0])
+
+X = sm.add_constant(information_df.drop(["Signal_to_Noise"],axis=1))
+
+y = information_df["Signal_to_Noise"]
+
+
+end_result = sm.OLS(y,X).fit()
+print(end_result.summary())
+sftp_client.close()

FullTester.ipynb

@@ -30,7 +30,7 @@
    "source": [
     "scenes = file_extractor(\"images\")\n",
     "images = image_extractor(scenes)\n",
-    "print(len(images)) #\n",
+    "print(len(images))\n",
     "newnamesforlater = []\n",
     "file_sizes_new = [] \n",
     "file_sizes_old = []\n",

Remove_Noise.py

@@ -211,7 +211,7 @@ def save_testable_images(images, selected_channel, quantity_of_images):
     images = find_only_in_channel(images, selected_channel)
 
     # image_locations = np.random.choice(len(images), quantity_of_images, replace=False)
-    image_locations = np.arange(quantity_of_images) + 5000
+    image_locations = np.arange(quantity_of_images) + 10000
 
     selected_images = np.array(images)[image_locations]
 
@@ -270,7 +270,7 @@ if __name__ == "__main__":
     images = remote_image_extractor(scenes)
     images = find_only_in_channel(images, "11")
     # average_image = np.array(Image.open("Average_On_Channel(" + "11" + ").tiff"))
-    # save_testable_images(images,"11",3)
+    save_testable_images(images,"11",5)
     # plt.imshow(color_adjust(average_image),cmap='gray',vmin = 0, vmax=1)
     # plt.show()
     
@@ -287,27 +287,27 @@ if __name__ == "__main__":
     # print(test_image)
     plt.subplot(121)
     plt.imshow(color_adjust(adjust_to_original(test_image,average_image)),cmap='gray',vmin = 0, vmax=1)
-    plt.subplot(122)
+    # plt.subplot(122)
     # plt.show()
     # print(gaussian_kernel)
-    little_more_blurred = convolve2d(np.pad(test_image,2, mode = 'edge'),gaussian_kernel, "valid")
+    # little_more_blurred = convolve2d(np.pad(test_image,2, mode = 'edge'),gaussian_kernel, "valid")
     # print(little_more_blurred)
-    print(little_more_blurred.shape)
+    # print(little_more_blurred.shape)
     # print(little_inverter(gaussian_kernel)@gaussian_kernel)
-    altered_image = Image.fromarray(little_more_blurred.astype(np.uint16))
+    # altered_image = Image.fromarray(little_more_blurred.astype(np.uint16))
 
-    altered_image.save("averaged_images(" + "11" + ")/innerfolder" + "/testable.tiff")
-    plt.imshow(color_adjust(little_more_blurred[3:-3,3:-3]),cmap='gray',vmin = 0, vmax=1)
+    # altered_image.save("averaged_images(" + "11" + ")/innerfolder" + "/testable.tiff")
+    # plt.imshow(color_adjust(little_more_blurred[3:-3,3:-3]),cmap='gray',vmin = 0, vmax=1)
     # plt.show()
     # renewed_array = 
     # H = fft(kernel)
 	# deconvolved = np.real(ifft(fft(signal)*np.conj(H)/(H*np.conj(H) + lambd**2)))
-    back_to_normal = wiener(little_more_blurred,gaussian_kernel,.02,clip=False)
+    # back_to_normal = wiener(little_more_blurred,gaussian_kernel,.02,clip=False)
     # print(back_to_normal)
     # print(back_to_normal.shape)
     
     # plt.imshow(color_adjust(back_to_normal), cmap='gray',vmin = 0, vmax=1)
-    plt.show()
+    # plt.show()
     # newimage = Image.fromarray(test_image - average_image)
     # newimage.save("NoInterference.tiff")
     
@@ -320,6 +320,6 @@ if __name__ == "__main__":
     # print(np.linalg.inv(np.array([[3,0,-1],[0,3,3],[1,-3,-4]])))
     # print(np.linalg.pinv(np.array([[-1,-1,1], [-1,0,1], [-1,1,1], [0,-1,1]])))
 
-    last_image = Image.open("NoInterference.tiff")
+    # last_image = Image.open("NoInterference.tiff")
     # plt.imshow(color_adjust(np.array(last_image)),cmap='gray',vmin = 0, vmax=1)
     # plt.show()