I am working on lung CT images from luna16 dataset, the dataset have a 3d lung image and a label from CSV file, I have a code for constructing 2d list from 3d array 25x25x25 (the 3d image) and a label [0,1] or [1,0] from CSV file, after creating the 2d list I want to save it in numpy file, below is my code for creating the 2d list and saving it in numpy file:
def getIDlist(csv_Dist,Data_Dist): # receive marked coords and ID in annotations.csv, and return the distination with coords. print('loading') data = np.loadtxt(csv_Dist, delimiter = ',', dtype = 'str') # delete the header file via 1:0, and receive the ID, x, y, z, r via 0:5 to a list. ID_coords = data[1:,0:5][0:10000] # get list of 'seriesuid' 'coordX' 'coordY' 'coordZ' 'class' (without header). # define the output file. ID_dist =  print('strat finding') process_bar = ShowProcess(len(ID_coords)) for ID,x,y,z,label in ID_coords: ID = ID +'.mhd' found = 0 for parent, dirnames, filenames in os.walk(Data_Dist): for filename in filenames:# loop inside all files if ID == filename: # ID + .mhd in csv equal to filename in files process_bar.show_process() ID = parent + '\\' + ID# ID gets full path of the founded file ID_dist.append([ID,x,y,z,label])# ID_dist gets info of founded files found = 1 #print("found: ", found) break if found == 1: break if found == 1: continue process_bar.close() return ID_dist def get3Dmatrix(ID_dist): print('preparing the 3d matrix') matrixlist =  for Dist, xcoords, ycoords, zcoords, label in tqdm(ID_dist): # read the image imagearray,origin,spacing = load_itk_image(Dist) # resample in to 1mm*1mm*1mm imagearray = resample(imagearray,spacing,(1,1,1)) # transfer world coordinates to voxel-coordinates, divide new spacing 1mm z = int(round((float(zcoords)-float(origin))/1)) y = int(round((float(ycoords)-float(origin))/1)) x = int(round((float(xcoords)-float(origin))/1)) # get the 3D array with shape 25*25*25 imagearray = imagearray[z-13:z+12,y-13:y+12,x-13:x+12] #converting the label number into a one-hot-encoding if int(label) == 1: label=np.array([0,1]) elif int(label) == 0: label=np.array([1,0]) # put it into output file matrixlist.append([imagearray,label])# 2d list consist of 3d array + label of all cases. return matrixlist def main(): start_time = time.time() # get ID_list from the csv and data dist. ID_list = getIDlist(candidates_V2_Dist, Data_Dist)# nested list - get file name with dist + x,y,z,class # Data_set[i] is the 3D array, Data_set[i] is the label Data_set = get3Dmatrix(ID_list) # 2d list consist of 3d array + label of all cases. print("Begin saving in numpy file") np.save(output_path+'np_ds(10000)-25-25-25(zyx)_one_hot.npy', Data_set) print("%s time takes in seconds" % (time.time() - start_time)) if __name__ == "__main__": main()
my problem is:
1- with approximatly 550 samples, the RAM gets fulled and I get memory error, I am working on dell inspiron core i7 with 16 gb ram laptop.
2- it takes 34 seconds for creating each sample, and I see this is huge amount of time for only one sample.