happyz
/
propagation_gan
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update 002.extract_measurements.py

This commit is contained in:
HappyZ 2019-06-10 11:55:44 -05:00
parent f085380ad7
commit 5a91b25503
1 changed files with 1 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
opencellid_parsing/002.extract_measurements.py
View File

@ -4,8 +4,7 @@ import os
import sys import sys
import pickle import pickle
print("load ~55GB data into memory! do not try this if not on server machine") print("load ~45GB data into memory! do not try this if not on server machine")
############################ ############################
# extract measurements # extract measurements
@ -51,26 +50,3 @@ for key in data:
pickle.dump(data_with_gt, open(pickle_fp_with_gt, 'wb')) pickle.dump(data_with_gt, open(pickle_fp_with_gt, 'wb'))
pickle.dump(data_no_gt, open(pickle_no_gt, 'wb')) pickle.dump(data_no_gt, open(pickle_no_gt, 'wb'))
pickle.dump(data_unseen, open(pickle_fp_unseen, 'wb')) pickle.dump(data_unseen, open(pickle_fp_unseen, 'wb'))
# print("extracting each file to disk (only groundtruth we cared)..")
# counter = 0
# for filepath in data:
# if "with_groundtruth" not in filepath:
# continue
# if not os.path.isfile(filepath):
# outputsubfolder = os.path.dirname(filepath)
# if not os.path.isdir(outputsubfolder):
# try:
# os.makedirs(outputsubfolder)
# except BaseException:
# print("err: cannot create folder {}!!!!".format(outputsubfolder))
# continue
# with open(filepath, 'w') as of:
# of.write(headlines.get(data[filepath][0], headline) + '\n')
# with open(filepath, 'a') as of:
# for dataline in data[filepath][1]:
# of.write(dataline + "\n")
# counter += 1
# if counter % 1000 == 0:
# print("progress: {} out of {} files extracted ({:.2f}%)".format(counter, len(data), 100.0 * counter / len(data)))