01234567890123456789012345678901234567890123456789012345678901234567890123456789
11651166116711681169117011711172117311741175117611771178117911801181118211831184 11851186118711881189119011911192119311941195119611971198119912001201120212031204 |
<----SKIPPED LINES---->
files = sorted([os.path.join(directory, f) for f in files if pattern.match(f)])
else:
if os.path.exists(full_path):
files = [full_path]
else:
return []
data = []
if filenames:
return files
for file in files:
try:
with open(file, 'rb') as f:
while True:
data.append(pickle.load(f))
except (EOFError, pickle.UnpicklingError):
pass
return data
cached_object_count = {}
def PickleObjectToFile(data, full_path, date_segmentation, timestamp=None, verify=False):
"""Append one pickled flight to the end of binary file.
Args:
data: data to pickle
full_path: name (potentially including path) of the pickled file
date_segmentation: boolean indicating whether the date string yyyy-mm-dd should be
prepended to the file name in full_path based on the current date, so that
pickled files are segmented by date.
timestamp: if date_segmentation is True, this is used rather than system time
to generate the file name.
verify: boolean indicating if we should verify that the pickled file object count
increments by one, rewriting entire pickle file if it doesn't. Note that since
this requires reading the entire pickle file and unpickling, it should only be
done for small files / objects.
<----SKIPPED LINES---->
|
01234567890123456789012345678901234567890123456789012345678901234567890123456789
1165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207 |
<----SKIPPED LINES---->
files = sorted([os.path.join(directory, f) for f in files if pattern.match(f)])
else:
if os.path.exists(full_path):
files = [full_path]
else:
return []
data = []
if filenames:
return files
for file in files:
try:
with open(file, 'rb') as f:
while True:
data.append(pickle.load(f))
except (EOFError, pickle.UnpicklingError):
pass
except (UnicodeDecodeError) as e:
Log('Process %s reading file %s gave error %s' % (
psutil.Process(os.getpid()).name(), f, e))
return data
cached_object_count = {}
def PickleObjectToFile(data, full_path, date_segmentation, timestamp=None, verify=False):
"""Append one pickled flight to the end of binary file.
Args:
data: data to pickle
full_path: name (potentially including path) of the pickled file
date_segmentation: boolean indicating whether the date string yyyy-mm-dd should be
prepended to the file name in full_path based on the current date, so that
pickled files are segmented by date.
timestamp: if date_segmentation is True, this is used rather than system time
to generate the file name.
verify: boolean indicating if we should verify that the pickled file object count
increments by one, rewriting entire pickle file if it doesn't. Note that since
this requires reading the entire pickle file and unpickling, it should only be
done for small files / objects.
<----SKIPPED LINES---->
|