01234567890123456789012345678901234567890123456789012345678901234567890123456789
11651166116711681169117011711172117311741175117611771178117911801181118211831184 11851186118711881189119011911192119311941195119611971198119912001201120212031204 | <----SKIPPED LINES----> files = sorted([os.path.join(directory, f) for f in files if pattern.match(f)]) else: if os.path.exists(full_path): files = [full_path] else: return [] data = [] if filenames: return files for file in files: try: with open(file, 'rb') as f: while True: data.append(pickle.load(f)) except (EOFError, pickle.UnpicklingError): pass return data cached_object_count = {} def PickleObjectToFile(data, full_path, date_segmentation, timestamp=None, verify=False): """Append one pickled flight to the end of binary file. Args: data: data to pickle full_path: name (potentially including path) of the pickled file date_segmentation: boolean indicating whether the date string yyyy-mm-dd should be prepended to the file name in full_path based on the current date, so that pickled files are segmented by date. timestamp: if date_segmentation is True, this is used rather than system time to generate the file name. verify: boolean indicating if we should verify that the pickled file object count increments by one, rewriting entire pickle file if it doesn't. Note that since this requires reading the entire pickle file and unpickling, it should only be done for small files / objects. <----SKIPPED LINES----> |
01234567890123456789012345678901234567890123456789012345678901234567890123456789
1165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207 | <----SKIPPED LINES----> files = sorted([os.path.join(directory, f) for f in files if pattern.match(f)]) else: if os.path.exists(full_path): files = [full_path] else: return [] data = [] if filenames: return files for file in files: try: with open(file, 'rb') as f: while True: data.append(pickle.load(f)) except (EOFError, pickle.UnpicklingError): pass except (UnicodeDecodeError) as e: Log('Process %s reading file %s gave error %s' % ( psutil.Process(os.getpid()).name(), f, e)) return data cached_object_count = {} def PickleObjectToFile(data, full_path, date_segmentation, timestamp=None, verify=False): """Append one pickled flight to the end of binary file. Args: data: data to pickle full_path: name (potentially including path) of the pickled file date_segmentation: boolean indicating whether the date string yyyy-mm-dd should be prepended to the file name in full_path based on the current date, so that pickled files are segmented by date. timestamp: if date_segmentation is True, this is used rather than system time to generate the file name. verify: boolean indicating if we should verify that the pickled file object count increments by one, rewriting entire pickle file if it doesn't. Note that since this requires reading the entire pickle file and unpickling, it should only be done for small files / objects. <----SKIPPED LINES----> |