01234567890123456789012345678901234567890123456789012345678901234567890123456789
115711581159116011611162116311641165116611671168116911701171117211731174117511761177 1178117911801181118211831184118511861187118811891190 1191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224 12251226122712281229123012311232123312341235123612371238123912401241124212431244 62696270627162726273627462756276627762786279628062816282628362846285628662876288 62896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323 | <----SKIPPED LINES----> Boolean indicating whether the write was successful. """ try: with open(filename, 'w') as content_file: content_file.write(text) except IOError: if log_exception: Log('Unable to write to '+filename) return False return True def PrependFileName(full_path, prefix): """Converts /dir/file.png to /dir/prefixfile.png.""" directory, file_name = os.path.split(full_path) file_name = prefix+file_name return os.path.join(directory, file_name) def UnpickleObjectFromFile( full_path, date_segmentation, max_days=None, filenames=False): """Load a repository of pickled data into memory. Args: full_path: name (potentially including path) of the pickled file date_segmentation: If true, searches for all files that have a prefix of yyyy-mm-dd as a prefix to the file name specified in the full path, and loads them in sequence for unpickling; if false, uses the full_path as is and loads just that single file. max_days: Integer that, if specified, indicates maximum number of days of files to load back in; otherwise, loads all. That is, at most max_days files will be read. filenames: If true, rather than returning the list of data, returns a list of the filenames that would have been read. Returns: Return a list - either of the data, or of all the file names that would have been read. """ if date_segmentation: directory, file = os.path.split(full_path) d = '[0-9]' sep = '-' date_format = d*4 + sep + d*2 + sep + d*2 # yyyy-mm-dd exp = date_format + sep + file pattern = re.compile(exp) files = os.listdir(directory) if max_days: # no need to read any files older than x days earliest_date = EpochDisplayTime( time.time() - (max_days - 1) * SECONDS_IN_DAY, '%Y-%m-%d') files = [f for f in files if f[:10] >= earliest_date] files = sorted( [os.path.join(directory, f) for f in files if pattern.match(f)]) else: if os.path.exists(full_path): files = [full_path] else: return [] data = [] if filenames: return files for file in files: try: with open(file, 'rb') as f: while True: try: data.append(pickle.load(f)) except (UnicodeDecodeError) as e: Log('Process %s reading file %s gave error %s' % ( psutil.Process(os.getpid()).name(), f, e)) except (EOFError, pickle.UnpicklingError): pass return data cached_object_count = {} def PickleObjectToFile( data, full_path, date_segmentation, timestamp=None, verify=False): """Append one pickled flight to the end of binary file. Args: <----SKIPPED LINES----> UpdateDashboard(False) # Send an all-clear message last_heartbeat_time = now return last_heartbeat_time def VersionControl(): """Copies current instances of messageboard.py and arduino.py into repository. To aid debugging, we want to keep past versions of the code easily accessible, and linked to the errors that have been logged. This function copies the python code into a version control directory after adding in a date / time stamp to the file name. """ global VERSION_MESSAGEBOARD global VERSION_ARDUINO VERSION_MESSAGEBOARD = MakeVersionCopy('messageboard') VERSION_ARDUINO = MakeVersionCopy('arduino') def MakeVersionCopy(python_prefix): file_extension = '.py' live_name = python_prefix + '.py' live_path = os.path.join(CODE_REPOSITORY, live_name) epoch = os.path.getmtime(live_path) last_modified_suffix = EpochDisplayTime( epoch, format_string='-%Y-%m-%d-%H%M') version_name = python_prefix + last_modified_suffix + file_extension version_path = os.path.join(VERSION_REPOSITORY, version_name) if not os.path.exists(version_path): shutil.copyfile(live_path, version_path) return version_name def main(): """Traffic cop between radio, configuration, and messageboard. This is the main logic, checking for new flights, augmenting the radio signal with additional web-scraped data, and generating messages in a form presentable to the messageboard. """ VersionControl() # Since this clears log files, it should occur first before we start logging if '-s' in sys.argv: global SIMULATION_COUNTER SimulationSetup() last_heartbeat_time = HeartbeatRestart() init_timing = [(time.time(), 0)] # This flag slows down simulation time around a flight, great for <----SKIPPED LINES----> |
01234567890123456789012345678901234567890123456789012345678901234567890123456789
11571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254 62796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313 63146315631663176318631963206321632263236324632563266327632863296330633163326333 | <----SKIPPED LINES----> Boolean indicating whether the write was successful. """ try: with open(filename, 'w') as content_file: content_file.write(text) except IOError: if log_exception: Log('Unable to write to '+filename) return False return True def PrependFileName(full_path, prefix): """Converts /dir/file.png to /dir/prefixfile.png.""" directory, file_name = os.path.split(full_path) file_name = prefix+file_name return os.path.join(directory, file_name) def UnpickleObjectFromFile( full_path, date_segmentation, max_days=None, filenames=False, heartbeat=False): """Load a repository of pickled data into memory. Args: full_path: name (potentially including path) of the pickled file date_segmentation: If true, searches for all files that have a prefix of yyyy-mm-dd as a prefix to the file name specified in the full path, and loads them in sequence for unpickling; if false, uses the full_path as is and loads just that single file. max_days: Integer that, if specified, indicates maximum number of days of files to load back in; otherwise, loads all. That is, at most max_days files will be read. filenames: If true, rather than returning the list of data, returns a list of the filenames that would have been read. heartbeat: boolean indicating whether we should log heartbeats between each histogram to make sure monitoring does not mistake this slow procedure for being hung; this should be set to false if this is called from outside of messageboard.main. Returns: Return a list - either of the data, or of all the file names that would have been read. """ if date_segmentation: directory, file = os.path.split(full_path) d = '[0-9]' sep = '-' date_format = d*4 + sep + d*2 + sep + d*2 # yyyy-mm-dd exp = date_format + sep + file pattern = re.compile(exp) files = os.listdir(directory) if max_days: # no need to read any files older than x days earliest_date = EpochDisplayTime( time.time() - (max_days - 1) * SECONDS_IN_DAY, '%Y-%m-%d') files = [f for f in files if f[:10] >= earliest_date] files = sorted( [os.path.join(directory, f) for f in files if pattern.match(f)]) else: if os.path.exists(full_path): files = [full_path] else: return [] data = [] if filenames: return files for file in files: if heartbeat: Heartbeat() try: with open(file, 'rb') as f: while True: try: data.append(pickle.load(f)) except (UnicodeDecodeError) as e: Log('Process %s reading file %s gave error %s' % ( psutil.Process(os.getpid()).name(), f, e)) except (EOFError, pickle.UnpicklingError): pass return data cached_object_count = {} def PickleObjectToFile( data, full_path, date_segmentation, timestamp=None, verify=False): """Append one pickled flight to the end of binary file. Args: <----SKIPPED LINES----> UpdateDashboard(False) # Send an all-clear message last_heartbeat_time = now return last_heartbeat_time def VersionControl(): """Copies current instances of messageboard.py and arduino.py into repository. To aid debugging, we want to keep past versions of the code easily accessible, and linked to the errors that have been logged. This function copies the python code into a version control directory after adding in a date / time stamp to the file name. """ global VERSION_MESSAGEBOARD global VERSION_ARDUINO VERSION_MESSAGEBOARD = MakeVersionCopy('messageboard') VERSION_ARDUINO = MakeVersionCopy('arduino') def MakeVersionCopy(python_prefix): """Copies current instance of python file into repository.""" file_extension = '.py' live_name = python_prefix + '.py' live_path = os.path.join(CODE_REPOSITORY, live_name) epoch = os.path.getmtime(live_path) last_modified_suffix = EpochDisplayTime( epoch, format_string='-%Y-%m-%d-%H%M') version_name = python_prefix + last_modified_suffix + file_extension version_path = os.path.join(VERSION_REPOSITORY, version_name) if not os.path.exists(version_path): shutil.copyfile(live_path, version_path) return version_name def main(): """Traffic cop between radio, configuration, and messageboard. This is the main logic, checking for new flights, augmenting the radio signal with additional web-scraped data, and generating messages in a form presentable to the messageboard. """ VersionControl() # Since this clears log files, it should occur first before we start logging if '-s' in sys.argv: global SIMULATION_COUNTER SimulationSetup() last_heartbeat_time = HeartbeatRestart() init_timing = [(time.time(), 0)] # This flag slows down simulation time around a flight, great for <----SKIPPED LINES----> |