01234567890123456789012345678901234567890123456789012345678901234567890123456789
58285829583058315832583358345835583658375838583958405841584258435844584558465847 5848584958505851585258535854 58555856585758585859586058615862586358645865 58665867586858695870587158725873587458755876 58775878 5879588058815882588358845885588658875888588958905891589258935894589558965897 589858995900590159025903590459055906590759085909 591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936 59375938593959405941 59425943594459455946594759485949595059515952595359545955595659575958595959605961 | <----SKIPPED LINES----> If we are logging the radio output faster than it is updating, then there will be sequential log files in the json list that are identical; we only need to process the first of these, and can ignore subsequent ones, without any change of output in the simulation results. This function identifies whether the current active json changed from the prior one. Returns: Boolean - True if different (and processing needed), False if identical """ if SIMULATION_COUNTER == 0: return True (this_json, unused_now) = DUMP_JSONS[SIMULATION_COUNTER] (last_json, unused_now) = DUMP_JSONS[SIMULATION_COUNTER - 1] return this_json != last_json def CheckRebootNeeded( startup_time, message_queue, json_desc_dict, configuration): """Reboot based on duration instance has been running. Reboot needed in one of the following situations: - All quiet: if running for over 24 hours and all is quiet (message queue empty, no planes in radio, and backup not currently in process). - Mostly quiet: if running for over 36 hours and message queue is empty and it's 4a. - Reboot requested via html form. - Persistent network problems failure that may be specific to the RPi Returns: Boolean indicating if a reboot is needed (True); otherwise, False. Also logs a message to the main log about why a reboot may be needed, and updates the global SHUTDOWN_SIGNAL with the same message sent to the log so that it can also be displayed on the html dashboard. """ reboot = False global SHUTDOWN_SIGNAL running_hours = (time.time() - startup_time) / SECONDS_IN_HOUR restart_days = configuration.get('restart_days', 1) min_hours = restart_days * HOURS_IN_DAY if ( running_hours >= min_hours and not message_queue and not json_desc_dict.get('radio_range_flights') and # script /home/pi/splitflap/backup.sh creates temp file in this # directory; after it is copied to the NAS, it is deleted not os.listdir('/media/backup')): msg = ('All quiet reboot triggered based on %d days (%d hours); ' 'actual runtime: %.2f hours' % (restart_days, min_hours, running_hours)) SHUTDOWN_SIGNAL = msg Log(msg) reboot = True # Wait another half day restart_days += 0.5 min_hours = restart_days * HOURS_IN_DAY if ( running_hours > min_hours and not message_queue and 6 >= int(EpochDisplayTime(time.time(), '%-H')) >= 4): msg = ('Early morning reboot triggered based on %.1f (%d hours); ' 'actual runtime: %.2f hours' % (restart_days, min_hours, running_hours)) SHUTDOWN_SIGNAL = msg Log(msg) reboot = True if 'soft_reboot' in configuration: msg = 'Soft reboot requested via web form' SHUTDOWN_SIGNAL = msg Log(msg) reboot = True RemoveSetting(configuration, 'soft_reboot') if 'end_process' in configuration: msg = 'Process end requested via web form' SHUTDOWN_SIGNAL = msg Log(msg) RemoveSetting(configuration, 'end_process') # Periodically, the RPi seems to lose the network connection even though # the router is up with a strong signal; when this happens, it does not # regain the signal until the RPi is restarted. This will be detected # by the network status being down for at least 30 minutes, as determined # by the .pk file capturing the network status (generated by the # network_monitor.py script). # # Specifically, if the most recent three values of the .pk are all 0s, and # we have been up and running for at least 30 minutes, restart. minimum_uptime_minutes = 30 # 30 minutes number_of_intervals = 3 run_time_minutes = (time.time() - startup_time) / SECONDS_IN_MINUTE if run_time_minutes > minimum_uptime_minutes: results = MostRecentNetworkStatuses(number_of_intervals) ( network_status_list, last_day, last_interval, first_day, first_interval) = results # Sum the list, handling strings in the list as if they were 0s sum_network_status_list = sum( [x if isinstance(x, int) else 0 for x in network_status_list]) if network_status_list and not sum_network_status_list: # all zeros msg = ( 'Running for %d yet no network for %d intervals (index %d of day ' '%s to index %d of day %s); rebooting in attempt to re-establish ' 'network connectivity' % ( run_time_minutes, number_of_intervals, first_day, first_interval, last_day, last_interval)) SHUTDOWN_SIGNAL = msg Log(msg) reboot = True return reboot def MostRecentNetworkStatuses(number_of_intervals): """Returns a list of the most recent number of network statuses. The network status is managed by network_monitory.py, which, every few minutes updates the .pk file with the current network status. The data structure is a dictionary with day names (i.e.: 12-30-2022) and a list of 0s & 1s indicating network down / up respectively, for consecutive time intervals. Because the .pk is updated infrequently (i.e.: only once every 10 min), we can cache it and re-read it only when it changes. If there are not enough intervals to provide the number_of_intervals requested, as many as are present will be provided. Args: number_of_intervals: integer of time periods desired. <----SKIPPED LINES----> |
01234567890123456789012345678901234567890123456789012345678901234567890123456789
5828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855 5856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880 5881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917 5918591959205921 59225923592459255926 59275928592959305931593259335934593559365937593859395940 594159425943594459455946594759485949595059515952595359545955595659575958595959605961 5962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986 | <----SKIPPED LINES----> If we are logging the radio output faster than it is updating, then there will be sequential log files in the json list that are identical; we only need to process the first of these, and can ignore subsequent ones, without any change of output in the simulation results. This function identifies whether the current active json changed from the prior one. Returns: Boolean - True if different (and processing needed), False if identical """ if SIMULATION_COUNTER == 0: return True (this_json, unused_now) = DUMP_JSONS[SIMULATION_COUNTER] (last_json, unused_now) = DUMP_JSONS[SIMULATION_COUNTER - 1] return this_json != last_json def CheckRebootNeeded( startup_time, message_queue, json_desc_dict, configuration): """Reboot based on duration instance has been running. Process restart needed in one of the following situations: - Process restart requested via html form. - All quiet: if it's all quiet, it's early morning, and we've been running for at least the configurable number of days - Mostly quiet: if it's mostly quiet, it's early morning, and we've been running for the time above plus an extra day Reboot needed in one of the following situations: - Reboot requested via html form. - Persistent network problems failure that may be specific to the RPi - All quiet: if it's all quiet, it's early morning, and we've been running for at least the configurable number of days - Mostly quiet: if it's mostly quiet, it's early morning, and we've been running for the time above plus an extra day "All Quiet": message queue empty, no planes in radio, backup not in process "Mostly Quiet": message queue empty Returns: Boolean indicating if a reboot is needed (True); otherwise, False. Also logs a message to the main log about why a reboot may be needed, and updates the global SHUTDOWN_SIGNAL with the same message sent to the log so that it can also be displayed on the html dashboard. """ rpi_restart = False global SHUTDOWN_SIGNAL running_minutes = (time.time() - startup_time) / SECONDS_IN_MINUTE running_days = running_minutes / MINUTES_IN_DAY process_restart_days = configuration.get('process_restart_days', 1) rpi_restart_days = configuration.get('rpi_restart_days', 1) mostly_quiet = not message_queue planes_in_radio = json_desc_dict.get('radio_range_flights') # script /home/pi/splitflap/backup.sh creates temp file in this # directory; after it is copied to the NAS, it is deleted backup_in_progress = os.listdir('/media/backup') all_quiet = mostly_quiet and not planes_in_radio and not backup_in_progress early_morn = 6 > int(EpochDisplayTime(time.time(), '%-H')) >= 5 # network test conditions minimum_uptime_minutes = 30 # 30 minutes number_of_intervals = 3 # at least three consecutive network failures # ---------------------------------------------------------------------------- # PROCESS RESTART SCENARIOS: restart process, but do not restart RPi # ---------------------------------------------------------------------------- process_restart = False if 'end_process' in configuration: process_restart = True msg = 'Process end requested via web form' RemoveSetting(configuration, 'end_process') elif all_quiet and early_morn and running_days >= process_restart_days: process_restart = True msg = ('All-quiet process restart triggered after %d days; ' 'actual runtime: %.2f days' % (process_restart_days, running_days)) elif mostly_quiet and early_morn and running_days >= process_restart_days + 1: process_restart = True msg = ('Mostly-quiet process restart triggered after %d days; ' 'actual runtime: %.2f days' % (process_restart_days + 1, running_days)) if process_restart: SHUTDOWN_SIGNAL = msg Log(msg) return rpi_restart # ---------------------------------------------------------------------------- # RPi RESTART SCENARIOS: restart RPi # ---------------------------------------------------------------------------- if 'soft_reboot' in configuration: RemoveSetting(configuration, 'soft_reboot') msg = 'Soft reboot requested via web form' rpi_restart = True elif all_quiet and early_morn and running_days >= rpi_restart_days: msg = ('All-quiet RPi reboot triggered after %d days; ' 'actual runtime: %.2f days' % (rpi_restart_days, running_days)) rpi_restart = True elif mostly_quiet and early_morn and running_days >= rpi_restart_days + 1: msg = ('Mostly-quiet RPi reboot triggered after %d days; ' 'actual runtime: %.2f days' % (rpi_restart_days + 1, running_days)) rpi_restart = True # Periodically, the RPi seems to lose the network connection even though # the router is up with a strong signal; when this happens, it does not # regain the signal until the RPi is restarted. This will be detected # by the network status being down for at least 30 minutes, as determined # by the .pk file capturing the network status (generated by the # network_monitor.py script). # # Specifically, if the most recent three values of the .pk are all 0s, and # we have been up and running for at least 30 minutes, restart. elif running_minutes > minimum_uptime_minutes: results = MostRecentNetworkStatuses(number_of_intervals) ( network_status_list, last_day, last_interval, first_day, first_interval) = results # Sum the list, handling strings in the list as if they were 0s sum_network_status_list = sum( [x if isinstance(x, int) else 0 for x in network_status_list]) if network_status_list and not sum_network_status_list: # all zeros msg = ( 'Running for %d minutes yet no network for %d intervals (index %d of ' 'day %s to index %d of day %s); rebooting in attempt to re-establish ' 'network connectivity' % ( running_minutes, number_of_intervals, first_day, first_interval, last_day, last_interval)) rpi_restart = True if rpi_restart: Log(msg) SHUTDOWN_SIGNAL = msg return rpi_restart # ---------------------------------------------------------------------------- # Only get here if neither process nor RPi restart # ---------------------------------------------------------------------------- return rpi_restart def MostRecentNetworkStatuses(number_of_intervals): """Returns a list of the most recent number of network statuses. The network status is managed by network_monitory.py, which, every few minutes updates the .pk file with the current network status. The data structure is a dictionary with day names (i.e.: 12-30-2022) and a list of 0s & 1s indicating network down / up respectively, for consecutive time intervals. Because the .pk is updated infrequently (i.e.: only once every 10 min), we can cache it and re-read it only when it changes. If there are not enough intervals to provide the number_of_intervals requested, as many as are present will be provided. Args: number_of_intervals: integer of time periods desired. <----SKIPPED LINES----> |