01234567890123456789012345678901234567890123456789012345678901234567890123456789
2425262728293031323334353637383940414243 4445464748495051525354555657585960616263 58795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919 5929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959 59605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985 | <----SKIPPED LINES----> import dateutils import numpy import matplotlib import matplotlib.pyplot import psutil import pycurl import pytz import requests import tzlocal import unidecode # pylint: disable=line-too-long from constants import RASPBERRY_PI, MESSAGEBOARD_PATH, WEBSERVER_PATH, SECURE_WEBSERVER_PATH, KEY, SECRET, SUBSCRIPTION_ID, LOCAL_KEY, LOCAL_VESTABOARD_ADDRESS # pylint: enable=line-too-long import arduino if RASPBERRY_PI: import gpiozero # pylint: disable=E0401 import RPi.GPIO # pylint: disable=E0401 VERBOSE = False # additional messages logged SHUTDOWN_SIGNAL = '' REBOOT_SIGNAL = False # to be tracked in the dashboard messages so that we know when a # restart due to exit (vs. a long delay in some processing) happened INSTANCE_START_TIME = time.time() SIMULATION = False SIMULATION_COUNTER = 0 SIMULATION_PREFIX = 'SIM_' PICKLE_DUMP_JSON_FILE = 'pickle/dump_json.pk' PICKLE_FA_JSON_FILE = 'pickle/fa_json.pk' MEMORY_DIRECTORY = 'memory/' DUMP_JSONS = None # loaded only if in simulation mode FA_JSONS = None # loaded only if in simulation mode <----SKIPPED LINES----> it can also be displayed on the html dashboard. """ rpi_restart = False global SHUTDOWN_SIGNAL running_minutes = (time.time() - startup_time) / SECONDS_IN_MINUTE running_days = running_minutes / MINUTES_IN_DAY process_restart_days = configuration.get('process_restart_days', 1) rpi_restart_days = configuration.get('rpi_restart_days', 1) mostly_quiet = not message_queue planes_in_radio = json_desc_dict.get('radio_range_flights') # script /home/pi/splitflap/backup.sh creates temp file in this # directory; after it is copied to the NAS, it is deleted backup_in_progress = os.listdir('/media/backup') all_quiet = mostly_quiet and not planes_in_radio and not backup_in_progress early_morn = 6 > int(EpochDisplayTime(time.time(), '%-H')) >= 5 # network test conditions number_of_intervals = 5 # at least three consecutive network failures minutes_per_interval = 10 minimum_uptime_minutes = number_of_intervals * minutes_per_interval # ---------------------------------------------------------------------------- # PROCESS RESTART SCENARIOS: restart process, but do not restart RPi # ---------------------------------------------------------------------------- process_restart = False if 'end_process' in configuration: process_restart = True msg = 'Process end requested via web form' RemoveSetting(configuration, 'end_process') elif all_quiet and early_morn and running_days >= process_restart_days: process_restart = True msg = ('All-quiet process restart triggered after %d days; ' 'actual runtime: %.2f days' % (process_restart_days, running_days)) elif mostly_quiet and early_morn and running_days >= process_restart_days + 1: process_restart = True msg = ('Mostly-quiet process restart triggered after %d days; ' 'actual runtime: %.2f days' % (process_restart_days + 1, running_days)) <----SKIPPED LINES----> RemoveSetting(configuration, 'soft_reboot') msg = 'Soft reboot requested via web form' rpi_restart = True elif all_quiet and early_morn and running_days >= rpi_restart_days: msg = ('All-quiet RPi reboot triggered after %d days; ' 'actual runtime: %.2f days' % (rpi_restart_days, running_days)) rpi_restart = True elif mostly_quiet and early_morn and running_days >= rpi_restart_days + 1: msg = ('Mostly-quiet RPi reboot triggered after %d days; ' 'actual runtime: %.2f days' % (rpi_restart_days + 1, running_days)) rpi_restart = True # Periodically, the RPi seems to lose the network connection even though # the router is up with a strong signal; when this happens, it does not # regain the signal until the RPi is restarted. This will be detected # by the network status being down for at least 30 minutes, as determined # by the .pk file capturing the network status (generated by the # network_monitor.py script). # # Specifically, if the most recent three values of the .pk are all 0s, and # we have been up and running for at least 30 minutes, restart. elif running_minutes > minimum_uptime_minutes: results = MostRecentNetworkStatuses(number_of_intervals) ( network_status_list, last_day, last_interval, first_day, first_interval) = results # Sum the list, handling strings in the list as if they were 0s sum_network_status_list = sum( [x if isinstance(x, int) else 0 for x in network_status_list]) if network_status_list and not sum_network_status_list: # all zeros msg = ( 'Running for %d minutes yet no network for %d intervals (index %d of ' 'day %s to index %d of day %s); rebooting in attempt to re-establish ' 'network connectivity' % ( running_minutes, number_of_intervals, first_interval, first_day, last_interval, last_day)) rpi_restart = True if rpi_restart: Log(msg) SHUTDOWN_SIGNAL = msg return rpi_restart # ---------------------------------------------------------------------------- # Only get here if neither process nor RPi restart # ---------------------------------------------------------------------------- return rpi_restart def MostRecentNetworkStatuses(number_of_intervals): """Returns a list of the most recent number of network statuses. The network status is managed by network_monitory.py, which, every few minutes updates the .pk file with the current network status. The data structure is a dictionary with day names (i.e.: 12-30-2022) and a list of 0s & 1s indicating network down / up respectively, for consecutive <----SKIPPED LINES----> |
01234567890123456789012345678901234567890123456789012345678901234567890123456789
2425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970 58865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926 59365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976 5977597859795980598159825983598459855986598759885989599059915992599359945995599659975998 | <----SKIPPED LINES----> import dateutils import numpy import matplotlib import matplotlib.pyplot import psutil import pycurl import pytz import requests import tzlocal import unidecode # pylint: disable=line-too-long from constants import RASPBERRY_PI, MESSAGEBOARD_PATH, WEBSERVER_PATH, SECURE_WEBSERVER_PATH, KEY, SECRET, SUBSCRIPTION_ID, LOCAL_KEY, LOCAL_VESTABOARD_ADDRESS # pylint: enable=line-too-long import arduino if RASPBERRY_PI: import gpiozero # pylint: disable=E0401 import RPi.GPIO # pylint: disable=E0401 sys.path.append('/home/pi/network_monitor') else: sys.path.append('/Users/davewertheimer/Desktop/art_projects/ping') # pylint: disable=wrong-import-order, wrong-import-position, import-error import network_monitor # pylint: enable=wrong-import-order, wrong-import-position, import-error VERBOSE = False # additional messages logged SHUTDOWN_SIGNAL = '' REBOOT_SIGNAL = False # to be tracked in the dashboard messages so that we know when a # restart due to exit (vs. a long delay in some processing) happened INSTANCE_START_TIME = time.time() SIMULATION = False SIMULATION_COUNTER = 0 SIMULATION_PREFIX = 'SIM_' PICKLE_DUMP_JSON_FILE = 'pickle/dump_json.pk' PICKLE_FA_JSON_FILE = 'pickle/fa_json.pk' MEMORY_DIRECTORY = 'memory/' DUMP_JSONS = None # loaded only if in simulation mode FA_JSONS = None # loaded only if in simulation mode <----SKIPPED LINES----> it can also be displayed on the html dashboard. """ rpi_restart = False global SHUTDOWN_SIGNAL running_minutes = (time.time() - startup_time) / SECONDS_IN_MINUTE running_days = running_minutes / MINUTES_IN_DAY process_restart_days = configuration.get('process_restart_days', 1) rpi_restart_days = configuration.get('rpi_restart_days', 1) mostly_quiet = not message_queue planes_in_radio = json_desc_dict.get('radio_range_flights') # script /home/pi/splitflap/backup.sh creates temp file in this # directory; after it is copied to the NAS, it is deleted backup_in_progress = os.listdir('/media/backup') all_quiet = mostly_quiet and not planes_in_radio and not backup_in_progress early_morn = 6 > int(EpochDisplayTime(time.time(), '%-H')) >= 5 # network test conditions number_of_intervals = 5 # at least three consecutive network failures minutes_per_interval = network_monitor.MINUTES_BETWEEN_PINGS minimum_uptime_minutes = number_of_intervals * minutes_per_interval # ---------------------------------------------------------------------------- # PROCESS RESTART SCENARIOS: restart process, but do not restart RPi # ---------------------------------------------------------------------------- process_restart = False if 'end_process' in configuration: process_restart = True msg = 'Process end requested via web form' RemoveSetting(configuration, 'end_process') elif all_quiet and early_morn and running_days >= process_restart_days: process_restart = True msg = ('All-quiet process restart triggered after %d days; ' 'actual runtime: %.2f days' % (process_restart_days, running_days)) elif mostly_quiet and early_morn and running_days >= process_restart_days + 1: process_restart = True msg = ('Mostly-quiet process restart triggered after %d days; ' 'actual runtime: %.2f days' % (process_restart_days + 1, running_days)) <----SKIPPED LINES----> RemoveSetting(configuration, 'soft_reboot') msg = 'Soft reboot requested via web form' rpi_restart = True elif all_quiet and early_morn and running_days >= rpi_restart_days: msg = ('All-quiet RPi reboot triggered after %d days; ' 'actual runtime: %.2f days' % (rpi_restart_days, running_days)) rpi_restart = True elif mostly_quiet and early_morn and running_days >= rpi_restart_days + 1: msg = ('Mostly-quiet RPi reboot triggered after %d days; ' 'actual runtime: %.2f days' % (rpi_restart_days + 1, running_days)) rpi_restart = True # Periodically, the RPi seems to lose the network connection even though # the router is up with a strong signal; when this happens, it does not # regain the signal until the RPi is restarted. This will be detected # by the network status being down for at least 30 minutes, as determined # by the .pk file capturing the network status (generated by the # network_monitor.py script). # # Specifically, if the most recent values of the .pk are all 0s, and # we have been up and running for at least that long, restart. elif running_minutes > minimum_uptime_minutes: results = MostRecentNetworkStatuses(number_of_intervals) ( network_status_list, last_day, last_interval, first_day, first_interval) = results # Sum the list, handling strings in the list as if they were 0s sum_network_status_list = sum( [x if isinstance(x, int) else 0 for x in network_status_list]) if network_status_list and not sum_network_status_list: # all zeros time_range_list = [ network_monitor.TimeFromTimeIndex(first_interval), '-', network_monitor.TimeFromTimeIndex(last_interval)] if first_day != last_day: time_range_list.insert(0, first_day) time_range_list.insert(3, last_day) time_range = ' '.join(time_range_list) msg = ( 'Running for %d minutes yet no network for %d intervals (%d minutes: ' '%s); rebooting in attempt to re-establish network connectivity' % ( running_minutes, number_of_intervals, minimum_uptime_minutes, time_range)) rpi_restart = True if rpi_restart: Log(msg) SHUTDOWN_SIGNAL = msg return rpi_restart # ---------------------------------------------------------------------------- # Only get here if neither process nor RPi restart # ---------------------------------------------------------------------------- return rpi_restart def MostRecentNetworkStatuses(number_of_intervals): """Returns a list of the most recent number of network statuses. The network status is managed by network_monitory.py, which, every few minutes updates the .pk file with the current network status. The data structure is a dictionary with day names (i.e.: 12-30-2022) and a list of 0s & 1s indicating network down / up respectively, for consecutive <----SKIPPED LINES----> |