01234567890123456789012345678901234567890123456789012345678901234567890123456789
2425262728293031323334353637383940414243 4445464748495051525354555657585960616263 58795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919 5929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959 59605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985 |
<----SKIPPED LINES---->
import dateutils
import numpy
import matplotlib
import matplotlib.pyplot
import psutil
import pycurl
import pytz
import requests
import tzlocal
import unidecode
# pylint: disable=line-too-long
from constants import RASPBERRY_PI, MESSAGEBOARD_PATH, WEBSERVER_PATH, SECURE_WEBSERVER_PATH, KEY, SECRET, SUBSCRIPTION_ID, LOCAL_KEY, LOCAL_VESTABOARD_ADDRESS
# pylint: enable=line-too-long
import arduino
if RASPBERRY_PI:
import gpiozero # pylint: disable=E0401
import RPi.GPIO # pylint: disable=E0401
VERBOSE = False # additional messages logged
SHUTDOWN_SIGNAL = ''
REBOOT_SIGNAL = False
# to be tracked in the dashboard messages so that we know when a
# restart due to exit (vs. a long delay in some processing) happened
INSTANCE_START_TIME = time.time()
SIMULATION = False
SIMULATION_COUNTER = 0
SIMULATION_PREFIX = 'SIM_'
PICKLE_DUMP_JSON_FILE = 'pickle/dump_json.pk'
PICKLE_FA_JSON_FILE = 'pickle/fa_json.pk'
MEMORY_DIRECTORY = 'memory/'
DUMP_JSONS = None # loaded only if in simulation mode
FA_JSONS = None # loaded only if in simulation mode
<----SKIPPED LINES---->
it can also be displayed on the html dashboard.
"""
rpi_restart = False
global SHUTDOWN_SIGNAL
running_minutes = (time.time() - startup_time) / SECONDS_IN_MINUTE
running_days = running_minutes / MINUTES_IN_DAY
process_restart_days = configuration.get('process_restart_days', 1)
rpi_restart_days = configuration.get('rpi_restart_days', 1)
mostly_quiet = not message_queue
planes_in_radio = json_desc_dict.get('radio_range_flights')
# script /home/pi/splitflap/backup.sh creates temp file in this
# directory; after it is copied to the NAS, it is deleted
backup_in_progress = os.listdir('/media/backup')
all_quiet = mostly_quiet and not planes_in_radio and not backup_in_progress
early_morn = 6 > int(EpochDisplayTime(time.time(), '%-H')) >= 5
# network test conditions
number_of_intervals = 5 # at least three consecutive network failures
minutes_per_interval = 10
minimum_uptime_minutes = number_of_intervals * minutes_per_interval
# ----------------------------------------------------------------------------
# PROCESS RESTART SCENARIOS: restart process, but do not restart RPi
# ----------------------------------------------------------------------------
process_restart = False
if 'end_process' in configuration:
process_restart = True
msg = 'Process end requested via web form'
RemoveSetting(configuration, 'end_process')
elif all_quiet and early_morn and running_days >= process_restart_days:
process_restart = True
msg = ('All-quiet process restart triggered after %d days; '
'actual runtime: %.2f days' % (process_restart_days, running_days))
elif mostly_quiet and early_morn and running_days >= process_restart_days + 1:
process_restart = True
msg = ('Mostly-quiet process restart triggered after %d days; '
'actual runtime: %.2f days' %
(process_restart_days + 1, running_days))
<----SKIPPED LINES---->
RemoveSetting(configuration, 'soft_reboot')
msg = 'Soft reboot requested via web form'
rpi_restart = True
elif all_quiet and early_morn and running_days >= rpi_restart_days:
msg = ('All-quiet RPi reboot triggered after %d days; '
'actual runtime: %.2f days' % (rpi_restart_days, running_days))
rpi_restart = True
elif mostly_quiet and early_morn and running_days >= rpi_restart_days + 1:
msg = ('Mostly-quiet RPi reboot triggered after %d days; '
'actual runtime: %.2f days' % (rpi_restart_days + 1, running_days))
rpi_restart = True
# Periodically, the RPi seems to lose the network connection even though
# the router is up with a strong signal; when this happens, it does not
# regain the signal until the RPi is restarted. This will be detected
# by the network status being down for at least 30 minutes, as determined
# by the .pk file capturing the network status (generated by the
# network_monitor.py script).
#
# Specifically, if the most recent three values of the .pk are all 0s, and
# we have been up and running for at least 30 minutes, restart.
elif running_minutes > minimum_uptime_minutes:
results = MostRecentNetworkStatuses(number_of_intervals)
(
network_status_list, last_day, last_interval,
first_day, first_interval) = results
# Sum the list, handling strings in the list as if they were 0s
sum_network_status_list = sum(
[x if isinstance(x, int) else 0 for x in network_status_list])
if network_status_list and not sum_network_status_list: # all zeros
msg = (
'Running for %d minutes yet no network for %d intervals (index %d of '
'day %s to index %d of day %s); rebooting in attempt to re-establish '
'network connectivity' % (
running_minutes, number_of_intervals,
first_interval, first_day, last_interval, last_day))
rpi_restart = True
if rpi_restart:
Log(msg)
SHUTDOWN_SIGNAL = msg
return rpi_restart
# ----------------------------------------------------------------------------
# Only get here if neither process nor RPi restart
# ----------------------------------------------------------------------------
return rpi_restart
def MostRecentNetworkStatuses(number_of_intervals):
"""Returns a list of the most recent number of network statuses.
The network status is managed by network_monitory.py, which, every few
minutes updates the .pk file with the current network status. The data
structure is a dictionary with day names (i.e.: 12-30-2022) and a list
of 0s & 1s indicating network down / up respectively, for consecutive
<----SKIPPED LINES---->
|
01234567890123456789012345678901234567890123456789012345678901234567890123456789
2425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970 58865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926 59365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976 5977597859795980598159825983598459855986598759885989599059915992599359945995599659975998 |
<----SKIPPED LINES---->
import dateutils
import numpy
import matplotlib
import matplotlib.pyplot
import psutil
import pycurl
import pytz
import requests
import tzlocal
import unidecode
# pylint: disable=line-too-long
from constants import RASPBERRY_PI, MESSAGEBOARD_PATH, WEBSERVER_PATH, SECURE_WEBSERVER_PATH, KEY, SECRET, SUBSCRIPTION_ID, LOCAL_KEY, LOCAL_VESTABOARD_ADDRESS
# pylint: enable=line-too-long
import arduino
if RASPBERRY_PI:
import gpiozero # pylint: disable=E0401
import RPi.GPIO # pylint: disable=E0401
sys.path.append('/home/pi/network_monitor')
else:
sys.path.append('/Users/davewertheimer/Desktop/art_projects/ping')
# pylint: disable=wrong-import-order, wrong-import-position, import-error
import network_monitor
# pylint: enable=wrong-import-order, wrong-import-position, import-error
VERBOSE = False # additional messages logged
SHUTDOWN_SIGNAL = ''
REBOOT_SIGNAL = False
# to be tracked in the dashboard messages so that we know when a
# restart due to exit (vs. a long delay in some processing) happened
INSTANCE_START_TIME = time.time()
SIMULATION = False
SIMULATION_COUNTER = 0
SIMULATION_PREFIX = 'SIM_'
PICKLE_DUMP_JSON_FILE = 'pickle/dump_json.pk'
PICKLE_FA_JSON_FILE = 'pickle/fa_json.pk'
MEMORY_DIRECTORY = 'memory/'
DUMP_JSONS = None # loaded only if in simulation mode
FA_JSONS = None # loaded only if in simulation mode
<----SKIPPED LINES---->
it can also be displayed on the html dashboard.
"""
rpi_restart = False
global SHUTDOWN_SIGNAL
running_minutes = (time.time() - startup_time) / SECONDS_IN_MINUTE
running_days = running_minutes / MINUTES_IN_DAY
process_restart_days = configuration.get('process_restart_days', 1)
rpi_restart_days = configuration.get('rpi_restart_days', 1)
mostly_quiet = not message_queue
planes_in_radio = json_desc_dict.get('radio_range_flights')
# script /home/pi/splitflap/backup.sh creates temp file in this
# directory; after it is copied to the NAS, it is deleted
backup_in_progress = os.listdir('/media/backup')
all_quiet = mostly_quiet and not planes_in_radio and not backup_in_progress
early_morn = 6 > int(EpochDisplayTime(time.time(), '%-H')) >= 5
# network test conditions
number_of_intervals = 5 # at least three consecutive network failures
minutes_per_interval = network_monitor.MINUTES_BETWEEN_PINGS
minimum_uptime_minutes = number_of_intervals * minutes_per_interval
# ----------------------------------------------------------------------------
# PROCESS RESTART SCENARIOS: restart process, but do not restart RPi
# ----------------------------------------------------------------------------
process_restart = False
if 'end_process' in configuration:
process_restart = True
msg = 'Process end requested via web form'
RemoveSetting(configuration, 'end_process')
elif all_quiet and early_morn and running_days >= process_restart_days:
process_restart = True
msg = ('All-quiet process restart triggered after %d days; '
'actual runtime: %.2f days' % (process_restart_days, running_days))
elif mostly_quiet and early_morn and running_days >= process_restart_days + 1:
process_restart = True
msg = ('Mostly-quiet process restart triggered after %d days; '
'actual runtime: %.2f days' %
(process_restart_days + 1, running_days))
<----SKIPPED LINES---->
RemoveSetting(configuration, 'soft_reboot')
msg = 'Soft reboot requested via web form'
rpi_restart = True
elif all_quiet and early_morn and running_days >= rpi_restart_days:
msg = ('All-quiet RPi reboot triggered after %d days; '
'actual runtime: %.2f days' % (rpi_restart_days, running_days))
rpi_restart = True
elif mostly_quiet and early_morn and running_days >= rpi_restart_days + 1:
msg = ('Mostly-quiet RPi reboot triggered after %d days; '
'actual runtime: %.2f days' % (rpi_restart_days + 1, running_days))
rpi_restart = True
# Periodically, the RPi seems to lose the network connection even though
# the router is up with a strong signal; when this happens, it does not
# regain the signal until the RPi is restarted. This will be detected
# by the network status being down for at least 30 minutes, as determined
# by the .pk file capturing the network status (generated by the
# network_monitor.py script).
#
# Specifically, if the most recent values of the .pk are all 0s, and
# we have been up and running for at least that long, restart.
elif running_minutes > minimum_uptime_minutes:
results = MostRecentNetworkStatuses(number_of_intervals)
(
network_status_list, last_day, last_interval,
first_day, first_interval) = results
# Sum the list, handling strings in the list as if they were 0s
sum_network_status_list = sum(
[x if isinstance(x, int) else 0 for x in network_status_list])
if network_status_list and not sum_network_status_list: # all zeros
time_range_list = [
network_monitor.TimeFromTimeIndex(first_interval), '-',
network_monitor.TimeFromTimeIndex(last_interval)]
if first_day != last_day:
time_range_list.insert(0, first_day)
time_range_list.insert(3, last_day)
time_range = ' '.join(time_range_list)
msg = (
'Running for %d minutes yet no network for %d intervals (%d minutes: '
'%s); rebooting in attempt to re-establish network connectivity' % (
running_minutes, number_of_intervals, minimum_uptime_minutes,
time_range))
rpi_restart = True
if rpi_restart:
Log(msg)
SHUTDOWN_SIGNAL = msg
return rpi_restart
# ----------------------------------------------------------------------------
# Only get here if neither process nor RPi restart
# ----------------------------------------------------------------------------
return rpi_restart
def MostRecentNetworkStatuses(number_of_intervals):
"""Returns a list of the most recent number of network statuses.
The network status is managed by network_monitory.py, which, every few
minutes updates the .pk file with the current network status. The data
structure is a dictionary with day names (i.e.: 12-30-2022) and a list
of 0s & 1s indicating network down / up respectively, for consecutive
<----SKIPPED LINES---->
|