messageboard-2022-11-23-1250.py
01234567890123456789012345678901234567890123456789012345678901234567890123456789









58285829583058315832583358345835583658375838583958405841584258435844584558465847       5848584958505851585258535854       58555856585758585859586058615862586358645865  58665867586858695870587158725873587458755876                     58775878  5879588058815882588358845885588658875888588958905891589258935894589558965897 589858995900590159025903590459055906590759085909    591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936    59375938593959405941   59425943594459455946594759485949595059515952595359545955595659575958595959605961











                            <----SKIPPED LINES---->




  If we are logging the radio output faster than it is updating, then there
  will be sequential log files in the json list that are identical; we only
  need to process the first of these, and can ignore subsequent ones, without
  any change of output in the simulation results. This function identifies
  whether the current active json changed from the prior one.

  Returns:
    Boolean - True if different (and processing needed), False if identical
  """
  if SIMULATION_COUNTER == 0:
    return True
  (this_json, unused_now) = DUMP_JSONS[SIMULATION_COUNTER]
  (last_json, unused_now) = DUMP_JSONS[SIMULATION_COUNTER - 1]
  return this_json != last_json


def CheckRebootNeeded(
    startup_time, message_queue, json_desc_dict, configuration):
  """Reboot based on duration instance has been running.








  Reboot needed in one of the following situations:
  - All quiet: if running for over 24 hours and all is quiet (message queue
    empty, no planes in radio, and backup not currently in process).
  - Mostly quiet: if running for over 36 hours and message queue is empty and
    it's 4a.
  - Reboot requested via html form.
  - Persistent network problems failure that may be specific to the RPi








  Returns:
    Boolean indicating if a reboot is needed (True); otherwise, False.  Also
    logs a message to the main log about why a reboot may be needed, and updates
    the global SHUTDOWN_SIGNAL with the same message sent to the log so that
    it can also be displayed on the html dashboard.
  """
  reboot = False
  global SHUTDOWN_SIGNAL

  running_hours = (time.time() - startup_time) / SECONDS_IN_HOUR


  restart_days = configuration.get('restart_days', 1)

  min_hours = restart_days * HOURS_IN_DAY
  if (
      running_hours >= min_hours and
      not message_queue and
      not json_desc_dict.get('radio_range_flights') and
      # script /home/pi/splitflap/backup.sh creates temp file in this
      # directory; after it is copied to the NAS, it is deleted
      not os.listdir('/media/backup')):
    msg = ('All quiet reboot triggered based on %d days (%d hours); '





















           'actual runtime: %.2f hours' %
           (restart_days, min_hours, running_hours))


    SHUTDOWN_SIGNAL = msg
    Log(msg)
    reboot = True

  # Wait another half day
  restart_days += 0.5
  min_hours = restart_days * HOURS_IN_DAY
  if (
      running_hours > min_hours and
      not message_queue and
      6 >= int(EpochDisplayTime(time.time(), '%-H')) >= 4):
    msg = ('Early morning reboot triggered based on %.1f (%d hours); '
           'actual runtime: %.2f hours' %
           (restart_days, min_hours, running_hours))
    SHUTDOWN_SIGNAL = msg
    Log(msg)
    reboot = True

  if 'soft_reboot' in configuration:

    msg = 'Soft reboot requested via web form'
    SHUTDOWN_SIGNAL = msg
    Log(msg)
    reboot = True
    RemoveSetting(configuration, 'soft_reboot')

  if 'end_process' in configuration:
    msg = 'Process end requested via web form'
    SHUTDOWN_SIGNAL = msg
    Log(msg)
    RemoveSetting(configuration, 'end_process')





  # Periodically, the RPi seems to lose the network connection even though
  # the router is up with a strong signal; when this happens, it does not
  # regain the signal until the RPi is restarted.  This will be detected
  # by the network status being down for at least 30 minutes, as determined
  # by the .pk file capturing the network status (generated by the
  # network_monitor.py script).
  #
  # Specifically, if the most recent three values of the .pk are all 0s, and
  # we have been up and running for at least 30 minutes, restart.
  minimum_uptime_minutes = 30 # 30 minutes
  number_of_intervals = 3
  run_time_minutes = (time.time() - startup_time) / SECONDS_IN_MINUTE
  if run_time_minutes > minimum_uptime_minutes:
    results = MostRecentNetworkStatuses(number_of_intervals)
    (
        network_status_list, last_day, last_interval,
        first_day, first_interval) = results
    # Sum the list, handling strings in the list as if they were 0s
    sum_network_status_list = sum(
        [x if isinstance(x, int) else 0 for x in network_status_list])
    if network_status_list and not sum_network_status_list:  # all zeros
      msg = (
          'Running for %d yet no network for %d intervals (index %d of day '
          '%s to index %d of day %s); rebooting in attempt to re-establish '
          'network connectivity' % (
              run_time_minutes, number_of_intervals,
              first_day, first_interval, last_day, last_interval))




      SHUTDOWN_SIGNAL = msg
      Log(msg)
      reboot = True

  return reboot





def MostRecentNetworkStatuses(number_of_intervals):
  """Returns a list of the most recent number of network statuses.

  The network status is managed by network_monitory.py, which, every few
  minutes updates the .pk file with the current network status.  The data
  structure is a dictionary with day names (i.e.: 12-30-2022) and a list
  of 0s & 1s indicating network down / up respectively, for consecutive
  time intervals.

  Because the .pk is updated infrequently (i.e.: only once every 10 min),
  we can cache it and re-read it only when it changes.

  If there are not enough intervals to provide the number_of_intervals
  requested, as many as are present will be provided.

  Args:
    number_of_intervals: integer of time periods desired.





                            <----SKIPPED LINES---->





01234567890123456789012345678901234567890123456789012345678901234567890123456789









5828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855    5856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880   5881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917           5918591959205921   59225923592459255926 59275928592959305931593259335934593559365937593859395940   594159425943594459455946594759485949595059515952595359545955595659575958595959605961 5962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986











                            <----SKIPPED LINES---->




  If we are logging the radio output faster than it is updating, then there
  will be sequential log files in the json list that are identical; we only
  need to process the first of these, and can ignore subsequent ones, without
  any change of output in the simulation results. This function identifies
  whether the current active json changed from the prior one.

  Returns:
    Boolean - True if different (and processing needed), False if identical
  """
  if SIMULATION_COUNTER == 0:
    return True
  (this_json, unused_now) = DUMP_JSONS[SIMULATION_COUNTER]
  (last_json, unused_now) = DUMP_JSONS[SIMULATION_COUNTER - 1]
  return this_json != last_json


def CheckRebootNeeded(
    startup_time, message_queue, json_desc_dict, configuration):
  """Reboot based on duration instance has been running.

  Process restart needed in one of the following situations:
  - Process restart requested via html form.
  - All quiet: if it's all quiet, it's early morning, and we've been running
    for at least the configurable number of days
  - Mostly quiet: if it's mostly quiet, it's early morning, and we've been
    running for the time above plus an extra day

  Reboot needed in one of the following situations:




  - Reboot requested via html form.
  - Persistent network problems failure that may be specific to the RPi
  - All quiet: if it's all quiet, it's early morning, and we've been running
    for at least the configurable number of days
  - Mostly quiet: if it's mostly quiet, it's early morning, and we've been
    running for the time above plus an extra day

  "All Quiet": message queue empty, no planes in radio, backup not in process
  "Mostly Quiet": message queue empty

  Returns:
    Boolean indicating if a reboot is needed (True); otherwise, False.  Also
    logs a message to the main log about why a reboot may be needed, and updates
    the global SHUTDOWN_SIGNAL with the same message sent to the log so that
    it can also be displayed on the html dashboard.
  """
  rpi_restart = False
  global SHUTDOWN_SIGNAL

  running_minutes = (time.time() - startup_time) / SECONDS_IN_MINUTE
  running_days = running_minutes / MINUTES_IN_DAY
  process_restart_days = configuration.get('process_restart_days', 1)
  rpi_restart_days = configuration.get('rpi_restart_days', 1)

  mostly_quiet = not message_queue



  planes_in_radio = json_desc_dict.get('radio_range_flights')
  # script /home/pi/splitflap/backup.sh creates temp file in this
  # directory; after it is copied to the NAS, it is deleted
  backup_in_progress = os.listdir('/media/backup')
  all_quiet = mostly_quiet and not planes_in_radio and not backup_in_progress
  early_morn = 6 > int(EpochDisplayTime(time.time(), '%-H')) >= 5

  # network test conditions
  minimum_uptime_minutes = 30 # 30 minutes
  number_of_intervals = 3 # at least three consecutive network failures

  # ----------------------------------------------------------------------------
  # PROCESS RESTART SCENARIOS: restart process, but do not restart RPi
  # ----------------------------------------------------------------------------
  process_restart = False
  if 'end_process' in configuration:
    process_restart = True
    msg = 'Process end requested via web form'
    RemoveSetting(configuration, 'end_process')
  elif all_quiet and early_morn and running_days >= process_restart_days:
    process_restart = True
    msg = ('All-quiet process restart triggered after %d days; '
           'actual runtime: %.2f days' % (process_restart_days, running_days))
  elif mostly_quiet and early_morn and running_days >= process_restart_days + 1:
    process_restart = True
    msg = ('Mostly-quiet process restart triggered after %d days; '
           'actual runtime: %.2f days' %
           (process_restart_days + 1, running_days))

  if process_restart:
    SHUTDOWN_SIGNAL = msg
    Log(msg)
    return rpi_restart

  # ----------------------------------------------------------------------------
  # RPi RESTART SCENARIOS: restart RPi
  # ----------------------------------------------------------------------------











  if 'soft_reboot' in configuration:
    RemoveSetting(configuration, 'soft_reboot')
    msg = 'Soft reboot requested via web form'
    rpi_restart = True




  elif all_quiet and early_morn and running_days >= rpi_restart_days:
    msg = ('All-quiet RPi reboot triggered after %d days; '
           'actual runtime: %.2f days' % (rpi_restart_days, running_days))
    rpi_restart = True


  elif mostly_quiet and early_morn and running_days >= rpi_restart_days + 1:
    msg = ('Mostly-quiet RPi reboot triggered after %d days; '
           'actual runtime: %.2f days' % (rpi_restart_days + 1, running_days))
    rpi_restart = True
  # Periodically, the RPi seems to lose the network connection even though
  # the router is up with a strong signal; when this happens, it does not
  # regain the signal until the RPi is restarted.  This will be detected
  # by the network status being down for at least 30 minutes, as determined
  # by the .pk file capturing the network status (generated by the
  # network_monitor.py script).
  #
  # Specifically, if the most recent three values of the .pk are all 0s, and
  # we have been up and running for at least 30 minutes, restart.



  elif running_minutes > minimum_uptime_minutes:
    results = MostRecentNetworkStatuses(number_of_intervals)
    (
        network_status_list, last_day, last_interval,
        first_day, first_interval) = results
    # Sum the list, handling strings in the list as if they were 0s
    sum_network_status_list = sum(
        [x if isinstance(x, int) else 0 for x in network_status_list])
    if network_status_list and not sum_network_status_list:  # all zeros
      msg = (
          'Running for %d minutes yet no network for %d intervals (index %d of '
          'day %s to index %d of day %s); rebooting in attempt to re-establish '
          'network connectivity' % (
              running_minutes, number_of_intervals,
              first_day, first_interval, last_day, last_interval))
      rpi_restart = True

  if rpi_restart:
    Log(msg)
    SHUTDOWN_SIGNAL = msg
    return rpi_restart


  # ----------------------------------------------------------------------------
  # Only get here if neither process nor RPi restart
  # ----------------------------------------------------------------------------
  return rpi_restart


def MostRecentNetworkStatuses(number_of_intervals):
  """Returns a list of the most recent number of network statuses.

  The network status is managed by network_monitory.py, which, every few
  minutes updates the .pk file with the current network status.  The data
  structure is a dictionary with day names (i.e.: 12-30-2022) and a list
  of 0s & 1s indicating network down / up respectively, for consecutive
  time intervals.

  Because the .pk is updated infrequently (i.e.: only once every 10 min),
  we can cache it and re-read it only when it changes.

  If there are not enough intervals to provide the number_of_intervals
  requested, as many as are present will be provided.

  Args:
    number_of_intervals: integer of time periods desired.





                            <----SKIPPED LINES---->