messageboard-2020-05-30-1552.py
01234567890123456789012345678901234567890123456789012345678901234567890123456789









2425262728293031323334353637383940414243444546474849505152535455565758596061626364








48144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836 483748384839484048414842 4843484448454846   484748484849   485048514852485348544855  48564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881








48954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935








51005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143








55965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636








572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759



                            <----SKIPPED LINES---->




import matplotlib
import matplotlib.pyplot
import psutil
import pycurl
import pytz
import requests
import tzlocal
import unidecode

from constants import RASPBERRY_PI, MESSAGEBOARD_PATH, WEBSERVER_PATH

import arduino

if RASPBERRY_PI:
  import gpiozero  # pylint: disable=E0401
  import RPi.GPIO  # pylint: disable=E0401


VERBOSE = False  # additional messages logged

SHUTDOWN_SIGNAL = False
REBOOT_SIGNAL = False

SIMULATION = False
SIMULATION_COUNTER = 0
SIMULATION_PREFIX = 'SIM_'
PICKLE_DUMP_JSON_FILE = 'pickle/dump_json.pk'
PICKLE_FA_JSON_FILE = 'pickle/fa_json.pk'
DUMP_JSONS = None  # loaded only if in simulation mode
FA_JSONS = None  # loaded only if in simulation mode

HOME_LAT = 37.64406
HOME_LON = -122.43463
HOME = (HOME_LAT, HOME_LON) # lat / lon tuple of antenna
HOME_ALT = 29  #altitude in meters
RADIUS = 6371.0e3  # radius of earth in meters

FEET_IN_METER = 3.28084
FEET_IN_MILE = 5280
METERS_PER_SECOND_IN_KNOTS = 0.514444





                            <----SKIPPED LINES---->




  """
  if SIMULATION_COUNTER == 0:
    return True
  (this_json, unused_now) = DUMP_JSONS[SIMULATION_COUNTER]
  (last_json, unused_now) = DUMP_JSONS[SIMULATION_COUNTER - 1]
  return this_json != last_json


def CheckRebootNeeded(startup_time, message_queue, json_desc_dict, configuration):
  """Reboot based on duration instance has been running.

  Reboot needed in one of the following situations:
  - All quiet: if running for over 24 hours and all is quiet (message queue empty and
    no planes in radio).
  - Mostly quiet: if running for over 36 hours and message queue is empty and it's 3a.
  - Reboot requested via html form.

  Also checks if reset requested via html form.
  """
  reboot = False
  end_process = False

  running_hours = (time.time() - startup_time) / SECONDS_IN_HOUR

  if (
      running_hours >= HOURS_IN_DAY and
      not message_queue and
      not json_desc_dict.get('radio_range_flights')):
    reboot = True
    Log('All quiet reboot needed after running for %.2f hours' % running_hours)

  if (
      running_hours > HOURS_IN_DAY * 1.5 and
      not message_queue and
      int(EpochDisplayTime(time.time(), '%-H')) >= 3):



    reboot = True
    Log('Early morning reboot needed after running for %.2f hours' % running_hours)
  if 'soft_reboot' in configuration:



    reboot = True
    Log('Soft reboot requested via web form')
    RemoveSetting(configuration, 'soft_reboot')

  if 'end_process' in configuration:
    Log('Process end requested via web form')


    RemoveSetting(configuration, 'end_process')
    end_process = True

  if reboot or end_process:
    global SHUTDOWN_SIGNAL
    SHUTDOWN_SIGNAL = True

  return reboot


def InterruptRebootFromButton():
  """Sets flag so that the main loop will terminate when it completes the iteration.

  This function is only triggered by an physical button press.
  """
  global SHUTDOWN_SIGNAL
  SHUTDOWN_SIGNAL = True

  global REBOOT_SIGNAL
  REBOOT_SIGNAL = True

  RPi.GPIO.output(GPIO_SOFT_RESET[1], False)  # signal that reset received
  Log('Soft reboot requested by button push')


def InterruptShutdownFromSignal(signalNumber, unused_frame):




                            <----SKIPPED LINES---->




def PerformGracefulShutdown(queues, shutdown, reboot):
  """Complete the graceful shutdown process by cleaning up.

  Args:
    queues: iterable of queues shared with child processes to be closed
    shutdown: tuple of shared flags with child processes to initiate shutdown in children
    reboot: boolean indicating whether we should trigger a reboot
  """
  reboot_msg = ''
  if reboot:
    reboot_msg = ' and rebooting'
  Log('Shutting down self (%d)%s' % (os.getpid(), reboot_msg))

  for q in queues:
    q.close()
  for v in shutdown:  # send the shutdown signal to child processes
    v.value = 1
  if RASPBERRY_PI:
    RPi.GPIO.cleanup()

  UpdateDashboard(True)

  if reboot or REBOOT_SIGNAL:
    time.sleep(10)  # wait 10 seconds for children to shut down as well
    os.system('sudo reboot')
  sys.exit()


def FindRunningParents():
  """Returns list of proc ids of processes with identically-named python file running.

  In case there are multiple children processes spawned with the same name, such as via
  multiprocessing, this will only return the parent id (since a killed child process
  will likely just be respawned).
  """
  this_process_id = os.getpid()
  this_process_name = os.path.basename(sys.argv[0])
  pids = []
  pid_pairs = []
  for proc in psutil.process_iter():
    try:




                            <----SKIPPED LINES---->




  with the attributes to follow thru.

  Possible commands are updating a GPIO pin, replaying a recent flight to the board,
  generating a histogram, or updating the saved settings.

  Args:
    q: multiprocessing queue provided to both the Arduino processes
    flights: list of flights
    configuration: dictionary of settings
    message_queue: current message queue
    next_message_time: epoch of the next message to display to screen

  Returns:
    A 2-tuple of the (possibly-updated) message_queue and next_message_time.
  """
  while not q.empty():
    command, args = q.get()

    if command == 'pin':
      UpdateStatusLight(*args)
      
      str_args = [str(a) for a in args]
      msg = '|'.join(str_args)  #TODO
      Log(msg)

    elif command == 'replay':
      # a command might request info about flight to be (re)displayed, irrespective of
      # whether the screen is on; if so, let's put that message at the front of the message
      # queue, and delete any subsequent messages in queue because presumably the button
      # was pushed either a) when the screen was off (so no messages in queue), or b)
      # because the screen was on, but the last flight details got lost after other screens
      # that we're no longer interested in
      messageboard_flight_index = IdentifyFlightDisplayed(
          flights, configuration, display_all_hours=True)
      if messageboard_flight_index is not None:
        message_queue = [m for m in message_queue if m[0] != FLAG_MSG_INTERESTING]
        flight_message = CreateMessageAboutFlight(flights[messageboard_flight_index])
        message_queue = [(FLAG_MSG_FLIGHT, flight_message)]
        next_message_time = time.time()

    elif command == 'histogram':
      if not flights:
        Log('Histogram requested by remote %s but no flights in memory' % str(args))
      else:




                            <----SKIPPED LINES---->




  persistent_path = {}
  histogram = {}

  # Next up to print is index 0; this is a list of tuples:
  # tuple element#1: flag indicating the type of message that this is
  # tuple element#2: the message itself
  message_queue = []
  next_message_time = time.time()

  # We repeat the loop every x seconds; this ensures that if the processing time is long,
  # we don't wait another x seconds after processing completes
  next_loop_time = time.time() + LOOP_DELAY_SECONDS

  # These files are read only if the version on disk has been modified more recently
  # than the last time it was read
  last_dump_json_timestamp = 0

  WaitUntilKillComplete(already_running_ids)

  Log('Finishing initialization of %d; starting radio polling loop' % os.getpid())
  while not SIMULATION or SIMULATION_COUNTER < len(DUMP_JSONS):

    last_heartbeat_time = Heartbeat(last_heartbeat_time)

    new_configuration = ReadAndParseSettings(CONFIG_FILE)
    CheckForNewFilterCriteria(configuration, new_configuration, message_queue, flights)
    configuration = new_configuration

    ResetLogs(configuration)  # clear the logs if requested

    # if this is a SIMULATION, then process every diff dump. But if it isn't a simulation,
    # then only read & do related processing for the next dump if the last-modified
    # timestamp indicates the file has been updated since it was last read.
    tmp_timestamp = 0
    if not SIMULATION:
      dump_json_exists = os.path.exists(DUMP_JSON_FILE)
      if dump_json_exists:
        tmp_timestamp = os.path.getmtime(DUMP_JSON_FILE)
    if (SIMULATION and DumpJsonChanges()) or (
        not SIMULATION and dump_json_exists and tmp_timestamp > last_dump_json_timestamp):





                            <----SKIPPED LINES---->




    # lingering histogram file at the time of history restart.
    if histogram and not flights:
      Log('Histogram requested (%s) but no flights in memory' % histogram)
    if histogram and flights:
      message_queue.extend(TriggerHistograms(flights, histogram))

    # check time & if appropriate, display next message from queue
    next_message_time = ManageMessageQueue(message_queue, next_message_time, configuration)

    reboot = CheckRebootNeeded(startup_time, message_queue, json_desc_dict, configuration)

    CheckTemperature()

    if not SIMULATION:
      time.sleep(max(0, next_loop_time - time.time()))
      next_loop_time = time.time() + LOOP_DELAY_SECONDS
    else:
      SIMULATION_COUNTER += 1
      if simulation_slowdown:
        SimulationSlowdownNearFlight(flights, persistent_nearby_aircraft)
    if SHUTDOWN_SIGNAL:  # do a graceful exit
      PerformGracefulShutdown((to_remote_q, to_servo_q, to_main_q), shutdown, reboot)

  if SIMULATION:
    SimulationEnd(message_queue, flights)
  PerformGracefulShutdown((to_remote_q, to_servo_q, to_main_q), shutdown, reboot)


if __name__ == "__main__":
  #interrupt, as in ctrl-c
  signal.signal(signal.SIGINT, InterruptShutdownFromSignal)  #TODO

  #terminate, when another instance found or via kill
  signal.signal(signal.SIGTERM, InterruptShutdownFromSignal)  #TODO

  if '-i' in sys.argv:
    BootstrapInsightList()
  else:
    main()

01234567890123456789012345678901234567890123456789012345678901234567890123456789









2425262728293031323334353637383940414243444546474849505152535455565758596061626364








481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858 4859486048614862486348644865     48664867486848694870487148724873487448754876487748784879488048814882488348844885








48994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939








51045105510651075108510951105111511251135114511551165117511851195120512151225123    51245125512651275128512951305131513251335134513551365137513851395140514151425143








55965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636








57215722572357245725572657275728572957305731573257335734573557365737573857395740  57415742574357445745574657475748574957505751575257535754575557565757



                            <----SKIPPED LINES---->




import matplotlib
import matplotlib.pyplot
import psutil
import pycurl
import pytz
import requests
import tzlocal
import unidecode

from constants import RASPBERRY_PI, MESSAGEBOARD_PATH, WEBSERVER_PATH

import arduino

if RASPBERRY_PI:
  import gpiozero  # pylint: disable=E0401
  import RPi.GPIO  # pylint: disable=E0401


VERBOSE = False  # additional messages logged

SHUTDOWN_SIGNAL = ''
REBOOT_SIGNAL = False

SIMULATION = False
SIMULATION_COUNTER = 0
SIMULATION_PREFIX = 'SIM_'
PICKLE_DUMP_JSON_FILE = 'pickle/dump_json.pk'
PICKLE_FA_JSON_FILE = 'pickle/fa_json.pk'
DUMP_JSONS = None  # loaded only if in simulation mode
FA_JSONS = None  # loaded only if in simulation mode

HOME_LAT = 37.64406
HOME_LON = -122.43463
HOME = (HOME_LAT, HOME_LON) # lat / lon tuple of antenna
HOME_ALT = 29  #altitude in meters
RADIUS = 6371.0e3  # radius of earth in meters

FEET_IN_METER = 3.28084
FEET_IN_MILE = 5280
METERS_PER_SECOND_IN_KNOTS = 0.514444





                            <----SKIPPED LINES---->




  """
  if SIMULATION_COUNTER == 0:
    return True
  (this_json, unused_now) = DUMP_JSONS[SIMULATION_COUNTER]
  (last_json, unused_now) = DUMP_JSONS[SIMULATION_COUNTER - 1]
  return this_json != last_json


def CheckRebootNeeded(startup_time, message_queue, json_desc_dict, configuration):
  """Reboot based on duration instance has been running.

  Reboot needed in one of the following situations:
  - All quiet: if running for over 24 hours and all is quiet (message queue empty and
    no planes in radio).
  - Mostly quiet: if running for over 36 hours and message queue is empty and it's 3a.
  - Reboot requested via html form.

  Also checks if reset requested via html form.
  """
  reboot = False
  global SHUTDOWN_SIGNAL

  running_hours = (time.time() - startup_time) / SECONDS_IN_HOUR

  if (
      running_hours >= HOURS_IN_DAY and
      not message_queue and
      not json_desc_dict.get('radio_range_flights')):
    reboot = True
    Log('All quiet reboot needed after running for %.2f hours' % running_hours)

  if (
      running_hours > HOURS_IN_DAY * 1.5 and
      not message_queue and
      int(EpochDisplayTime(time.time(), '%-H')) >= 3):
    msg = 'Early morning reboot needed after running for %.2f hours' % running_hours
    SHUTDOWN_SIGNAL = msg
    Log(msg)
    reboot = True

  if 'soft_reboot' in configuration:
    msg = 'Soft reboot requested via web form'
    SHUTDOWN_SIGNAL = msg
    Log(msg)
    reboot = True

    RemoveSetting(configuration, 'soft_reboot')

  if 'end_process' in configuration:
    msg = 'Process end requested via web form'
    SHUTDOWN_SIGNAL = msg
    Log(msg)
    RemoveSetting(configuration, 'end_process')






  return reboot


def InterruptRebootFromButton():
  """Sets flag so that the main loop will terminate when it completes the iteration.

  This function is only triggered by an physical button press.
  """
  global SHUTDOWN_SIGNAL
  SHUTDOWN_SIGNAL = True

  global REBOOT_SIGNAL
  REBOOT_SIGNAL = True

  RPi.GPIO.output(GPIO_SOFT_RESET[1], False)  # signal that reset received
  Log('Soft reboot requested by button push')


def InterruptShutdownFromSignal(signalNumber, unused_frame):




                            <----SKIPPED LINES---->




def PerformGracefulShutdown(queues, shutdown, reboot):
  """Complete the graceful shutdown process by cleaning up.

  Args:
    queues: iterable of queues shared with child processes to be closed
    shutdown: tuple of shared flags with child processes to initiate shutdown in children
    reboot: boolean indicating whether we should trigger a reboot
  """
  reboot_msg = ''
  if reboot:
    reboot_msg = ' and rebooting'
  Log('Shutting down self (%d)%s' % (os.getpid(), reboot_msg))

  for q in queues:
    q.close()
  for v in shutdown:  # send the shutdown signal to child processes
    v.value = 1
  if RASPBERRY_PI:
    RPi.GPIO.cleanup()

  UpdateDashboard(True, failure_message=SHUTDOWN_SIGNAL)

  if reboot or REBOOT_SIGNAL:
    time.sleep(10)  # wait 10 seconds for children to shut down as well
    os.system('sudo reboot')
  sys.exit()


def FindRunningParents():
  """Returns list of proc ids of processes with identically-named python file running.

  In case there are multiple children processes spawned with the same name, such as via
  multiprocessing, this will only return the parent id (since a killed child process
  will likely just be respawned).
  """
  this_process_id = os.getpid()
  this_process_name = os.path.basename(sys.argv[0])
  pids = []
  pid_pairs = []
  for proc in psutil.process_iter():
    try:




                            <----SKIPPED LINES---->




  with the attributes to follow thru.

  Possible commands are updating a GPIO pin, replaying a recent flight to the board,
  generating a histogram, or updating the saved settings.

  Args:
    q: multiprocessing queue provided to both the Arduino processes
    flights: list of flights
    configuration: dictionary of settings
    message_queue: current message queue
    next_message_time: epoch of the next message to display to screen

  Returns:
    A 2-tuple of the (possibly-updated) message_queue and next_message_time.
  """
  while not q.empty():
    command, args = q.get()

    if command == 'pin':
      UpdateStatusLight(*args)





    elif command == 'replay':
      # a command might request info about flight to be (re)displayed, irrespective of
      # whether the screen is on; if so, let's put that message at the front of the message
      # queue, and delete any subsequent messages in queue because presumably the button
      # was pushed either a) when the screen was off (so no messages in queue), or b)
      # because the screen was on, but the last flight details got lost after other screens
      # that we're no longer interested in
      messageboard_flight_index = IdentifyFlightDisplayed(
          flights, configuration, display_all_hours=True)
      if messageboard_flight_index is not None:
        message_queue = [m for m in message_queue if m[0] != FLAG_MSG_INTERESTING]
        flight_message = CreateMessageAboutFlight(flights[messageboard_flight_index])
        message_queue = [(FLAG_MSG_FLIGHT, flight_message)]
        next_message_time = time.time()

    elif command == 'histogram':
      if not flights:
        Log('Histogram requested by remote %s but no flights in memory' % str(args))
      else:




                            <----SKIPPED LINES---->




  persistent_path = {}
  histogram = {}

  # Next up to print is index 0; this is a list of tuples:
  # tuple element#1: flag indicating the type of message that this is
  # tuple element#2: the message itself
  message_queue = []
  next_message_time = time.time()

  # We repeat the loop every x seconds; this ensures that if the processing time is long,
  # we don't wait another x seconds after processing completes
  next_loop_time = time.time() + LOOP_DELAY_SECONDS

  # These files are read only if the version on disk has been modified more recently
  # than the last time it was read
  last_dump_json_timestamp = 0

  WaitUntilKillComplete(already_running_ids)

  Log('Finishing initialization of %d; starting radio polling loop' % os.getpid())
  while (not SIMULATION or SIMULATION_COUNTER < len(DUMP_JSONS)) and not SHUTDOWN_SIGNAL:

    last_heartbeat_time = Heartbeat(last_heartbeat_time)

    new_configuration = ReadAndParseSettings(CONFIG_FILE)
    CheckForNewFilterCriteria(configuration, new_configuration, message_queue, flights)
    configuration = new_configuration

    ResetLogs(configuration)  # clear the logs if requested

    # if this is a SIMULATION, then process every diff dump. But if it isn't a simulation,
    # then only read & do related processing for the next dump if the last-modified
    # timestamp indicates the file has been updated since it was last read.
    tmp_timestamp = 0
    if not SIMULATION:
      dump_json_exists = os.path.exists(DUMP_JSON_FILE)
      if dump_json_exists:
        tmp_timestamp = os.path.getmtime(DUMP_JSON_FILE)
    if (SIMULATION and DumpJsonChanges()) or (
        not SIMULATION and dump_json_exists and tmp_timestamp > last_dump_json_timestamp):





                            <----SKIPPED LINES---->




    # lingering histogram file at the time of history restart.
    if histogram and not flights:
      Log('Histogram requested (%s) but no flights in memory' % histogram)
    if histogram and flights:
      message_queue.extend(TriggerHistograms(flights, histogram))

    # check time & if appropriate, display next message from queue
    next_message_time = ManageMessageQueue(message_queue, next_message_time, configuration)

    reboot = CheckRebootNeeded(startup_time, message_queue, json_desc_dict, configuration)

    CheckTemperature()

    if not SIMULATION:
      time.sleep(max(0, next_loop_time - time.time()))
      next_loop_time = time.time() + LOOP_DELAY_SECONDS
    else:
      SIMULATION_COUNTER += 1
      if simulation_slowdown:
        SimulationSlowdownNearFlight(flights, persistent_nearby_aircraft)



  if SIMULATION:
    SimulationEnd(message_queue, flights)
  PerformGracefulShutdown((to_remote_q, to_servo_q, to_main_q), shutdown, reboot)


if __name__ == "__main__":
  #interrupt, as in ctrl-c
  signal.signal(signal.SIGINT, InterruptShutdownFromSignal)  #TODO

  #terminate, when another instance found or via kill
  signal.signal(signal.SIGTERM, InterruptShutdownFromSignal)  #TODO

  if '-i' in sys.argv:
    BootstrapInsightList()
  else:
    main()