<----SKIPPED LINES---->




import shutil
import signal
import statistics
import sys
import textwrap
import time

import bs4
import dateutil.relativedelta
import filelock
import numpy
import matplotlib
import matplotlib.pyplot
import psutil
import pycurl
import pytz
import requests
import tzlocal
import unidecode

import arduino








RASPBERRY_PI = psutil.sys.platform.title() == 'Linux'
if RASPBERRY_PI:
  import gpiozero  # pylint: disable=E0401
  import RPi.GPIO  # pylint: disable=E0401

SHUTDOWN_SIGNAL = False

SIMULATION = False
SIMULATION_COUNTER = 0
SIMULATION_PREFIX = 'SIM_'
PICKLE_DUMP_JSON_FILE = 'pickle/dump_json.pk'
PICKLE_FA_JSON_FILE = 'pickle/fa_json.pk'
DUMP_JSONS = None  # loaded only if in simulation mode
FA_JSONS = None  # loaded only if in simulation mode

HOME_LAT = 37.64406
HOME_LON = -122.43463
HOME = (HOME_LAT, HOME_LON) # lat / lon tuple of antenna
HOME_ALT = 29  #altitude in meters
RADIUS = 6371.0e3  # radius of earth in meters

FEET_IN_METER = 3.28084
FEET_IN_MILE = 5280
METERS_PER_SECOND_IN_KNOTS = 0.514444

MIN_METERS = 5000/FEET_IN_METER # only planes within this distance will be detailed
# planes not seen within MIN_METERS in PERSISTENCE_SECONDS seconds will be dropped from
# the nearby list
PERSISTENCE_SECONDS = 10
TRUNCATE = 50  # max number of keys to include in a histogram image file
# number of seconds to pause between each radio poll / command processing loop
LOOP_DELAY_SECONDS = 1



# This file is where the radio drops its json file
DUMP_JSON_FILE = '/run/readsb/aircraft.json'

# This is the directory that stores all the ancillary messageboard configuration files
# that do not need to be exposed via the webserver
MESSAGEBOARD_PATH = '/home/pi/splitflap/'
# This is the directory of the webserver; files placed here are available at
# http://adsbx-custom.local/; files placed in this directly are visible via a browser
WEBSERVER_PATH = '/var/www/html/'

# At the time a flight is first identified as being of interest (in that it falls
# within MIN_METERS meters of HOME), it - and core attributes derived from FlightAware,
# if any - is appended to the end of this pickle file. However, since this file is
# cached in working memory, flights older than 30 days are flushed from this periodically.
PICKLE_FLIGHTS = 'pickle/flights.pk'

CACHED_ELEMENT_PREFIX = 'cached_'

# This web-exposed file is used for non-error messages that might highlight data or
# code logic to check into. It is only cleared out manually.
LOGFILE = 'log.txt'
LOGFILE_LOCK = 'log.txt.lock'
# Identical to the LOGFILE, except it includes just the most recent n lines. Newest
# lines are at the end.
ROLLING_LOGFILE = 'rolling_log.txt' #file for error messages

# Users can trigger .png histograms analogous to the text ones from the web interface;
# this is the folder (within WEBSERVER_PATH) where those files are placed
WEBSERVER_IMAGE_FOLDER = 'images/'
# Multiple histograms can be generated, i.e. for airline, aircraft, day of week, etc.
# The output files are named by the prefix & suffix, i.e.: prefix + type + . + suffix,
# as in histogram_aircraft.png. These names match up to the names expected by the html
# page that displays the images. Also, note that the suffix is interpreted by matplotlib
# to identify the image format to create.
HISTOGRAM_IMAGE_PREFIX = 'histogram_'
HISTOGRAM_IMAGE_SUFFIX = 'png'
# For those of the approximately ten different types of histograms _not_ generated,
# an empty image is copied into the location expected by the webpage instead; this is
# the location of that "empty" image file.
HISTOGRAM_EMPTY_IMAGE_FILE = 'empty.png'

# This file indicates a pending request for histograms - either png, text-based, or
# both - that may have come from either the web or Arduino interfaces; regardless,
# once it is processed, this file is deleted. The contents are concatenated key-value
# pairs, histogram=all;histogram_history=24h; etc.
HISTOGRAM_CONFIG_FILE = 'histogram.txt'

# This contains concatenated key-value configuration attributes in a similar format
# to the HISTOGRAM_CONFIG_FILE that are exposed to the user via the web interface or,
# for a subset of them, through the Arduino interface. They are polled at every iteration
# so that the most current value is always leveraged by the running software.
CONFIG_FILE = 'settings.txt'

# A few key settings for the messageboard are its sensitivity to displaying flights -
# though it logs all flights within range, it may not be desirable to display all
# flights to the user. Two key parameters are the maximum altitude, and the furthest
# away we anticipate the flight being at its closest point to HOME. As those two
# parameters are manipulated in the settings, a histogram is displayed with one or
# potentially two series, showing the present and potentially prior-set distribution
# of flights, by hour throughout the day, over the last seven days, normalized to
# flights per day. This allows those parameters to be fine-tuned in a useful way.
# This file is the location, on the webserver, of that image, which needs to be in
# alignment with the html page that displays it.
HOURLY_IMAGE_FILE = 'hours.png'

# Communication with the asynchronously-running Arduino interface is done thru files;
# this file includes the superset of key-value pairs that could potentially be sent
# directly, or after some manipulation, to the Arduino.
ARDUINO_FILE = 'arduino.txt'
# One potential command from the Arduino is to "display the last flight"; this request
# is communicated to the Arduino by the presence of this file. After that request is
# processed, this file is deleted. The contents of the file are not used in any way.
LAST_FLIGHT_FILE = 'last_flight.txt'

# This is all messages that have been sent to the board since the last time the
# file was manually cleared. Newest messages are at the bottom. It is visible at the
# webserver.
ALL_MESSAGE_FILE = 'all_messages.txt'  #enumeration of all messages sent to board
# This shows the most recent n messages sent to the board. Newest messages are at the
# top for easier viewing of "what did I miss".
ROLLING_MESSAGE_FILE = 'rolling_messages.txt'

STDERR_FILE = 'stderr.txt'
BACKUP_FILE = 'backup.txt'
SERVICE_VERIFICATION_FILE = 'service-verification.txt'

FLAG_MSG_FLIGHT = 1  # basic flight details
FLAG_MSG_INTERESTING = 2  # random tidbit about a flight
FLAG_MSG_HISTOGRAM = 3 # histogram message

FLAG_INSIGHT_LAST_SEEN = 0
FLAG_INSIGHT_DIFF_AIRCRAFT = 1
FLAG_INSIGHT_NTH_FLIGHT = 2
FLAG_INSIGHT_GROUNDSPEED = 3
FLAG_INSIGHT_ALTITUDE = 4
FLAG_INSIGHT_VERTRATE = 5
FLAG_INSIGHT_FIRST_DEST = 6
FLAG_INSIGHT_FIRST_ORIGIN = 7
FLAG_INSIGHT_FIRST_AIRLINE = 8
FLAG_INSIGHT_FIRST_AIRCRAFT = 9
FLAG_INSIGHT_LONGEST_DELAY = 10
FLAG_INSIGHT_FLIGHT_DELAY_FREQUENCY = 11
FLAG_INSIGHT_FLIGHT_DELAY_TIME = 12
FLAG_INSIGHT_AIRLINE_DELAY_FREQUENCY = 13
FLAG_INSIGHT_AIRLINE_DELAY_TIME = 14
FLAG_INSIGHT_DESTINATION_DELAY_FREQUENCY = 15
FLAG_INSIGHT_DESTINATION_DELAY_TIME = 16
FLAG_INSIGHT_HOUR_DELAY_FREQUENCY = 17
FLAG_INSIGHT_HOUR_DELAY_TIME = 18
FLAG_INSIGHT_DATE_DELAY_FREQUENCY = 19
FLAG_INSIGHT_DATE_DELAY_TIME = 20
INSIGHT_TYPES = 21

# error lights - value is the RPi pin number for the error light
GPIO_ERROR_VESTABOARD_CONNECTION = (22, None, None)  # error messages printed by main
GPIO_ERROR_FLIGHT_AWARE_CONNECTION = (23, None, None)  # error messages printed by main
GPIO_ERROR_ARDUINO_SERVO_CONNECTION = (
    24, 'lost connection with servos', 'restored connection with servos')
GPIO_ERROR_ARDUINO_REMOTE_CONNECTION = (
    25, 'lost connection with remote', 'restored connection with remote')
GPIO_ERROR_BATTERY_CHARGE = (26, 'low battery', 'battery recharged')
GPIO_FAN = (27, None, None)  # error messages printed by main


TEMP_FAN_TURN_ON_CELSIUS = 65
TEMP_FAN_TURN_OFF_CELSIUS = 55

#if running on raspberry, then need to prepend path to file names
if RASPBERRY_PI:
  PICKLE_FLIGHTS = MESSAGEBOARD_PATH + PICKLE_FLIGHTS
  LOGFILE = MESSAGEBOARD_PATH + LOGFILE
  PICKLE_DUMP_JSON_FILE = MESSAGEBOARD_PATH + PICKLE_DUMP_JSON_FILE
  PICKLE_FA_JSON_FILE = MESSAGEBOARD_PATH + PICKLE_FA_JSON_FILE
  LAST_FLIGHT_FILE = MESSAGEBOARD_PATH + LAST_FLIGHT_FILE
  ARDUINO_FILE = MESSAGEBOARD_PATH + ARDUINO_FILE

  HISTOGRAM_CONFIG_FILE = WEBSERVER_PATH + HISTOGRAM_CONFIG_FILE
  CONFIG_FILE = WEBSERVER_PATH + CONFIG_FILE
  ROLLING_MESSAGE_FILE = WEBSERVER_PATH + ROLLING_MESSAGE_FILE
  ALL_MESSAGE_FILE = WEBSERVER_PATH + ALL_MESSAGE_FILE
  ROLLING_LOGFILE = WEBSERVER_PATH + ROLLING_LOGFILE
  STDERR_FILE = WEBSERVER_PATH + STDERR_FILE
  BACKUP_FILE = WEBSERVER_PATH + BACKUP_FILE
  SERVICE_VERIFICATION_FILE = WEBSERVER_PATH + SERVICE_VERIFICATION_FILE

  HISTOGRAM_IMAGE_PREFIX = WEBSERVER_PATH + WEBSERVER_IMAGE_FOLDER + HISTOGRAM_IMAGE_PREFIX
  HISTOGRAM_EMPTY_IMAGE_FILE = (
      WEBSERVER_PATH + WEBSERVER_IMAGE_FOLDER + HISTOGRAM_EMPTY_IMAGE_FILE)
  HOURLY_IMAGE_FILE = WEBSERVER_PATH + WEBSERVER_IMAGE_FOLDER + HOURLY_IMAGE_FILE

TIMEZONE = 'US/Pacific' # timezone of display
TZ = pytz.timezone(TIMEZONE)

KNOWN_AIRPORTS = ('SJC', 'SFO', 'OAK')  # iata codes that we don't need to expand





                            <----SKIPPED LINES---->




aircraft_length['Canadair Challenger 350 (twin-jet)'] = 20.9
aircraft_length['Bombardier Challenger 300 (twin-jet)'] = 20.92
aircraft_length['Embraer 170/175 (twin-jet)'] = (29.90 + 31.68) / 2
aircraft_length['EMBRAER 175 (long wing) (twin-jet)'] = 31.68
aircraft_length['Embraer ERJ-135 (twin-jet)'] = 26.33
aircraft_length['Cessna Caravan (single-turboprop)'] = 11.46
aircraft_length['Cessna Citation II (twin-jet)'] = 14.54
aircraft_length['Cessna Citation V (twin-jet)'] = 14.91
aircraft_length['Cessna Citation X (twin-jet)'] = 22.04
aircraft_length['Cessna Skyhawk (piston-single)'] = 8.28
aircraft_length['Cessna Skylane (piston-single)'] = 8.84
aircraft_length['Cessna Citation Sovereign (twin-jet)'] = 19.35
aircraft_length['Cessna T206 Turbo Stationair (piston-single)'] = 8.61
aircraft_length['Beechcraft Bonanza (33) (piston-single)'] = 7.65
aircraft_length['Beechcraft Super King Air 200 (twin-turboprop)'] = 13.31
aircraft_length['Beechcraft Super King Air 350 (twin-turboprop)'] = 14.22
aircraft_length['Beechcraft King Air 90 (twin-turboprop)'] = 10.82
aircraft_length['Pilatus PC-12 (single-turboprop)'] = 14.4


def CheckIfProcessRunning():
  """Returns proc id if process with identically-named python file running; else None."""
  this_process_id = os.getpid()
  this_process_name = os.path.basename(sys.argv[0])
  for proc in psutil.process_iter():
    try:
      # Check if process name contains this_process_name.
      commands = proc.as_dict(attrs=['cmdline', 'pid'])
      if commands['cmdline']:
        command_running = any(
            [this_process_name in s for s in commands['cmdline']])
        if command_running and commands['pid'] != this_process_id:
          return commands['pid']
    except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
      pass
  return None


def Log(message, file=None):
  """Write a message to a logfile along with a timestamp.

  Args:
    message: string message to write
    file: string representing file name and, if needed, path to the file to write to
  """
  # can't define as a default parameter because LOGFILE name is potentially
  # modified based on SIMULATION flag
  if not file:
    file = LOGFILE

  if file == LOGFILE:
    lock = filelock.FileLock(LOGFILE_LOCK)
    lock.acquire()

  try:
    with open(file, 'a') as f:
      if not SIMULATION:  # by excluding the timestamp, file diffs become easy between runs

        f.write('='*80+'\n')
        f.write(str(datetime.datetime.now(TZ))+'\n')
        f.write('\n')
      f.write(str(message)+'\n')
  except IOError:
    Log('Unable to append to ' + file)

  if file == LOGFILE:
    lock.release()


  existing_log_lines = ReadFile(LOGFILE).splitlines()
  with open(ROLLING_LOGFILE, 'w') as f:
    f.write('\n'.join(existing_log_lines[-1000:]))


def MaintainRollingWebLog(message, max_count, filename=None):
  """Maintains a rolling text file of at most max_count printed messages.

  Newest data at top and oldest data at the end, of at most max_count messages,
  where the delimiter between each message is identified by a special fixed string.

  Args:
    message: text message to prepend to the file.
    max_count: maximum number of messages to keep in the file; the max_count+1st message
      is deleted.
    filename: the file to update.
  """
  # can't define as a default parameter because ROLLING_MESSAGE_FILE name is potentially
  # modified based on SIMULATION flag




                            <----SKIPPED LINES---->






def UnpickleObjectFromFile(full_path, date_segmentation, max_days=None):
  """Load a repository of pickled flight data into memory.

  Args:
    full_path: name (potentially including path) of the pickled file
    date_segmentation: If true, searches for all files that have a prefix of yyyy-mm-dd
      as a prefix to the file name specified in the full path, and loads them in
      sequence for unpickling; if false, uses the full_path as is and loads just that
      single file.
    max_days: Integer that, if specified, indicates maximum number of days of files to
      load back in; otherwise, loads all.

  Returns:
    Return a list of all the flights, in the same sequence as written to the file.
  """
  if date_segmentation:
    directory, file = os.path.split(full_path)
    files = os.listdir(directory)
    if max_days:
      earliest_date = EpochDisplayTime(time.time() - max_days*SECONDS_IN_DAY, '%Y-%m-%d')
      files = [f for f in files if f[:10] >= earliest_date]
    files = sorted([os.path.join(directory, f) for f in files if file in f])
  else:
    if os.path.exists(full_path):
      files = [full_path]
    else:
      return []

  data = []
  for file in files:
    try:
      with open(file, 'rb') as f:
        while True:
          data.append(pickle.load(f))
    except (EOFError, pickle.UnpicklingError):
      pass

  return data





                            <----SKIPPED LINES---->





  Args:
    flights: list of flight dictionaries.
    configuration: dictionary of settings.
    display_all_hours: boolean indicating whether we should ignore the time constraints
      (i.e.: whether the screen is enabled, and its turn-on or turn-off times) in
      identifying the most recent flight. That is, if False, then this will only return
      flights that would have been displayed in the ordinarily usage, vs. if True,
      a flight irrespective of the time it would be displayed.

  Returns:
    A flight dictionary if one can be found; None otherwise.
  """
  for n in range(len(flights)-1, -1, -1):  # traverse the flights in reverse
    if FlightMeetsDisplayCriteria(
        flights[n], configuration, display_all_hours=display_all_hours):
      return n
  return None


def FlightMessageTestHarness(flights=None, display=True):
  """Simulates what flight messages might be displayed by replaying past flights."""
  if flights is None:
    flights = UnpickleObjectFromFile(PICKLE_FLIGHTS, True)

  messages = []
  for flight in flights:

    flight_message = Screenify(CreateMessageAboutFlight(flight), False)
    if display:
      print(flight_message)
    messages.append(flight_message)

  return messages


def CreateMessageAboutFlight(flight):
  """Creates a message to describe interesting attributes about a single flight.

  Generates a multi-line description of a flight. A typical message might look like:
  UAL300 - UNITED        <- Flight number and airline
  BOEING 777-200 (TWIN)  <- Aircraft type
  SFO-HNL HONOLULU       <- Origin & destination
  DEP 02:08 ER REM 5:14  <- Time details: departure time; early / late / ontime; remaining
  185MPH 301DEG D:117FT  <- Trajectory details: speed; bearing; forecast min dist to HOME
  1975FT (+2368FPM)      <- Altitude details: current altitude & rate or ascent / descent

  However, not all of these details are always present, so some may be listed as unknown,
  or entire lines may be left out.

  Args:
    flight: dictionary of flight attributes.

  Returns:
    Printable string (with embedded new line characters)
  """




                            <----SKIPPED LINES---->




    if departure_time_details.get('delay_text'):
      line_elements.append(departure_time_details['delay_text'])

    remaining_seconds = DisplaySecondsRemaining(flight)
    if remaining_seconds is not None:
      line_elements.append('Rem ' + SecondsToHhMm(remaining_seconds, colon=True))

  if line_elements:
    lines.append(EvenlySpace(line_elements))

  # LINE5: 123mph 297deg D:1383ft
  #        ======================
  speed = flight.get('speed')
  heading = flight.get('track')
  min_feet = flight.get('min_feet')

  line_elements = []
  if speed is not None:
    line_elements.append(str(round(speed)) + SPEED_UNITS)
  if heading is not None:
    line_elements.append(str(heading) + u'\u00b0')  # degrees deg unicode
  if min_feet is not None:
    line_elements.append('D:' + str(round(min_feet)) + DISTANCE_UNITS)
  if line_elements:
    lines.append(EvenlySpace(line_elements))

  # LINE6: Alt: 12345ft +1234fpm
  #        ======================
  altitude = flight.get('altitude')
  vert_rate = flight.get('vert_rate')

  line_elements = []
  if altitude:
    line_elements.append('Alt:%d%s' % (altitude, DISTANCE_UNITS))
  if vert_rate:
    line_elements.append('%+d%s' % (vert_rate, CLIMB_RATE_UNITS))
  if line_elements:
    lines.append(EvenlySpace(line_elements))

  return lines





                            <----SKIPPED LINES---->




  """
  divider = '+' + '-'*SPLITFLAP_CHARS_PER_LINE + '+'
  border_character = '|'
  append_character = '\n'

  if splitflap:
    border_character = ''
    append_character = ''

  for unused_n in range(SPLITFLAP_LINE_COUNT-len(lines)):
    lines.append('')
  lines = [
      border_character + line.ljust(SPLITFLAP_CHARS_PER_LINE).upper() + border_character
      for line in lines]

  if not splitflap:
    lines.insert(0, divider)
    lines.append(divider)

  return append_character.join(lines)


def FlightInsightsTestHarness(
    flights=None,
    display=True,
    flight_insights_enabled_string='all'):
  """Simulates what insightful messages might be displayed by replaying past flights."""
  if flights is None:
    flights = UnpickleObjectFromFile(PICKLE_FLIGHTS, True)

  distribution = {}
  messages = []
  for (n, flight) in enumerate(flights):

    flight_message = CreateMessageAboutFlight(flight)
    if display:
      print('='*25)
      print(Screenify(flight_message, False))
    messages.append(flight_message)

    insights = CreateFlightInsights(
        flights[:n+1], flight_insights_enabled_string, distribution)

    FlightInsightNextFlight(flights[:n+1])

    if display:
      for insight in insights:
        print(Screenify(insight, False))
    messages.extend(insights)

  return messages


def FlightInsightLastSeen(flights, days_ago=2):
  """Generates string indicating when flight was last seen.

  Generates text of the following form.
  - KAL214 was last seen 2d0h ago

  Args:
    flights: the list of the raw data from which the insights will be generated,
      where the flights are listed in order of observation - i.e.: flights[0] was the
      earliest seen, and flights[-1] is the most recent flight for which we are
      attempting to generate an insight.
    days_ago: the minimum time difference for which a message should be generated -
      i.e.: many flights are daily, and so we are not necessarily interested to see
      about every daily flight that it was seen yesterday. However, more infrequent
      flights might be of interest.

  Returns:
    Printable string message; if no message or insights to generate, then an empty string.




                            <----SKIPPED LINES---->




  - ASA1337 was the 4th flight to PHX in the last 53 minutes, served by Alaska Airlines,
    American Airlines, Southwest and United
  - SWA3102 was the 2nd flight to SAN in the last 25 minutes, both with Southwest

  Args:
    flights: the list of the raw data from which the insights will be generated,
      where the flights are listed in order of observation - i.e.: flights[0] was the
      earliest seen, and flights[-1] is the most recent flight for which we are
      attempting to generate an insight.
    hours: the time horizon over which to look for flights with the same destination.
    min_multiple_flights: the minimum number of flights to that same destination to
      warrant generating an insight.

  Returns:
    Printable string message; if no message or insights to generate, then an empty string.
  """
  message = ''
  this_flight = flights[-1]
  this_flight_number = this_flight.get('flight_number', 'This')
  this_destination = this_flight.get('destination_iata', '')
  this_airline = this_flight.get('airline_short_name', KEY_NOT_PRESENT_STRING)
  if not this_airline:
    this_airline = KEY_NOT_PRESENT_STRING # in case airline was stored as, say, ''
  this_timestamp = this_flight['now']
  if this_destination and this_destination not in ['SFO', 'LAX']:
    similar_flights = [f for f in flights[:-1] if
                       this_timestamp - f['now'] < SECONDS_IN_HOUR*hours and
                       this_destination == f.get('destination_iata', '')]
    similar_flights_count = len(similar_flights) + 1  # +1 for this_flight
    similar_flights_airlines = list(
        {f.get('airline_short_name', KEY_NOT_PRESENT_STRING) for f in similar_flights})

    same_airline = [this_airline] == similar_flights_airlines

    if similar_flights_count >= min_multiple_flights:
      n_minutes = (
          (this_flight['now'] - similar_flights[0]['now'])
          / SECONDS_IN_MINUTE)
      message = ('%s was the %s flight to %s in the last %d minutes' % (
          this_flight_number, Ordinal(similar_flights_count),
          this_destination, n_minutes))
      if same_airline and similar_flights_count == 2:
        message += ', both with %s' % this_airline
      elif same_airline:
        message += ', all with %s' % this_airline
      else:
        similar_flights_airlines.append(this_airline)
        similar_flights_airlines.sort()
        message += ', served by %s and %s' % (
            ', '.join(similar_flights_airlines[:-1]),
            similar_flights_airlines[-1])




                            <----SKIPPED LINES---->




        absolute_string = absolute_list[1]
        other_value = value_max
      elif (
          isinstance(this_value, numbers.Number) and
          isinstance(value_min, numbers.Number) and
          this_value < value_min and
          insight_min):
        absolute_string = absolute_list[0]
        other_value = value_min
      else:
        superlative = False

      if superlative:
        message = '%s has the %s %s (%d%s vs. %d%s) in last %d hours' % (
            this_flight_number, absolute_string, label,
            this_value, units, other_value, units, hours)

  return message


def FlightInsightNextFlight(flights):
  """Generates string about estimated wait until next flight.

  Generates text of the following form for the "focus" flight in the data.
  - Last flight at 2:53a; avg wait is 1h58m & median is 42m, but could be as long as
    8h43m, based on last 20 days

  Args:
    flights: the list of the raw data from which the insights will be generated,
      where the flights are listed in order of observation - i.e.: flights[0] was the
      earliest seen, and flights[-1] is the most recent flight for which we are
      attempting to generate an insight.


  Returns:
    Printable string message; if no message because not enough history, then an
    empty string.
  """
  msg = ''



  # m = min of day of this flight
  # find minute of day of prior flights st
  # -- that flight not seen in last 12 hrs
  # -- that min of day >= this
  this_flight = flights[-1]
  this_hour = int(DisplayTime(this_flight, '%-H'))
  this_minute = int(DisplayTime(this_flight, '%-M'))
  this_date = DisplayTime(this_flight, '%x')

  # Flights that we've already seen in the last few hours we do not expect to see
  # again for another few hours, so let's exclude them from the calculation
  exclude_flights_hours = 12
  flight_numbers_seen_in_last_n_hours = [
      f['flight_number'] for f in flights
      if f['now'] > this_flight['now'] - exclude_flights_hours*SECONDS_IN_HOUR
      and 'flight_number' in f]
  still_to_come_flights = [
      f for f in flights[:-1]
      if f.get('flight_number') not in flight_numbers_seen_in_last_n_hours
      and this_date != DisplayTime(f, '%x')]

  # exclude flights that would be filtered out
  configuration = ReadAndParseSettings(CONFIG_FILE)
  still_to_come_flights = [
      f for f in still_to_come_flights if FlightMeetsDisplayCriteria(f, configuration)]







  minimum_minutes_next_flight = {}  # min minutes to next flight by day
  for flight in still_to_come_flights:
    date = DisplayTime(flight, '%x')
    hour = int(DisplayTime(flight, '%-H'))
    minutes = int(DisplayTime(flight, '%-M'))
    minutes_after = (hour - this_hour) * MINUTES_IN_HOUR +(minutes - this_minute)
    if minutes_after < 0:
      minutes_after += MINUTES_IN_DAY
    minimum_minutes_next_flight[date] = min(
        minimum_minutes_next_flight.get(date, minutes_after), minutes_after)

  minutes = list(minimum_minutes_next_flight.values())
  if len(minutes) > 1:
    average_seconds = (sum(minutes) / len(minutes)) * SECONDS_IN_MINUTE
    max_seconds = max(minutes) * SECONDS_IN_MINUTE

    median_seconds = statistics.median(minutes) * SECONDS_IN_MINUTE
    minimum_percent_diff = 0.5
    median_different = (
        median_seconds > average_seconds * (1 + minimum_percent_diff) or
        average_seconds > median_seconds * (1+ minimum_percent_diff))
    median_text = ''
    if median_different:
      median_text = ' & median is %s' % SecondsToHhMm(median_seconds)

    msg = ('Last flight at %s; avg wait is %s%s, but could '
           'be as long as %s, based on last %d days' % (
               DisplayTime(this_flight, '%-I:%M%p'), SecondsToHhMm(average_seconds),
               median_text, SecondsToHhMm(max_seconds), len(minutes)))

  return msg
























def PercentileScore(scores, value):
  """Returns the percentile that a particular value is in a list of numbers.

  Roughly inverts numpy.percentile. That is, numpy.percentile(scores_list, percentile)
  to get the value of the list that is at that percentile;
  PercentileScore(scores_list, value) will yield back approximately that percentile.

  If the value matches identical elements in the list, this function takes the average
  position of those identical values to compute a percentile. Thus, for some lists
  (i.e.: where there are lots of flights that have a 0 second delay, or a 100% delay
  frequency), you may not get a percentile of 0 or 100 even with values equal to the
  min or max element in the list.

  Args:
    scores: the list of numbers, including value.
    value: the value for which we want to determine the percentile.

  Returns:
    Returns an integer percentile in the range [0, 100] inclusive.
  """
  count_values_below_score = len([1 for s in scores if s < value])
  # -1 is because value is already in scores
  count_values_at_score = len([1 for s in scores if s == value]) - 1
  percentile = (count_values_below_score + count_values_at_score / 2) / len(scores)
  return round(percentile*100)


def FlightInsightGroupPercentile(
    flights,
    group_function,
    value_function,
    value_string_function,
    group_label,
    value_label,
    filter_function=lambda this, other: True,
    min_days=1,
    lookback_days=30,
    min_this_group_size=0,
    min_comparison_group_size=0,
    min_group_qty=0,
    percentile_low=float('-inf'),
    percentile_high=float('inf')):
  """Generates a string about extreme values of groups of flights.

  Generates text of the following form for the "focus" flight in the data.
  - flight SIA31 (n=7) has a delay frequency in the 95th %tile, with 100% of flights
    delayed an average of 6m over the last 4d1h
  - flight UAL300 (n=5) has a delay time in the 1st %tile, with an average delay of 0m
  over the last 4d5h

  Args:
    flights: the list of the raw data from which the insights will be generated,
      where the flights are listed in order of observation - i.e.: flights[0] was the
      earliest seen, and flights[-1] is the most recent flight for which we are
      attempting to generate an insight.
    group_function: function that, when called with a flight, returns the grouping key.
      That is, for example, group_function(flight) = 'B739'




                            <----SKIPPED LINES---->




      if this_value is None:
        print('A this_value %s' % str(this_value))
      elif len(grouped_values) < min_group_qty:
        print('A passed')
        print('B len(grouped_values) %d >= min_group_qty %d' % (
            len(grouped_values), min_group_qty))
      print('grouped_values: %s' % grouped_values)

  return message


def FlightInsightSuperlativeGroup(
    flights,
    group_function,
    value_function,
    value_string_function,
    group_label,
    value_label,
    absolute_list,
    min_days=1,
    lookback_days=30,
    min_this_group_size=0,
    min_comparison_group_size=0,
    insight_min=True,
    insight_max=True):
  """Generates a string about extreme values of groups of flights.

  Generates text of the following form for the "focus" flight in the data.
  - aircraft B739 (n=7) is tied with B738 and A303 for the most flights at 7 flights
    over the last 3d7h amongst aircraft with a least 5 flights
  - aircraft B739 (n=7) is tied with 17 others for the most flights at 7 flights over
    the last 3d7h amongst aircraft with a least 5 flights
  - flight UAL1075 (n=12) has the most flights with 12 flights; the next most flights
    is 11 flights over the last 7d5h

  Args:
    flights: the list of the raw data from which the insights will be generated,
      where the flights are listed in order of observation - i.e.: flights[0] was the
      earliest seen, and flights[-1] is the most recent flight for which we are
      attempting to generate an insight.
    group_function: function that, when called with a flight, returns the grouping key.




                            <----SKIPPED LINES---->




                   'than next slowest) in last %d hours' % (
                       this_flight_number, this_ascent, CLIMB_RATE_UNITS,
                       ascent_min - this_ascent, CLIMB_RATE_UNITS, hours))
      elif this_descent and other_descents and this_descent < descent_min:
        message = ('%s has the fastest descent rate (%d%s, %d%s faster '
                   'than next fastest) in last %d hours' % (
                       this_flight_number, this_descent, CLIMB_RATE_UNITS,
                       this_descent - descent_min, CLIMB_RATE_UNITS, hours))
      elif this_descent and other_descents and this_descent > descent_max:
        message = ('%s has the slowest descent rate (%d%s, %d%s slower '
                   'than next slowest) in last %d hours' % (
                       this_flight_number, this_descent, CLIMB_RATE_UNITS,
                       descent_max - this_descent, CLIMB_RATE_UNITS, hours))

  return message


def FlightInsightDelays(
    flights,
    min_days=1,
    lookback_days=30,
    min_late_percentage=0.75,
    min_this_delay_minutes=0,
    min_average_delay_minutes=0):
  """Generates string about the delays this flight has seen in the past.

  Only if this flight has a caclculable delay itself, this will generate text of the
  following form for the "focus" flight in the data.
  - This 8m delay is the longest UAL1175 has seen in the last 9 days (avg delay is 4m);
    overall stats: 1 early; 9 late; 10 total
  - With todays delay of 7m, UAL1175 is delayed 88% of the time in the last 8 days for
    avg delay of 4m; overall stats: 1 early; 8 late; 9 total

  Args:
    flights: the list of the raw data from which the insights will be generated,
      where the flights are listed in order of observation - i.e.: flights[0] was the
      earliest seen, and flights[-1] is the most recent flight for which we are
      attempting to generate an insight.
    min_days: the minimum amount of history required to start generating insights
      about delays.
    lookback_days: the maximum amount of history which will be considered in generating




                            <----SKIPPED LINES---->




      additional_descriptor_fcn=lambda f: f['origin_friendly']))
  AppendMessageType(FLAG_INSIGHT_FIRST_AIRLINE, FlightInsightFirstInstance(
      flights, 'airline_short_name', 'airline', days=7))
  AppendMessageType(FLAG_INSIGHT_FIRST_AIRCRAFT, FlightInsightFirstInstance(
      flights, 'aircraft_type_code', 'aircraft', days=7,
      additional_descriptor_fcn=lambda f: f['aircraft_type_friendly']))

  # This is the longest / shortest delay this flight has seen in the last 30 days at
  # 2h5m; including today, this flight has been delayed x of the last y times.
  AppendMessageType(FLAG_INSIGHT_LONGEST_DELAY, FlightInsightDelays(
      flights, min_late_percentage=0.75,
      min_this_delay_minutes=0,
      min_average_delay_minutes=0))

  def DelayTimeAndFrequencyMessage(
      types_tuple,
      group_function,
      group_label,
      filter_function=lambda this, other: True,
      min_days=1,
      lookback_days=30,
      min_this_group_size=0,
      min_comparison_group_size=0,
      min_group_qty=0,
      percentile_low=float('-inf'),
      percentile_high=float('inf')):
    value_function_tuple = (PercentDelay, AverageDelay)
    value_string_function_tuple = (
        lambda flights, value: '%d%% of flights delayed an average of %s' % (
            round(value*100), SecondsToHhMm(AverageDelay(flights))),
        lambda flights, value: 'average delay of %s' % SecondsToHhMm(value))
    value_label_tuple = ('delay frequency', 'delay time')
    for n in range(2):
      if types_tuple[n]:
        AppendMessageType(types_tuple[n], FlightInsightGroupPercentile(
            flights,
            group_function=group_function,
            value_function=value_function_tuple[n],
            value_string_function=value_string_function_tuple[n],
            group_label=group_label,
            value_label=value_label_tuple[n],




                            <----SKIPPED LINES---->




    frequencies_of_insights = [
        insight_message_distribution.get(t, 0) for t in types_of_messages]
    min_frequency = min(frequencies_of_insights)
    for t in sorted(types_of_messages):
      if insight_message_distribution.get(t, 0) == min_frequency:
        break

    insight_message_distribution[t] = insight_message_distribution.get(t, 0) + 1
    for message_tuple in insight_messages:
      if message_tuple[0] == t:
        naked_messages.append(message_tuple[1])
        this_flight_insights.append(t)
        break

  # Save the distribution displayed for this flight so we needn't regen it in future
  flights[-1]['insight_types'] = this_flight_insights

  return naked_messages


def MessageboardHistogramsTestHarness(
    flights=None,
    hist_type='day_of_month',
    hist_history='30d',
    max_screens='all',
    summary=False):
  """Test harness to generate messageboard histograms."""
  if flights is None:
    flights = UnpickleObjectFromFile(PICKLE_FLIGHTS, True)

  messages = MessageboardHistograms(
      flights, hist_type, hist_history, max_screens, summary)

  for message in messages:
    print(message)

  return messages


def FlightCriteriaHistogramPng(
    flights,
    max_distance_feet,
    max_altitude_feet,
    max_days,
    filename=HOURLY_IMAGE_FILE,
    last_max_distance_feet=None,
    last_max_altitude_feet=None):
  """Saves as a png file the histogram of the hourly flight data for the given filters.

  Generates a png histogram of the count of flights by hour that meet the specified
  criteria: max altitude, max distance, and within the last number of days. Also
  optionally generates as a separate data series in same chart a histogram with a
  different max altitude and distance. Saves this histogram to disk.

  Args:
    flights: list of the flights.
    max_distance_feet: max distance for which to include flights in the histogram.
    max_altitude_feet: max altitude for which to include flights in the histogram.
    max_days: maximum number of days as described.
    filename: file into which to save the csv.
    last_max_distance_feet: if provided, along with last_max_altitude_feet, generates
      a second data series with different criteria for distance and altitude, for
      which the histogram data will be plotted alongside the first series.
    last_max_altitude_feet: see above.
  """


  (values, keys, unused_filtered_data) = GenerateHistogramData(
      flights,
      HourString,
      HOURS,
      hours=max_days*HOURS_IN_DAY,
      max_distance_feet=max_distance_feet,
      max_altitude_feet=max_altitude_feet,
      normalize_factor=max_days,
      exhaustive=True)

  comparison = last_max_distance_feet is not None and last_max_altitude_feet is not None
  if comparison:
    (last_values, unused_last_keys, unused_filtered_data) = GenerateHistogramData(
        flights,
        HourString,
        HOURS,
        hours=max_days*HOURS_IN_DAY,
        max_distance_feet=last_max_distance_feet,
        max_altitude_feet=last_max_altitude_feet,
        normalize_factor=max_days,




                            <----SKIPPED LINES---->




  for histogram in histograms_to_generate:
    this_histogram = which_histograms
    if this_histogram == 'all':
      this_histogram = histogram['generate']
    (key, sort, title, hours) = HistogramSettingsKeySortTitle(this_histogram, hours)

    histogram = MessageboardHistogram(
        flights,
        key,
        sort,
        title,
        screen_limit=screen_limit,
        columns=histogram.get('columns', 2),
        suppress_percent_sign=histogram.get('suppress_percent_sign', False),
        column_divider=histogram.get('column_divider', ' '),
        data_summary=data_summary,
        hours=hours,
        absolute=histogram.get('absolute', False))
    messages.extend(histogram)



  return messages


def MessageboardHistogram(
    data,
    keyfunction,
    sort_type,
    title,
    screen_limit=1,
    columns=2,
    column_divider=' ',
    data_summary=False,
    hours=0,
    suppress_percent_sign=False,
    absolute=False):
  """Generates a text representation of one histogram that can be rendered on the display.

  Args:
    data: the iterable of the raw data from which the histogram will be generated;
      each element of the iterable is a dictionary, that contains at least the key




                            <----SKIPPED LINES---->




      'now_date', 'now_time', 'now_datetime', 'now', 'flight_number', 'origin_iata',
      'destination_iata', 'altitude', 'min_feet', 'vert_rate', 'speed', 'distance',
      'delay_seconds', 'airline_call_sign', 'aircraft_type_friendly',
      'azimuth_degrees_00s', 'azimuth_degrees_10s', 'azimuth_degrees_20s',
      'altitude_degrees_00s', 'altitude_degrees_10s', 'altitude_degrees_20s',
      'ground_distance_feet_00s', 'ground_distance_feet_10s', 'ground_distance_feet_20s',
      'crow_distance_feet_00s', 'crow_distance_feet_10s', 'crow_distance_feet_20s']
  for key in all_keys:
    if key not in keys_logical_order:
      keys_logical_order.append(key)

  f = open(filename, 'w')
  f.write(','.join(keys_logical_order)+'\n')
  for flight in flights:
    f.write(','.join(['"'+str(flight.get(k))+'"' for k in keys_logical_order])+'\n')
  f.close()


def SimulationSetup():
  """Updates global variable file names and loads in JSON data for simulation runs."""






  global SIMULATION
  SIMULATION = True

  global DUMP_JSONS
  DUMP_JSONS = UnpickleObjectFromFile(PICKLE_DUMP_JSON_FILE, True)

  global FA_JSONS
  FA_JSONS = UnpickleObjectFromFile(PICKLE_FA_JSON_FILE, True)

  global ALL_MESSAGE_FILE
  ALL_MESSAGE_FILE = PrependFileName(ALL_MESSAGE_FILE, SIMULATION_PREFIX)
  if os.path.exists(ALL_MESSAGE_FILE):
    os.remove(ALL_MESSAGE_FILE)

  global LOGFILE
  LOGFILE = PrependFileName(LOGFILE, SIMULATION_PREFIX)
  if os.path.exists(LOGFILE):
    os.remove(LOGFILE)

  global ROLLING_LOGFILE
  ROLLING_LOGFILE = PrependFileName(ROLLING_LOGFILE, SIMULATION_PREFIX)
  if os.path.exists(ROLLING_LOGFILE):
    os.remove(ROLLING_LOGFILE)

  global ROLLING_MESSAGE_FILE
  ROLLING_MESSAGE_FILE = PrependFileName(ROLLING_MESSAGE_FILE, SIMULATION_PREFIX)
  if os.path.exists(ROLLING_MESSAGE_FILE):
    os.remove(ROLLING_MESSAGE_FILE)

  global PICKLE_FLIGHTS
  PICKLE_FLIGHTS = PrependFileName(PICKLE_FLIGHTS, SIMULATION_PREFIX)
  if os.path.exists(PICKLE_FLIGHTS):
    os.remove(PICKLE_FLIGHTS)

  if os.path.exists(ARDUINO_FILE):
    os.remove(ARDUINO_FILE)


def SimulationEnd(message_queue, flights):
  """Clears message buffer, exercises histograms, and other misc test & status code.

  Args:
    message_queue: List of flight messages that have not yet been printed.
    flights: List of flights dictionaries.
  """
  if flights:
    histogram = {
        'type': 'both',
        'histogram':'all',
        'histogram_history':'30d',
        'histogram_max_screens': '_2',
        'histogram_data_summary': 'on'}
    histogram_messages = TriggerHistograms(flights, histogram)
    histogram_messages = [(FLAG_MSG_HISTOGRAM, m) for m in histogram_messages]
    message_queue.extend(histogram_messages)

    while message_queue:
      ManageMessageQueue(message_queue, 0, {'setting_delay': 0})
    SaveFlightsByAltitudeDistanceCSV(flights)
    SaveFlightsToCSV(flights)

  # repickle to a new .pk with full track info
  file_parts = PICKLE_FLIGHTS.split('.')
  new_pickle_file = '.'.join([file_parts[0] + '_full_path', file_parts[1]])
  if os.path.exists(new_pickle_file):
    os.remove(new_pickle_file)
  for flight in flights:
    PickleObjectToFile(flight, new_pickle_file, False)

  print('Simulation complete after %s dump json messages processed' % len(DUMP_JSONS))








def DumpJsonChanges():
  """Identifies if sequential dump json files changes, for simulation optimization.

  If we are logging the radio output faster than it is updating, then there will be
  sequential log files in the json list that are identical; we only need to process the
  first of these, and can ignore subsequent ones, without any change of output in the
  simulation results. This function identifies whether the current active json changed
  from the prior one.

  Returns:
    Boolean - True if different (and processing needed), False if identical
  """
  if SIMULATION_COUNTER == 0:
    return True
  (this_json, unused_now) = DUMP_JSONS[SIMULATION_COUNTER]
  (last_json, unused_now) = DUMP_JSONS[SIMULATION_COUNTER - 1]
  return this_json != last_json


def EnqueueArduinos(flights, json_desc_dict, to_servo_q, to_remote_q):
  last_flight = None
  if flights:
    last_flight = flights[-1]
  message = (last_flight, json_desc_dict)
  to_servo_q.put(message)
  to_remote_q.put(message)













def ValidateRunning(start_function, p=None, args=()):
  if p is None or not p.is_alive():
    Log('Process (%s) for %s is not alive; restarting with args %s' % (
        str(p), str(start_function), str(args)))
    p = multiprocessing.Process(target=start_function, args=args)
    p.daemon = True
    p.start()




















































































  return p


























def RefreshArduinos(remote, servo, to_remote_q, to_servo_q, to_main_q, flights, json_desc_dict):
  remote = ValidateRunning(arduino_remote.main, args=(to_remote_q, to_main_q))
  servo = ValidateRunning(arduino_servo.main, args=(to_servo_q, to_main_q))
  EnqueueArduinos(flights, json_desc_dict, to_servo_q, to_remote_q)
  return(remote, servo)


def InitArduinos():
  to_remote_q = multiprocessing.Queue()
  to_servo_q = multiprocessing.Queue()
  to_main_q = multiprocessing.Queue()
  remote = ValidateRunning(arduino_remote.main, args=(to_remote_q, to_main_q))
  servo = ValidateRunning(arduino_servo.main, args=(to_servo_q, to_main_q))




  return (remote, servo, to_remote_q, to_servo_q, to_main_q)


def UpdateArduinoFile(flights, json_desc_dict):
  """Saves a file that can be read by arduino.py with necessary flight attributes.








  The independently-running arduino python modules need basic information about the
  flight and radio in order to send useful information to be displayed by the digital
  alphanumeric displays.

  Args:
    flights: list of the flight dictionaries.
    json_desc_dict: dict with details about the radio range and similar radio details.










  Returns:
    String that is also written to disk.
  """
  # Start with radio_range_miles & radio_range_flights
  d = json_desc_dict
























  if flights:
    flight = flights[-1]
    d['flight_number'] = DisplayFlightNumber(flight)
    d['flight_origin'] = DisplayOriginIata(flight)
    d['flight_destination'] = DisplayDestinationIata(flight)
    d['now'] = flight['now']

    requested_time = time.time()
    requested_elapsed_seconds = requested_time - flight['now']
    updated_loc = ClosestKnownLocation(flight, requested_elapsed_seconds)
    actual_elapsed_seconds = requested_elapsed_seconds - updated_loc[1]
    d['speed'] = updated_loc[0]['speed']
    d['lat'] = updated_loc[0]['lat']
    d['lon'] = updated_loc[0]['lon']
    d['track'] = updated_loc[0]['track']
    d['altitude'] = updated_loc[0]['altitude']
    d['vertrate'] = updated_loc[0]['vertrate']
    d['flight_loc_now'] = flight['now'] + actual_elapsed_seconds

  today = datetime.datetime.now(TZ).strftime('%x')



  flight_count_today = len([1 for f in flights if DisplayTime(f, '%x') == today])

  d['flight_count_today'] = flight_count_today

  settings_string = BuildSettings(d)
  if False:
  #if not SIMULATION:
    if os.path.exists(ARDUINO_FILE):
      existing_data = ReadAndParseSettings(ARDUINO_FILE)
      if d != existing_data:
        WriteFile(ARDUINO_FILE, settings_string)































































    else:
      WriteFile(ARDUINO_FILE, settings_string)

  return settings_string


def PublishMessage(
    s,
    subscription_id='12fd73cd-75ef-4cae-bbbf-29b2678692c1',
    key='c5f62d44-e30d-4c43-a43e-d4f65f4eb399',
    secret='b00aeb24-72f3-467c-aad2-82ba5e5266ca',
    timeout=3):
  """Publishes a text string to a Vestaboard.

  The message is pushed to the vestaboard splitflap display by way of its web services;
  see https://docs.vestaboard.com/introduction for more details.

  Args:
    s: String to publish.
    subscription_id: string subscription id from Vestaboard.
    key: string key from Vestaboard.
    secret: string secret from Vestaboard.
    timeout: Max duration in seconds that we should wait to establish a connection.
  """




                            <----SKIPPED LINES---->




        shutil.move(tmp_f, f)
      else:
        print('Aborted: failed to bootstrap %s: file changed while in process' % full_path)
        return


def ResetLogs(config):
  """Clears the non-scrolling logs if reset_logs in config."""
  if 'reset_logs' in config:
    Log('Reset logs')
    for f in (STDERR_FILE, BACKUP_FILE, SERVICE_VERIFICATION_FILE):
      if os.path.exists(f):
        os.remove(f)
        open(f, 'a').close()
    config.pop('reset_logs')
    config = BuildSettings(config)
    WriteFile(CONFIG_FILE, config)
  return config


def main():
  """Traffic cop between incoming radio flight messages, configuration, and messageboard.

  This is the main logic, checking for new flights, augmenting the radio signal with
  additional web-scraped data, and generating messages in a form presentable to the
  messageboard.
  """
  Log('Starting up process %d' % os.getpid())
  already_running_id = CheckIfProcessRunning()
  if already_running_id:
    Log('Sending termination signal to %d' % already_running_id)
    os.kill(already_running_id, signal.SIGTERM)

  if '-s' in sys.argv:
    global SIMULATION_COUNTER
    SimulationSetup()

  # Redirect any errors to a log file instead of the screen, and add a datestamp
  if not SIMULATION:
    sys.stderr = open(STDERR_FILE, 'a')
    Log('', STDERR_FILE)

  if RASPBERRY_PI:
    RPi.GPIO.setmode(RPi.GPIO.BCM)
  SetPinsAsOutput()
  fan_power = False

  configuration = ReadAndParseSettings(CONFIG_FILE)
  startup_time = time.time()
  json_desc_dict = {}

  # If we're displaying just a single insight message, we want it to be something
  # unique, to the extent possible; this dict holds a count of the diff types of messages
  # displayed so far
  insight_message_distribution = {}

  flights = UnpickleObjectFromFile(PICKLE_FLIGHTS, True, max_days=30)
  # Clear the loaded flight of any cached data since code fixes may change
  # the values for some of those cached elements
  for flight in flights:
    for key in list(flight.keys()):
      if key[:len(CACHED_ELEMENT_PREFIX)] == CACHED_ELEMENT_PREFIX:
        flight.pop(key)
  # bootstrap the flight insights distribution
  for flight in flights:
    distribution = flight['insight_types']
    for key in distribution:
      insight_message_distribution[key] = (
          insight_message_distribution.get(key, 0) + 1)

  if False:
    remote, servo, to_remote_q, to_servo_q, to_main_q = InitArduinos()

  # used in simulation to print the hour of simulation once per simulated hour
  prev_simulated_hour = ''

  persistent_nearby_aircraft = {} # key = flight number; value = last seen
  persistent_path = {}
  histogram = {}

  # Next up to print is element 0; this is a list of tuples:
  # Element#1: flag indicating the type of message that this is
  # Element#2: the message itself
  message_queue = []
  next_message_time = time.time()

  # We repeat the loop every x seconds; this ensures that if the processing time is long,
  # we don't wait another x seconds after processing completes
  next_loop_time = time.time() + LOOP_DELAY_SECONDS

  # These files are read only if the version on disk has been modified more recently
  # than the last time it was read
  last_dump_json_timestamp = 0

  WaitUntilKillComplete(already_running_id)

  Log('Finishing initialization of %d; starting radio polling loop' % os.getpid())
  while not SIMULATION or SIMULATION_COUNTER < len(DUMP_JSONS):

    new_configuration = ReadAndParseSettings(CONFIG_FILE)
    CheckForNewFilterCriteria(configuration, new_configuration, message_queue, flights)
    configuration = new_configuration

    ResetLogs(configuration)  # clear the logs if requested

    # if this is a SIMULATION, then process every diff dump. But if it isn't a simulation,
    # then only read & do related processing for the next dump if the last-modified
    # timestamp indicates the file has been updated since it was last read.
    tmp_timestamp = 0
    if not SIMULATION:
      dump_json_exists = os.path.exists(DUMP_JSON_FILE)
      if dump_json_exists:
        tmp_timestamp = os.path.getmtime(DUMP_JSON_FILE)
    if (SIMULATION and DumpJsonChanges()) or (
        not SIMULATION and dump_json_exists and tmp_timestamp > last_dump_json_timestamp):

      last_dump_json_timestamp = tmp_timestamp

      # a command might request info about flight to be (re)displayed, irrespective of
      # whether the screen is on; if so, let's put that message at the front of the message
      # queue, and delete any subsequent messages in queue because presumably the button
      # was pushed either a) when the screen was off (so no messages in queue), or b)
      # because the screen was on, but the last flight details got lost after other screens
      if os.path.exists(LAST_FLIGHT_FILE):
        messageboard_flight_index = IdentifyFlightDisplayed(
            flights, configuration, display_all_hours=True)
        if messageboard_flight_index is not None:
          message_queue = [m for m in message_queue if m[0] != FLAG_MSG_INTERESTING]
          flight_message = CreateMessageAboutFlight(flights[messageboard_flight_index])
          message_queue = [FLAG_MSG_FLIGHT, flight_message]
          next_message_time = time.time()
        os.remove(LAST_FLIGHT_FILE)

      (persistent_nearby_aircraft,
       flight, now,
       json_desc_dict,
       persistent_path) = ScanForNewFlights(
           persistent_nearby_aircraft,
           persistent_path,
           configuration.get('log_jsons', False))

      if flight:
        flights.append(flight)
        if False:
          remote, servo = RefreshArduinos(
              remote, servo, to_remote_q, to_servo_q, to_main_q, flights, json_desc_dict)

        flight_meets_display_criteria = FlightMeetsDisplayCriteria(
            flight, configuration, log=True)
        if flight_meets_display_criteria:
          flight_message = (FLAG_MSG_FLIGHT, CreateMessageAboutFlight(flight))

          # display the next message about this flight now!
          next_message_time = time.time()
          message_queue.insert(0, flight_message)
          # and delete any queued insight messages about other flights that have
          # not yet displayed, since a newer flight has taken precedence
          messages_to_delete = [m for m in message_queue if m[0] == FLAG_MSG_INTERESTING]
          if messages_to_delete:
            Log(
                'Deleting messages from queue due to new-found plane: %s' %
                messages_to_delete)
          message_queue = [m for m in message_queue if m[0] != FLAG_MSG_INTERESTING]

          # Though we also manage the message queue outside this conditional as well,
          # because it can take a half second to generate the flight insights, this allows
          # this message to start displaying on the board immediately, so it's up there
          # when it's most relevant
          next_message_time = ManageMessageQueue(
              message_queue, next_message_time, configuration)

          insight_messages = CreateFlightInsights(
              flights, configuration.get('insights'), insight_message_distribution)
          if configuration.get('next_flight', 'off') == 'on':
            next_flight_text = FlightInsightNextFlight(flights)
            if next_flight_text:
              insight_messages.insert(0, next_flight_text)

          insight_messages = [(FLAG_MSG_INTERESTING, m) for m in insight_messages]

          for insight_message in insight_messages:
            message_queue.insert(0, insight_message)

        else:  # flight didn't meet display criteria
          flight['insight_types'] = []

        PickleObjectToFile(flight, PICKLE_FLIGHTS, not SIMULATION)

      else:
        if False:
          remote, servo = RefreshArduinos(
              remote, servo, to_remote_q, to_servo_q, to_main_q, flights, json_desc_dict)

    if SIMULATION:
      if now:
        simulated_hour = EpochDisplayTime(now, '%Y-%m-%d %H:00%z')
      if simulated_hour != prev_simulated_hour:
        print(simulated_hour)
        prev_simulated_hour = simulated_hour

    histogram = ReadAndParseSettings(HISTOGRAM_CONFIG_FILE)
    if os.path.exists(HISTOGRAM_CONFIG_FILE):
      os.remove(HISTOGRAM_CONFIG_FILE)

    # We also need to make sure there are flights on which to generate a histogram! Why
    # might there not be any flights? Primarily during a simulation, if there's a
    # lingering histogram file at the time of history restart.
    if histogram and not flights:
      Log('Histogram requested (%s) but no flights in memory' % histogram)
    if histogram and flights:
      histogram_messages = TriggerHistograms(flights, histogram)
      histogram_messages = [(FLAG_MSG_HISTOGRAM, m) for m in histogram_messages]
      message_queue.extend(histogram_messages)

    # check time & if appropriate, display next message from queue
    next_message_time = ManageMessageQueue(message_queue, next_message_time, configuration)

    CheckRebootNeeded(startup_time, message_queue, json_desc_dict)

    fan_power = CheckTemperature(fan_power)

    if not SIMULATION:
      time.sleep(max(0, next_loop_time - time.time()))
      next_loop_time = time.time() + LOOP_DELAY_SECONDS
    else:
      SIMULATION_COUNTER += 1
    if SHUTDOWN_SIGNAL:  # do a graceful exit
      sys.exit()

  if SIMULATION:
    SimulationEnd(message_queue, flights)


def CheckForNewFilterCriteria(prev, new, message_queue, flights):
  """If filter criteria changed, generate new image and perhaps new message."""
  if (new.get('setting_max_distance') != prev.get('setting_max_distance') or
      new.get('setting_max_altitude') != prev.get('setting_max_altitude')):
    FlightCriteriaHistogramPng(
        flights,
        new['setting_max_distance'],
        new['setting_max_altitude'],
        7,
        last_max_distance_feet=prev.get('setting_max_distance'),
        last_max_altitude_feet=prev.get('setting_max_altitude'))

  if (new.get('setting_max_distance') != prev.get('setting_max_distance') or
      new.get('setting_max_altitude') != prev.get('setting_max_altitude') or
      new.get('setting_off_time') != prev.get('setting_off_time') or
      new.get('setting_on_time') != prev.get('setting_on_time')):
    next_flight_message = FlightInsightNextFlight(flights)
    if next_flight_message:
      message_queue.append((FLAG_MSG_INTERESTING, next_flight_message))


def CheckRebootNeeded(startup_time, message_queue, json_desc_dict):
  """Reboot if running for over 24 hours and all is quiet."""
  running_hours = (time.time() - startup_time) / SECONDS_IN_HOUR
  if (
      running_hours >= HOURS_IN_DAY and
      not message_queue and
      not json_desc_dict.get('radio_range_flights')):
    Log('About to reboot after running for %.2f hours' % running_hours)
    os.system('sudo reboot')
    sys.exit()


def CheckTemperature(fan_power):
  """Turn on fan if temperature exceeds threshold."""
  if RASPBERRY_PI:
    temperature = gpiozero.CPUTemperature().temperature
    if temperature > TEMP_FAN_TURN_ON_CELSIUS and not fan_power:
      fan_power = True
      RPi.GPIO.output(GPIO_FAN, RPi.GPIO.HIGH)
      Log('Fan turned on at temperature %.1f' % temperature)
    elif temperature < TEMP_FAN_TURN_OFF_CELSIUS and fan_power:
      fan_power = False
      RPi.GPIO.output(GPIO_FAN, RPi.GPIO.LOW)
      Log('Fan turned off at temperature %.1f' % temperature)
  return fan_power


pin_values = {}
def SetPinsAsOutput():
  """Initialize output GPIO pins for output on Raspberry Pi."""
  global pin_values
  for pin in (
      GPIO_ERROR_VESTABOARD_CONNECTION, GPIO_ERROR_FLIGHT_AWARE_CONNECTION,
      GPIO_ERROR_ARDUINO_SERVO_CONNECTION, GPIO_ERROR_ARDUINO_REMOTE_CONNECTION,
      GPIO_ERROR_BATTERY_CHARGE, GPIO_FAN):
    if RASPBERRY_PI:
      RPi.GPIO.setup(pin[0], RPi.GPIO.OUT)
      RPi.GPIO.output(pin[0], RPi.GPIO.LOW)
    pin_values[pin] = False


def UpdateStatusLight(pin, value):
  """Sets the Raspberry Pi GPIO pin high (True) or low (False) based on value."""
  global pin_values
  if RASPBERRY_PI:
    if value:
      RPi.GPIO.output(pin[0], RPi.GPIO.HIGH)
    else:
      RPi.GPIO.output(pin[0], RPi.GPIO.LOW)

  if pin_values[pin] != value:
    if value:
      message = pin[1]










































































































































































































    else:
      message = pin[2]
    if message:
      Log('Error light on pin %d set to %s: %s' % (pin, value, message))
    pin_values[pin] = value



def TerminateProcess(signalNumber, unused_frame):
  """Sets flag so that the main loop will terminate when it completes the iteration."""
  global SHUTDOWN_SIGNAL
  SHUTDOWN_SIGNAL = True
  Log('%d received termination signal %d (%s)' % (
      os.getpid(), signalNumber,
      signal.Signals(signalNumber).name))  # pylint: disable=E1101


def WaitUntilKillComplete(already_running_id, max_seconds=10):
  """Prevents main loop from starting until other instance, if any, completes shutdown.

  A termination command send to any other identically-named process may take a few
  seconds to complete because that other process is allowed to finish the current iteration
  in the main loop. Typically, that iteration in the other process will complete before
  this process finishes the initialization and starts. But in limited scenarios, that might
  not happen, such as if the other process is in the middle of generating a lot of
  histogram images, or if this process does not have much data to load.

  This function ensures that this process does not start the main loop until the other
  process terminates. If it detects that the other process is still running, it waits
  for up to max_seconds. If the other process does not terminate before that time limit,
  then this process terminates.
  """
  still_running_id = CheckIfProcessRunning()
  if still_running_id and still_running_id != already_running_id:
    # uh-oh! another process started up in the interim? exit!
    Log('Kill signal sent to %d from this process %d, but it seems like there is '
        'another process running, %d!' % (
            already_running_id, os.getpid(), still_running_id))
    sys.exit()
  elif still_running_id and still_running_id == already_running_id:
    # wait a few seconds for other process to terminate
    n = 0
    while CheckIfProcessRunning():
      if n == max_seconds:
        Log('Kill signal sent from this process %d to other %d, but other still '
            'running; exiting after %d seconds' % (
                os.getpid(), already_running_id, n+1))
        sys.exit()
      n += 1
      time.sleep(1)
      Log('Kill signal sent from this process %d to other %d, but other still '
          'running; waiting %d seconds' % (os.getpid(), already_running_id, n+1))


if __name__ == "__main__":
  signal.signal(signal.SIGINT, TerminateProcess)  #interrupt, as in ctrl-c
  signal.signal(signal.SIGTERM, TerminateProcess)  #terminate, when another instance found




  if '-i' in sys.argv:
    BootstrapInsightList()
  else:
    main()