<----SKIPPED LINES---->





# This file is where the radio drops its json file
DUMP_JSON_FILE = '/run/readsb/aircraft.json'

# This file is where the query history to the flight aware webpage goes
FLIGHTAWARE_HISTORY_FILE = 'secure/flightaware.txt'

# At the time a flight is first identified as being of interest (in that
# it falls within MIN_METERS meters of HOME), it - and core attributes
# derived from FlightAware, if any - is appended to the end of this pickle
# file. However, since this file is cached in working memory, flights older
# than 30 days are flushed from this periodically.
PICKLE_FLIGHTS = 'pickle/flights.pk'

# This allows us to identify the full history (including what was last sent
# to the splitflap display in a programmatic fashion. While it may be
# interesting in its own right, its real use is to handle the "replay"
# button, so we know to enable it if what is displayed is the last flight.
PICKLE_SCREENS = 'pickle/screens.pk'

# Status data about messageboard - is it running, etc.  Specifically, has tuples
# of data (timestamp, system_id, status), where system_id is either the pin id
# of GPIO, or a 0 to indicate overall system, and status is boolean
PICKLE_DASHBOARD = 'pickle/dashboard.pk'







CACHED_ELEMENT_PREFIX = 'cached_'

# This web-exposed file is used for non-error messages that might highlight
# data or code logic to check into. It is only cleared out manually.
LOGFILE = 'log.txt'
# Identical to the LOGFILE, except it includes just the most recent n lines.
# Newest lines are at the end.
ROLLING_LOGFILE = 'secure/rolling_log.txt' #file for error messages

# default number of lines which may be overridden by settings file
ROLLING_LOG_SIZE = 1000

# Users can trigger .png histograms analogous to the text ones from the web
# interface; this is the folder (within WEBSERVER_PATH) where those files are
# placed
WEBSERVER_IMAGE_RELATIVE_FOLDER = 'images/'
# Multiple histograms can be generated, i.e. for airline, aircraft, day of
# week, etc. The output files are named by the prefix & suffix, i.e.: prefix +
# type + . + suffix, as in histogram_aircraft.png. These names match up to the




                            <----SKIPPED LINES---->




    8, 'Unused', False)

# GPIO pushbutton connections - (GPIO pin switch in; GPIO pin LED out)
GPIO_SOFT_RESET = (20, 21)

GOOGLE_ANALYTICS_TAG = (
    '<!-- Global site tag (gtag.js) - Google Analytics -->\n'
    '<script async src="https://www.googletagmanager.com/gtag/'
    'js?id=UA-99931533-2"></script>\n'
    '<script>\n'
    '  window.dataLayer = window.dataLayer || [];\n'
    '  function gtag(){dataLayer.push(arguments);}\n'
    "  gtag('js', new Date());\n"
    "  gtag('config', 'UA-99931533-2');\n"
    '</script>\n')

#if running on raspberry, then need to prepend path to file names
if RASPBERRY_PI:
  MEMORY_DIRECTORY = MESSAGEBOARD_PATH + MEMORY_DIRECTORY
  PICKLE_FLIGHTS = MESSAGEBOARD_PATH + PICKLE_FLIGHTS
  PICKLE_DASHBOARD = MESSAGEBOARD_PATH + PICKLE_DASHBOARD

  LOGFILE = MESSAGEBOARD_PATH + LOGFILE
  PICKLE_DUMP_JSON_FILE = MESSAGEBOARD_PATH + PICKLE_DUMP_JSON_FILE
  PICKLE_FA_JSON_FILE = MESSAGEBOARD_PATH + PICKLE_FA_JSON_FILE
  PICKLE_SCREENS = MESSAGEBOARD_PATH + PICKLE_SCREENS
  CODE_REPOSITORY = MESSAGEBOARD_PATH

  HISTOGRAM_CONFIG_FILE = WEBSERVER_PATH + HISTOGRAM_CONFIG_FILE
  CONFIG_FILE = WEBSERVER_PATH + CONFIG_FILE
  ROLLING_MESSAGE_FILE = WEBSERVER_PATH + ROLLING_MESSAGE_FILE
  ALL_MESSAGE_FILE = WEBSERVER_PATH + ALL_MESSAGE_FILE
  ROLLING_LOGFILE = WEBSERVER_PATH + ROLLING_LOGFILE
  STDERR_FILE = WEBSERVER_PATH + STDERR_FILE
  BACKUP_FILE = WEBSERVER_PATH + BACKUP_FILE
  SERVICE_VERIFICATION_FILE = WEBSERVER_PATH + SERVICE_VERIFICATION_FILE
  UPTIMES_FILE = WEBSERVER_PATH + UPTIMES_FILE
  CODE_HISTORY_FILE = WEBSERVER_PATH + CODE_HISTORY_FILE
  NEW_AIRCRAFT_FILE = WEBSERVER_PATH + NEW_AIRCRAFT_FILE
  FLIGHTAWARE_HISTORY_FILE = WEBSERVER_PATH + FLIGHTAWARE_HISTORY_FILE

  HISTOGRAM_IMAGE_HTML = WEBSERVER_PATH + HISTOGRAM_IMAGE_HTML




                            <----SKIPPED LINES---->




    persistent_nearby_aircraft: dictionary where keys are flight numbers, and
      the values are the time the flight was last seen.
    persistent_path: dictionary where keys are flight numbers, and the values
      are a sequential list of the location-attributes in the json file; allows
      for tracking the flight path over time.
    log_jsons: boolean indicating whether we should pickle the JSONs.
    flights: list of flight dictionaries; if no json is returned, used to
      find a recent flight with same flight number to augment this flight with
      origin / destination / airline.

  Returns:
    A tuple:
    - updated persistent_nearby_aircraft
    - (possibly empty) dictionary of flight attributes of the new flight upon
      its first observation.
    - the time of the radio observation if present; None if no radio dump
    - a dictionary of attributes about the dump itself (i.e.: # of flights;
      furthest observed flight, etc.)
    - persistent_path, a data structure containing past details of a flight's
      location as described in ParseDumpJson


  """
  flight_details = {}

  now = time.time()
  if SIMULATION:
    (dump_json, json_time) = DUMP_JSONS[SIMULATION_COUNTER]
  else:
    dump_json = ReadFile(DUMP_JSON_FILE, log_exception=True)

  json_desc_dict = {}
  current_nearby_aircraft = {}
  if dump_json:
    (current_nearby_aircraft, now,
     json_desc_dict, persistent_path) = ParseDumpJson(
         dump_json, persistent_path)

    if not SIMULATION and log_jsons:
      PickleObjectToFile((dump_json, now), PICKLE_DUMP_JSON_FILE, True)

    newly_nearby_flight_identifiers = UpdateAircraftList(
        persistent_nearby_aircraft, current_nearby_aircraft, now)

    if newly_nearby_flight_identifiers:




                            <----SKIPPED LINES---->




      if SIMULATION:
        json_times = [j[1] for j in FA_JSONS]
        if json_time in json_times:
          flight_aware_json = FA_JSONS[json_times.index(json_time)][0]
      elif flight_identifier[0]:
        flight_number = flight_identifier[0]
        flight_aware_json, error_message = GetFlightAwareJson(flight_number)
        if flight_aware_json:
          UpdateStatusLight(GPIO_ERROR_FLIGHT_AWARE_CONNECTION, False)
        else:
          failure_message = 'No json from Flightaware for flight %s: %s' % (
              flight_number, error_message[:500])
          Log(failure_message)
          UpdateStatusLight(
              GPIO_ERROR_FLIGHT_AWARE_CONNECTION, True, failure_message)

      flight_details = {}
      if flight_aware_json:
        flight_details = ParseFlightAwareJson(flight_aware_json)
      elif flight_identifier[0]:  # if there's a flight number but no json
        flight_details = FindAttributesFromSimilarFlights(
            flight_identifier[0], flights)


      if not SIMULATION and log_jsons:
        PickleObjectToFile((flight_aware_json, now), PICKLE_FA_JSON_FILE, True)

      # Augment FlightAware details with radio / radio-derived details
      flight_details.update(current_nearby_aircraft[flight_identifier])

      # Augment with the past location data; the [1] is because recall that
      # persistent_path[key] is actually a 2-tuple, the first element being
      # the most recent time seen, and the second element being the actual
      # path. But we do not need to keep around the most recent time seen any
      # more.
      flight_details['persistent_path'] = persistent_path[flight_identifier][1]

  return (
      persistent_nearby_aircraft,
      flight_details,
      now,
      json_desc_dict,
      persistent_path)



def DescribeDumpJson(parsed):
  """Generates dict with descriptive attributes about the dump json file.

  Args:
    parsed: The parsed json file.

  Returns:
    Dictionary with attributes about radio range, number of flights seen, etc.
  """
  json_desc_dict = {}
  json_desc_dict['now'] = parsed['now']

  aircraft = [a for a in parsed['aircraft'] if a['seen'] < PERSISTENCE_SECONDS]
  json_desc_dict['radio_range_flights'] = len(aircraft)

  aircraft_with_pos = [a for a in aircraft if 'lat' in a and 'lon' in a]
  current_distances = [HaversineDistanceMeters(
      HOME, (a['lat'], a['lon'])) for a in aircraft_with_pos]




                            <----SKIPPED LINES---->




  if s is not None:
    s = unidecode.unidecode(s)
  return s


def FindAttributesFromSimilarFlights(this_flight_number, flights):
  """Returns a dictionary with info about a flight based on other flights.

  We may not get a json from the internet about this flight for any number
  of reasons: internet down; website down; too frequent queries; etc.  However,
  there are still some basic attributes we can derive about this flight
  from past observations of the same flight number, or past observations
  about the flight number prefix.  Specifically, we can get the flight's
  airline, origin, and destination.

  Args:
    this_flight_number: String of this flight number.
    flights: List of past flights.

  Returns:

    Dictionary of flight attributes extracted from the FlightAware json.

  """
  derived_attributes = {}

  if not this_flight_number:
    return derived_attributes

  potential_values = {
      'origin_friendly': [],
      'origin_iata': [],
      'destination_friendly': [],
      'destination_iata': [],
      'airline_call_sign': [],
      'airline_short_name': [],
      'airline_full_name': []
      }

  def AppendIfPresent(k, flight):
    val = flight.get(k)
    if val and val != KEY_NOT_PRESENT_STRING:
      potential_values[k].append(val)

  for flight in flights:
    flight_number = flight.get('flight_number')




                            <----SKIPPED LINES---->




      if flight_number[:3] == this_flight_number[:3]:
        AppendIfPresent('airline_call_sign', flight)
        AppendIfPresent('airline_short_name', flight)
        AppendIfPresent('airline_full_name', flight)

  def Mode(lst):
    freq = {}
    for val in lst:
      freq[val] = freq.get(val, 0) + 1
    max_freq = max(freq.values())
    for val in freq:
      if freq[val] == max_freq:
        return val
    return None

  for attribute, lst in potential_values.items():
    if lst:
      derived_attributes[attribute] = Mode(lst)

  if derived_attributes:
    Log(
        'Derived attributes for %s based on past flights: %s' %
        (this_flight_number, str(derived_attributes)))

  return derived_attributes


def ParseFlightAwareJson(flight_json):
  """Strips relevant data about the flight from FlightAware feed.

  The FlightAware json has hundreds of fields about a flight, only a fraction
  of which are relevant to extract. Note that some of the fields are
  inconsistently populated (i.e.: scheduled and actual times for departure and
  take-off).

  Args:
    flight_json: Text representation of the FlightAware json about a single
      flight.

  Returns:
    Dictionary of flight attributes extracted from the FlightAware json.
  """
  flight = {}
  parsed_json = json.loads(flight_json)





                            <----SKIPPED LINES---->




    flight, configuration, display_all_hours=False, log=False):
  """Returns boolean indicating whether the screen accepting new flight data.

  Based on the configuration file, determines whether the flight data should be
  displayed. Specifically, the configuration:
  - may include 'enabled' indicating whether screen should be driven at all
  - should include 'on' & 'off' parameters indicating minute (from midnight) of
    operation
  - should include altitude & elevation parameters indicating max values of
    interest

  Args:
    flight: dictionary of flight attributes.
    configuration: dictionary of configuration attributes.
    display_all_hours: a boolean indicating whether we should ignore whether the
      screen is turned off (either via the enabling, or via the hour settings)
    log: optional boolean indicating whether a flight that fails the criteria
      should be logged with the reason

  Returns:
    Boolean as described.

  """
  flight_altitude = flight.get('altitude', float('inf'))
  config_max_altitude = configuration['setting_max_altitude']


  flight_meets_criteria = True
  if flight_altitude > config_max_altitude:
    flight_meets_criteria = False




    if log:
      Log(
          '%s not displayed because it fails altitude criteria - '
          'flight altitude: %.0f; required altitude: %.0f' % (
              DisplayFlightNumber(flight),
              flight_altitude, config_max_altitude))
  else:
    flight_distance = flight.get('min_feet', float('inf'))
    config_max_distance = configuration['setting_max_distance']
    if flight_distance > config_max_distance:
      flight_meets_criteria = False




      if log:
        Log(
            '%s not displayed because it fails distance criteria - '
            'flight distance: %.0f; required distance: %.0f' % (
                DisplayFlightNumber(flight), flight_distance,
                config_max_distance))

  if not display_all_hours and flight_meets_criteria:
    flight_timestamp = flight['now']
    minute_of_day = MinuteOfDay(flight_timestamp)
    if minute_of_day < configuration['setting_on_time']:
      flight_meets_criteria = False




      if log:
        Log(
            '%s not displayed because it occurs too early - minute_of_day: '
            '%d; setting_on_time: %d' % (
                DisplayFlightNumber(flight), minute_of_day,
                configuration['setting_on_time']))
    elif minute_of_day > configuration['setting_off_time'] + 1:
      flight_meets_criteria = False




      if log:
        Log(
            '%s not displayed because it occurs too late - minute_of_day: '
            '%d; setting_off_time: %d' % (
                DisplayFlightNumber(flight), minute_of_day,
                configuration['setting_off_time']))
    elif configuration.get('setting_screen_enabled', 'off') == 'off':
      flight_meets_criteria = False


      if log:
        Log(
            '%s not displayed because screen disabled' %
            DisplayFlightNumber(flight))

  return flight_meets_criteria


def MessageMeetsDisplayCriteria(configuration, log=False):
  """Returns boolean indicating whether screen is active.

  Based on the configuration file, determines whether the flight data should be
  displayed. Specifically, the configuration:
  - may include 'enabled' indicating whether screen should be driven at all
  - should include 'on' & 'off' parameters indicating minute (from midnight) of
    operation
  Note that this differs from FlightMeetsDisplayCriteria in that this function
  does not use any attribute about the flight (elevation or distance), only
  whether the screen is still active.

  Args:
    configuration: dictionary of configuration attributes.
    log: optional boolean indicating whether a flight that fails the criteria
      should be logged with the reason

  Returns:




                            <----SKIPPED LINES---->






def IdentifyFlightDisplayed(flights, configuration, display_all_hours=False):
  """Finds the most recent flight in flights that meet the display criteria.

  Args:
    flights: list of flight dictionaries.
    configuration: dictionary of settings.
    display_all_hours: boolean indicating whether we should ignore the time
    constraints (i.e.: whether the screen is enabled, and its turn-on or
    turn-off times) in identifying the most recent flight. That is, if False,
    then this will only return flights that would have been displayed in the
    ordinarily usage, vs. if True, a flight irrespective of the time it would
    be displayed.

  Returns:
    A flight dictionary if one can be found; None otherwise.
  """
  for n in range(len(flights)-1, -1, -1):  # traverse the flights in reverse
    if FlightMeetsDisplayCriteria(
        flights[n], configuration, display_all_hours=display_all_hours):
      return n
  return None


def CreateMessageAboutFlight(flight):
  """Creates a message to describe interesting attributes about a single flight.

  Generates a multi-line description of a flight. A typical message might look
  like:
  UAL300 - UNITED        <- Flight number and airline
  BOEING 777-200 (TWIN)  <- Aircraft type
  SFO-HNL HONOLULU       <- Origin & destination
  DEP 02:08 ER REM 5:14  <- Time details: departure; early / late; remaining
  185MPH 301DEG D:117FT  <- Trajectory: speed; bearing; fcst min dist to HOME
  1975FT (+2368FPM)      <- Altitude: current altitude & rate of ascent

  However, not all of these details are always present, so some may be listed
  as unknown, or entire lines may be left out.

  Args:




                            <----SKIPPED LINES---->




  this_flight = flights[-1]
  this_hour = int(DisplayTime(this_flight, '%-H'))
  this_minute = int(DisplayTime(this_flight, '%-M'))
  this_date = DisplayTime(this_flight, '%x')

  # Flights that we've already seen in the last few hours we do not expect to
  # see again for another few hours, so let's exclude them from the calculation
  exclude_flights_hours = 12
  flight_numbers_seen_in_last_n_hours = [
      f['flight_number'] for f in flights
      if f['now'] > this_flight['now'] - exclude_flights_hours*SECONDS_IN_HOUR
      and 'flight_number' in f]
  still_to_come_flights = [
      f for f in flights[:-1]
      if f.get('flight_number') not in flight_numbers_seen_in_last_n_hours
      and this_date != DisplayTime(f, '%x')]

  # exclude flights that would be filtered out by altitude or distance
  still_to_come_flights = [
      f for f in still_to_come_flights
      if FlightMeetsDisplayCriteria(f, configuration)]

  # exclude flights more than 30 days in the past
  now = time.time()
  still_to_come_flights = [
      f for f in still_to_come_flights
      if now - f['now'] < MAX_INSIGHT_HORIZON_DAYS * SECONDS_IN_DAY]

  minimum_minutes_next_flight = {}  # min minutes to next flight by day
  for flight in still_to_come_flights:
    date = DisplayTime(flight, '%x')
    hour = int(DisplayTime(flight, '%-H'))
    minutes = int(DisplayTime(flight, '%-M'))
    minutes_after = (hour - this_hour) * MINUTES_IN_HOUR + (
        minutes - this_minute)
    if minutes_after < 0:
      minutes_after += MINUTES_IN_DAY
    minimum_minutes_next_flight[date] = min(
        minimum_minutes_next_flight.get(date, minutes_after), minutes_after)

  minutes = list(minimum_minutes_next_flight.values())




                            <----SKIPPED LINES---->




  global LOGFILE
  LOGFILE = PrependFileName(LOGFILE, SIMULATION_PREFIX)
  ClearFile(LOGFILE)

  global ROLLING_LOGFILE
  ROLLING_LOGFILE = PrependFileName(ROLLING_LOGFILE, SIMULATION_PREFIX)
  ClearFile(ROLLING_LOGFILE)

  global ROLLING_MESSAGE_FILE
  ROLLING_MESSAGE_FILE = PrependFileName(
      ROLLING_MESSAGE_FILE, SIMULATION_PREFIX)
  ClearFile(ROLLING_MESSAGE_FILE)

  global PICKLE_FLIGHTS
  PICKLE_FLIGHTS = PrependFileName(PICKLE_FLIGHTS, SIMULATION_PREFIX)
  filenames = UnpickleObjectFromFile(
      PICKLE_FLIGHTS, True, max_days=None, filenames=True)
  for file in filenames:
    ClearFile(file)

  global PICKLE_DASHBOARD
  PICKLE_DASHBOARD = PrependFileName(PICKLE_DASHBOARD, SIMULATION_PREFIX)

  filenames = UnpickleObjectFromFile(
      PICKLE_DASHBOARD, True, max_days=None, filenames=True)
  for file in filenames:
    ClearFile(file)









def SimulationEnd(message_queue, flights, screens):
  """Clears message buffer, exercises histograms, and other simulation wrap up.

  Args:
    message_queue: List of flight messages that have not yet been printed.
    flights: List of flights dictionaries.
    screens: List of past screens displayed to splitflap screen.
  """
  if flights:
    histogram = {
        'type': 'both',
        'histogram':'all',
        'histogram_history':'30d',
        'histogram_max_screens': '_2',
        'histogram_data_summary': 'on'}
    message_queue.extend(TriggerHistograms(flights, histogram))

    while message_queue:
      ManageMessageQueue(message_queue, 0, {'setting_delay': 0}, screens)




                            <----SKIPPED LINES---->




    s,
    subscription_id=SUBSCRIPTION_ID,
    key=KEY,
    secret=SECRET,
    timeout=5):
  """Publishes a text string to a Vestaboard.

  The message is pushed to the vestaboard splitflap display by way of its
  web services; see https://docs.vestaboard.com/introduction for more details.

  TODO: rewrite to use the easier-to-follow requests library, more in line
  with PublishMessageLocal.

  Args:
    s: String to publish.
    subscription_id: string subscription id from Vestaboard.
    key: string key from Vestaboard.
    secret: string secret from Vestaboard.
    timeout: Max duration in seconds that we should wait to establish a
      connection.




  """
  error_code = False
  curl = pycurl.Curl()

  # See https://stackoverflow.com/questions/31826814/
  # curl-post-request-into-pycurl-code
  # Set URL value
  curl.setopt(
      pycurl.URL,
      'https://platform.vestaboard.com/subscriptions/%s/message'
      % subscription_id)
  curl.setopt(pycurl.HTTPHEADER, [
      'X-Vestaboard-Api-Key:%s' % key, 'X-Vestaboard-Api-Secret:%s' % secret])
  curl.setopt(pycurl.TIMEOUT_MS, timeout*1000)
  curl.setopt(pycurl.POST, 1)

  curl.setopt(pycurl.WRITEFUNCTION, lambda x: None) # to keep stdout clean

  # preparing body the way pycurl.READDATA wants it
  body_as_dict = {'characters': StringToCharArray(s)}
  body_as_json_string = json.dumps(body_as_dict) # dict to json
  body_as_file_object = io.StringIO(body_as_json_string)

  # prepare and send. See also: pycurl.READFUNCTION to pass function instead
  curl.setopt(pycurl.READDATA, body_as_file_object)
  curl.setopt(pycurl.POSTFIELDSIZE, len(body_as_json_string))
  failure_message = ''
  try:
    curl.perform()

  except pycurl.error as e:
    timing_message = CurlTimingDetailsToString(curl)
    failure_message = (
        'curl.perform() failed with message %s; timing details: %s' %
        (e, timing_message))
    # Using the remote webservice failed, but maybe the local API will
    # be more successful?  If this succeeds, then we should not indicate
    # a failure on the status light / dashboard, but we should still log
    # the remote failure
    Log(failure_message)
    error_code = PublishMessageLocal(s, timeout=timeout, update_dashboard=False)








  else:
    # you may want to check HTTP response code, e.g.
    timing_message = CurlTimingDetailsToString(curl)
    status_code = curl.getinfo(pycurl.RESPONSE_CODE)
    if status_code != 200:
      failure_message = (
          'Server returned HTTP status code %d for message %s; '
          'timing details: %s' % (status_code, s, timing_message))
      Log(failure_message)
      error_code = PublishMessageLocal(
          s, timeout=timeout, update_dashboard=False)









  # We've logged the error code from the external web service, but the
  # Vestaboard local API was able to recover from the error, so we need not
  # log the failure message / error code in the dashboard.
  if not error_code:
    failure_message = ''

  curl.close()
  UpdateStatusLight(
      GPIO_ERROR_VESTABOARD_CONNECTION, error_code, failure_message)




def PublishMessageLocal(
    s,
    local_key=LOCAL_KEY,
    local_address=LOCAL_VESTABOARD_ADDRESS,
    timeout=5,
    update_dashboard=True):
  """Publishes a text string to a Vestaboard via local API.

  The message is pushed to the vestaboard splitflap display by way of its
  local API; see https://docs.vestaboard.com/local for more details.

  Args:
    s: String to publish.
    local_key: string key from Vestaboard for local API access.
    local_address: the address and port to the local Vestaboard service.
    timeout: Max duration in seconds that we should wait to establish a
      connection.
    update_dashboard: Boolean indicating whether this method should update the




                            <----SKIPPED LINES---->





  return personal_message


def ManageMessageQueue(
    message_queue, next_message_time, configuration, screens):
  """Check time & if appropriate, display next message from queue.

  Args:
    message_queue: FIFO list of message tuples of (message type,
      message string).
    next_message_time: epoch at which next message should be displayed
    configuration: dictionary of configuration attributes.
    screens: List of past screens displayed to splitflap screen.

  Returns:
    Next_message_time, potentially updated if a message has been displayed,
    or unchanged if no message was displayed.
  """
  if message_queue and (time.time() >= next_message_time or SIMULATION):


    if SIMULATION:  # drain the queue because the messages come so fast
      messages_to_display = list(message_queue)
      # passed by reference, so clear it out since we drained it to the display
      del message_queue[:]
    else:  # display only one message, being mindful of the display timing
      messages_to_display = [message_queue.pop(0)]

    for message in messages_to_display:
      # we cannot just unpack the tuple because messages of type
      # FLAG_MSG_FLIGHT are 3-tuples (with the third element being the flight
      # dictionary) whereas other message types are 2-tuples
      message_type = message[0]
      message_text = message[1]

      # There may be one or several insight messages that were added to the
      # message queue along with the flight at a time when the screen was
      # enabled, but by the time it comes to display them, the screen is now
      # disabled.  These should not be displayed.  Note that this check only
      # needs to be done for insight messages because other message types
      # are user initiated and so presumably should be displayed irrespective
      # of when the user triggered it to be displayed.
      if message_type == FLAG_MSG_INSIGHT and not MessageMeetsDisplayCriteria(
          configuration):

        Log('Message %s purged' % message_text)

      else:
        if isinstance(message_text, str):
          message_text = textwrap.wrap(
              message_text,
              width=SPLITFLAP_CHARS_PER_LINE)
        display_message = Screenify(message_text, False)
        Log(display_message, file=ALL_MESSAGE_FILE)

        # Saving this to disk allows us to identify
        # persistently whats currently on the screen
        PickleObjectToFile(message, PICKLE_SCREENS, True)
        screens.append(message)

        MaintainRollingWebLog(display_message, 25)
        if not SIMULATION:
          splitflap_message = Screenify(message_text, True)
          PublishMessageWeb(splitflap_message)





















    next_message_time = time.time() + configuration['setting_delay']
  return next_message_time


def DeleteMessageTypes(q, types_to_delete):
  """Delete messages from the queue if type is in the iterable types."""
  if VERBOSE:
    messages_to_delete = [m for m in q if m[0] in types_to_delete]
    if messages_to_delete:
      Log('Deleting messages from queue due to new-found plane: %s'
          % messages_to_delete)
  updated_q = [m for m in q if m[0] not in types_to_delete]
  return updated_q


def BootstrapInsightList(full_path=PICKLE_FLIGHTS):
  """(Re)populate flight pickle files with flight insight distributions.

  The set of insights generated for each flight is created at the time




                            <----SKIPPED LINES---->




  Args:
    value: Boolean indicating whether a failure has occurred (True) or
      system is nominal (False).
    subsystem: A tuple describing the system; though that description may
      have multiple attributes, the 0th element is the numeric identifier
      of that system.  monitoring.py depends on other attributes of that
      tuple being present as well.  Since the overall system does not have
      a tuple defined for it, it gets a default identifier of 0.
    failure_message: an (optional) message describing why the system /
      subsystem is being disabled or failing.
    iteration: integer indicating how many times the main loop has been
      completed.
  """
  versions = (VERSION_MESSAGEBOARD, VERSION_ARDUINO)
  if subsystem:
    subsystem = subsystem[0]
  PickleObjectToFile((
      time.time(), subsystem, value, versions,
      failure_message, INSTANCE_START_TIME,
      iteration, gpiozero.CPUTemperature().temperature),
      PICKLE_DASHBOARD, True)


def RemoveFile(file):
  """Removes a file, returning a boolean indicating if it had existed."""
  if os.path.exists(file):

    try:
      os.remove(file)
    except PermissionError:
      return False
    return True

  return False


def ConfirmNewFlight(flight, flights):
  """Replaces last-seen flight with new flight if identifiers overlap.

  Flights are identified by the radio over time by a tuple of identifiers:
  flight_number and squawk.  Due to unknown communication issues, one or the




                            <----SKIPPED LINES---->




    configuration = new_configuration

    ResetLogs(configuration)  # clear the logs if requested
    UpdateRollingLogSize(configuration)

    # if this is a SIMULATION, then process every diff dump. But if it
    # isn't a simulation, then only read & do related processing for the
    # next dump if the last-modified timestamp indicates the file has been
    # updated since it was last read.
    tmp_timestamp = 0
    if not SIMULATION:
      dump_json_exists = os.path.exists(DUMP_JSON_FILE)
      if dump_json_exists:
        tmp_timestamp = os.path.getmtime(DUMP_JSON_FILE)
    if (SIMULATION and DumpJsonChanges()) or (
        not SIMULATION and dump_json_exists and
        tmp_timestamp > last_dump_json_timestamp):

      last_dump_json_timestamp = tmp_timestamp

      (persistent_nearby_aircraft,
       flight, now,
       json_desc_dict,
       persistent_path) = ScanForNewFlights(
           persistent_nearby_aircraft,
           persistent_path,
           configuration.get('log_jsons', False),
           flights)

      # Logging: As part of the memory instrumentation, let's track
      # the length of these data structures
      if not iteration % 1000:
        lengths = [len(flights), len(screen_history)]
        Log('Iteration: %d: object lengths: %s' % (iteration, lengths))

      # because this might just be an updated instance of the previous
      # flight as more identifier information (squawk and or flight number)
      # comes in, we only want to process this if its a truly new flight
      new_flight_flag = ConfirmNewFlight(flight, flights)

      if new_flight_flag:

        flights.append(flight)
        remote, servo = RefreshArduinos(
            remote, servo,
            to_remote_q, to_servo_q, to_main_q, shutdown,
            flights, json_desc_dict, configuration, screen_history)


        if FlightMeetsDisplayCriteria(flight, configuration, log=True):








          personal_message = None  # Any personal message displayed now cleared
          flight_message = (
              FLAG_MSG_FLIGHT, CreateMessageAboutFlight(flight), flight)

          # display the next message about this flight now!
          next_message_time = time.time()
          message_queue.insert(0, flight_message)

          # and delete any queued insight messages about other flights that have
          # not yet displayed, since a newer flight has taken precedence
          message_queue = DeleteMessageTypes(message_queue, (FLAG_MSG_INSIGHT,))

          # Though we also manage the message queue outside this conditional
          # as well, because it can take a half second to generate the flight
          # insights, this allows this message to start displaying on the
          # board immediately, so it's up there when it's most relevant
          next_message_time = ManageMessageQueue(
              message_queue, next_message_time, configuration, screen_history)

          insight_messages = CreateFlightInsights(
              flights,
              configuration.get('insights'),
              insight_message_distribution)
          if configuration.get('next_flight', 'off') == 'on':
            next_flight_text = FlightInsightNextFlight(flights, configuration)
            if next_flight_text:
              insight_messages.insert(0, next_flight_text)

          insight_messages = [(FLAG_MSG_INSIGHT, m) for m in insight_messages]


          for insight_message in insight_messages:
            message_queue.insert(0, insight_message)

        else:  # flight didn't meet display criteria
          flight['insight_types'] = []

        PickleObjectToFile(
            flight, PICKLE_FLIGHTS, True, timestamp=flight['now'])























      else:
        remote, servo = RefreshArduinos(
            remote, servo,
            to_remote_q, to_servo_q, to_main_q, shutdown,
            flights, json_desc_dict, configuration, screen_history)

    message_queue, next_message_time = ProcessArduinoCommmands(
        to_main_q, flights, configuration, message_queue, next_message_time)

    personal_message = PersonalMessage(
        configuration, message_queue, personal_message)

    # MEMORY MANAGEMENT
    # now that we've saved the flight, we have no more need to keep the
    # memory-hogging persistent_path key of that flight in live memory
    if 'persistent_path' in flights[-1]:
      del flights[-1]['persistent_path']
    # it turns out we only need the last screen in the screen history, not
    # the entire history, so we can purge all the rest from active memory




                            <----SKIPPED LINES---->

                            <----SKIPPED LINES---->





# This file is where the radio drops its json file
DUMP_JSON_FILE = '/run/readsb/aircraft.json'

# This file is where the query history to the flight aware webpage goes
FLIGHTAWARE_HISTORY_FILE = 'secure/flightaware.txt'

# At the time a flight is first identified as being of interest (in that
# it falls within MIN_METERS meters of HOME), it - and core attributes
# derived from FlightAware, if any - is appended to the end of this pickle
# file. However, since this file is cached in working memory, flights older
# than 30 days are flushed from this periodically.
PICKLE_FLIGHTS = 'pickle/flights.pk'

# This allows us to identify the full history (including what was last sent
# to the splitflap display in a programmatic fashion. While it may be
# interesting in its own right, its real use is to handle the "replay"
# button, so we know to enable it if what is displayed is the last flight.
PICKLE_SCREENS = 'pickle/screens.pk'

# Status data about messageboard systems - are they running, etc.  Specifically,
# has tuples of data (timestamp, system_id, status), where system_id is either
# the pin id of GPIO, or a 0 to indicate overall system, and status is boolean
PICKLE_SYSTEM_DASHBOARD = 'pickle/dashboard.pk'

# Flight-centric status data - what time was the flight first detected as
# in range, does it pass the display criteria, was a json available for it and
# when, was the local or web api used to communicate with the vestaboard
# and how much later, etc.
PICKLE_FLIGHT_DASHBOARD = 'pickle/flight_status.pk'

CACHED_ELEMENT_PREFIX = 'cached_'

# This web-exposed file is used for non-error messages that might highlight
# data or code logic to check into. It is only cleared out manually.
LOGFILE = 'log.txt'
# Identical to the LOGFILE, except it includes just the most recent n lines.
# Newest lines are at the end.
ROLLING_LOGFILE = 'secure/rolling_log.txt' #file for error messages

# default number of lines which may be overridden by settings file
ROLLING_LOG_SIZE = 1000

# Users can trigger .png histograms analogous to the text ones from the web
# interface; this is the folder (within WEBSERVER_PATH) where those files are
# placed
WEBSERVER_IMAGE_RELATIVE_FOLDER = 'images/'
# Multiple histograms can be generated, i.e. for airline, aircraft, day of
# week, etc. The output files are named by the prefix & suffix, i.e.: prefix +
# type + . + suffix, as in histogram_aircraft.png. These names match up to the




                            <----SKIPPED LINES---->




    8, 'Unused', False)

# GPIO pushbutton connections - (GPIO pin switch in; GPIO pin LED out)
GPIO_SOFT_RESET = (20, 21)

GOOGLE_ANALYTICS_TAG = (
    '<!-- Global site tag (gtag.js) - Google Analytics -->\n'
    '<script async src="https://www.googletagmanager.com/gtag/'
    'js?id=UA-99931533-2"></script>\n'
    '<script>\n'
    '  window.dataLayer = window.dataLayer || [];\n'
    '  function gtag(){dataLayer.push(arguments);}\n'
    "  gtag('js', new Date());\n"
    "  gtag('config', 'UA-99931533-2');\n"
    '</script>\n')

#if running on raspberry, then need to prepend path to file names
if RASPBERRY_PI:
  MEMORY_DIRECTORY = MESSAGEBOARD_PATH + MEMORY_DIRECTORY
  PICKLE_FLIGHTS = MESSAGEBOARD_PATH + PICKLE_FLIGHTS
  PICKLE_SYSTEM_DASHBOARD = MESSAGEBOARD_PATH + PICKLE_SYSTEM_DASHBOARD
  PICKLE_FLIGHT_DASHBOARD = MESSAGEBOARD_PATH + PICKLE_FLIGHT_DASHBOARD
  LOGFILE = MESSAGEBOARD_PATH + LOGFILE
  PICKLE_DUMP_JSON_FILE = MESSAGEBOARD_PATH + PICKLE_DUMP_JSON_FILE
  PICKLE_FA_JSON_FILE = MESSAGEBOARD_PATH + PICKLE_FA_JSON_FILE
  PICKLE_SCREENS = MESSAGEBOARD_PATH + PICKLE_SCREENS
  CODE_REPOSITORY = MESSAGEBOARD_PATH

  HISTOGRAM_CONFIG_FILE = WEBSERVER_PATH + HISTOGRAM_CONFIG_FILE
  CONFIG_FILE = WEBSERVER_PATH + CONFIG_FILE
  ROLLING_MESSAGE_FILE = WEBSERVER_PATH + ROLLING_MESSAGE_FILE
  ALL_MESSAGE_FILE = WEBSERVER_PATH + ALL_MESSAGE_FILE
  ROLLING_LOGFILE = WEBSERVER_PATH + ROLLING_LOGFILE
  STDERR_FILE = WEBSERVER_PATH + STDERR_FILE
  BACKUP_FILE = WEBSERVER_PATH + BACKUP_FILE
  SERVICE_VERIFICATION_FILE = WEBSERVER_PATH + SERVICE_VERIFICATION_FILE
  UPTIMES_FILE = WEBSERVER_PATH + UPTIMES_FILE
  CODE_HISTORY_FILE = WEBSERVER_PATH + CODE_HISTORY_FILE
  NEW_AIRCRAFT_FILE = WEBSERVER_PATH + NEW_AIRCRAFT_FILE
  FLIGHTAWARE_HISTORY_FILE = WEBSERVER_PATH + FLIGHTAWARE_HISTORY_FILE

  HISTOGRAM_IMAGE_HTML = WEBSERVER_PATH + HISTOGRAM_IMAGE_HTML




                            <----SKIPPED LINES---->




    persistent_nearby_aircraft: dictionary where keys are flight numbers, and
      the values are the time the flight was last seen.
    persistent_path: dictionary where keys are flight numbers, and the values
      are a sequential list of the location-attributes in the json file; allows
      for tracking the flight path over time.
    log_jsons: boolean indicating whether we should pickle the JSONs.
    flights: list of flight dictionaries; if no json is returned, used to
      find a recent flight with same flight number to augment this flight with
      origin / destination / airline.

  Returns:
    A tuple:
    - updated persistent_nearby_aircraft
    - (possibly empty) dictionary of flight attributes of the new flight upon
      its first observation.
    - the time of the radio observation if present; None if no radio dump
    - a dictionary of attributes about the dump itself (i.e.: # of flights;
      furthest observed flight, etc.)
    - persistent_path, a data structure containing past details of a flight's
      location as described in ParseDumpJson
    - a text message indicating any errors in querying FlightAware or
      populating flight details
  """
  flight_details = {}
  error_message = ''
  now = time.time()
  if SIMULATION:
    (dump_json, json_time) = DUMP_JSONS[SIMULATION_COUNTER]
  else:
    dump_json = ReadFile(DUMP_JSON_FILE, log_exception=True)

  json_desc_dict = {}
  current_nearby_aircraft = {}
  if dump_json:
    (current_nearby_aircraft, now,
     json_desc_dict, persistent_path) = ParseDumpJson(
         dump_json, persistent_path)

    if not SIMULATION and log_jsons:
      PickleObjectToFile((dump_json, now), PICKLE_DUMP_JSON_FILE, True)

    newly_nearby_flight_identifiers = UpdateAircraftList(
        persistent_nearby_aircraft, current_nearby_aircraft, now)

    if newly_nearby_flight_identifiers:




                            <----SKIPPED LINES---->




      if SIMULATION:
        json_times = [j[1] for j in FA_JSONS]
        if json_time in json_times:
          flight_aware_json = FA_JSONS[json_times.index(json_time)][0]
      elif flight_identifier[0]:
        flight_number = flight_identifier[0]
        flight_aware_json, error_message = GetFlightAwareJson(flight_number)
        if flight_aware_json:
          UpdateStatusLight(GPIO_ERROR_FLIGHT_AWARE_CONNECTION, False)
        else:
          failure_message = 'No json from Flightaware for flight %s: %s' % (
              flight_number, error_message[:500])
          Log(failure_message)
          UpdateStatusLight(
              GPIO_ERROR_FLIGHT_AWARE_CONNECTION, True, failure_message)

      flight_details = {}
      if flight_aware_json:
        flight_details = ParseFlightAwareJson(flight_aware_json)
      elif flight_identifier[0]:  # if there's a flight number but no json
        flight_details, derived_attr_msg = FindAttributesFromSimilarFlights(
            flight_identifier[0], flights)
        error_message += derived_attr_msg

      if not SIMULATION and log_jsons:
        PickleObjectToFile((flight_aware_json, now), PICKLE_FA_JSON_FILE, True)

      # Augment FlightAware details with radio / radio-derived details
      flight_details.update(current_nearby_aircraft[flight_identifier])

      # Augment with the past location data; the [1] is because recall that
      # persistent_path[key] is actually a 2-tuple, the first element being
      # the most recent time seen, and the second element being the actual
      # path. But we do not need to keep around the most recent time seen any
      # more.
      flight_details['persistent_path'] = persistent_path[flight_identifier][1]

  return (
      persistent_nearby_aircraft,
      flight_details,
      now,
      json_desc_dict,
      persistent_path,
      error_message)


def DescribeDumpJson(parsed):
  """Generates dict with descriptive attributes about the dump json file.

  Args:
    parsed: The parsed json file.

  Returns:
    Dictionary with attributes about radio range, number of flights seen, etc.
  """
  json_desc_dict = {}
  json_desc_dict['now'] = parsed['now']

  aircraft = [a for a in parsed['aircraft'] if a['seen'] < PERSISTENCE_SECONDS]
  json_desc_dict['radio_range_flights'] = len(aircraft)

  aircraft_with_pos = [a for a in aircraft if 'lat' in a and 'lon' in a]
  current_distances = [HaversineDistanceMeters(
      HOME, (a['lat'], a['lon'])) for a in aircraft_with_pos]




                            <----SKIPPED LINES---->




  if s is not None:
    s = unidecode.unidecode(s)
  return s


def FindAttributesFromSimilarFlights(this_flight_number, flights):
  """Returns a dictionary with info about a flight based on other flights.

  We may not get a json from the internet about this flight for any number
  of reasons: internet down; website down; too frequent queries; etc.  However,
  there are still some basic attributes we can derive about this flight
  from past observations of the same flight number, or past observations
  about the flight number prefix.  Specifically, we can get the flight's
  airline, origin, and destination.

  Args:
    this_flight_number: String of this flight number.
    flights: List of past flights.

  Returns:
    2-tuple of:
    - Dictionary of flight attributes extracted from the FlightAware json.
    - Status message indicating what attributes derived
  """
  derived_attributes = {}
  status_message = ''
  if not this_flight_number:
    return derived_attributes

  potential_values = {
      'origin_friendly': [],
      'origin_iata': [],
      'destination_friendly': [],
      'destination_iata': [],
      'airline_call_sign': [],
      'airline_short_name': [],
      'airline_full_name': []
      }

  def AppendIfPresent(k, flight):
    val = flight.get(k)
    if val and val != KEY_NOT_PRESENT_STRING:
      potential_values[k].append(val)

  for flight in flights:
    flight_number = flight.get('flight_number')




                            <----SKIPPED LINES---->




      if flight_number[:3] == this_flight_number[:3]:
        AppendIfPresent('airline_call_sign', flight)
        AppendIfPresent('airline_short_name', flight)
        AppendIfPresent('airline_full_name', flight)

  def Mode(lst):
    freq = {}
    for val in lst:
      freq[val] = freq.get(val, 0) + 1
    max_freq = max(freq.values())
    for val in freq:
      if freq[val] == max_freq:
        return val
    return None

  for attribute, lst in potential_values.items():
    if lst:
      derived_attributes[attribute] = Mode(lst)

  if derived_attributes:

    status_message = 'Derived attributes for %s based on past flights: %s' % (
      this_flight_number, str(derived_attributes))
    Log(status_message)
  return derived_attributes, status_message


def ParseFlightAwareJson(flight_json):
  """Strips relevant data about the flight from FlightAware feed.

  The FlightAware json has hundreds of fields about a flight, only a fraction
  of which are relevant to extract. Note that some of the fields are
  inconsistently populated (i.e.: scheduled and actual times for departure and
  take-off).

  Args:
    flight_json: Text representation of the FlightAware json about a single
      flight.

  Returns:
    Dictionary of flight attributes extracted from the FlightAware json.
  """
  flight = {}
  parsed_json = json.loads(flight_json)





                            <----SKIPPED LINES---->




    flight, configuration, display_all_hours=False, log=False):
  """Returns boolean indicating whether the screen accepting new flight data.

  Based on the configuration file, determines whether the flight data should be
  displayed. Specifically, the configuration:
  - may include 'enabled' indicating whether screen should be driven at all
  - should include 'on' & 'off' parameters indicating minute (from midnight) of
    operation
  - should include altitude & elevation parameters indicating max values of
    interest

  Args:
    flight: dictionary of flight attributes.
    configuration: dictionary of configuration attributes.
    display_all_hours: a boolean indicating whether we should ignore whether the
      screen is turned off (either via the enabling, or via the hour settings)
    log: optional boolean indicating whether a flight that fails the criteria
      should be logged with the reason

  Returns:
    2-tuple: Boolean as described; text indication for why message not
    displayed.
  """
  flight_altitude = flight.get('altitude', float('inf'))
  config_max_altitude = configuration['setting_max_altitude']
  failure_message = ''

  flight_meets_criteria = True
  if flight_altitude > config_max_altitude:
    flight_meets_criteria = False
    failure_message = (
        '%s not displayed because it fails altitude criteria - '
        'flight altitude: %.0f; required altitude: %.0f' %
        (DisplayFlightNumber(flight), flight_altitude, config_max_altitude))
    if log:
      Log(failure_message)




  else:
    flight_distance = flight.get('min_feet', float('inf'))
    config_max_distance = configuration['setting_max_distance']
    if flight_distance > config_max_distance:
      flight_meets_criteria = False
      failure_message = (
          '%s not displayed because it fails distance criteria - '
          'flight distance: %.0f; required distance: %.0f' %
          (DisplayFlightNumber(flight), flight_distance, config_max_distance))
      if log:
        Log(failure_message)





  if not display_all_hours and flight_meets_criteria:
    flight_timestamp = flight['now']
    minute_of_day = MinuteOfDay(flight_timestamp)
    if minute_of_day < configuration['setting_on_time']:
      flight_meets_criteria = False
      failure_message = (
          '%s not displayed because it occurs too early - minute_of_day: '
          '%d; setting_on_time: %d' % (DisplayFlightNumber(flight),
          minute_of_day, configuration['setting_on_time']))
      if log:
        Log(failure_message)




    elif minute_of_day > configuration['setting_off_time'] + 1:
      flight_meets_criteria = False
      failure_message = (
        '%s not displayed because it occurs too late - minute_of_day: '
        '%d; setting_off_time: %d' % (DisplayFlightNumber(flight),
        minute_of_day, configuration['setting_off_time']))
      if log:
        Log(failure_message)




    elif configuration.get('setting_screen_enabled', 'off') == 'off':
      flight_meets_criteria = False
      failure_message = '%s not displayed because screen disabled' % (
          DisplayFlightNumber(flight))
      if log:
        Log(failure_message)



  return flight_meets_criteria, failure_message


def MessageMeetsDisplayCriteria(configuration, log=False):
  """Returns boolean indicating whether screen is active.

  Based on the configuration file, determines whether the flight data should be
  displayed. Specifically, the configuration:
  - may include 'enabled' indicating whether screen should be driven at all
  - should include 'on' & 'off' parameters indicating minute (from midnight) of
    operation
  Note that this differs from FlightMeetsDisplayCriteria in that this function
  does not use any attribute about the flight (elevation or distance), only
  whether the screen is still active.

  Args:
    configuration: dictionary of configuration attributes.
    log: optional boolean indicating whether a flight that fails the criteria
      should be logged with the reason

  Returns:




                            <----SKIPPED LINES---->






def IdentifyFlightDisplayed(flights, configuration, display_all_hours=False):
  """Finds the most recent flight in flights that meet the display criteria.

  Args:
    flights: list of flight dictionaries.
    configuration: dictionary of settings.
    display_all_hours: boolean indicating whether we should ignore the time
    constraints (i.e.: whether the screen is enabled, and its turn-on or
    turn-off times) in identifying the most recent flight. That is, if False,
    then this will only return flights that would have been displayed in the
    ordinarily usage, vs. if True, a flight irrespective of the time it would
    be displayed.

  Returns:
    A flight dictionary if one can be found; None otherwise.
  """
  for n in range(len(flights)-1, -1, -1):  # traverse the flights in reverse
    if FlightMeetsDisplayCriteria(
        flights[n], configuration, display_all_hours=display_all_hours)[0]:
      return n
  return None


def CreateMessageAboutFlight(flight):
  """Creates a message to describe interesting attributes about a single flight.

  Generates a multi-line description of a flight. A typical message might look
  like:
  UAL300 - UNITED        <- Flight number and airline
  BOEING 777-200 (TWIN)  <- Aircraft type
  SFO-HNL HONOLULU       <- Origin & destination
  DEP 02:08 ER REM 5:14  <- Time details: departure; early / late; remaining
  185MPH 301DEG D:117FT  <- Trajectory: speed; bearing; fcst min dist to HOME
  1975FT (+2368FPM)      <- Altitude: current altitude & rate of ascent

  However, not all of these details are always present, so some may be listed
  as unknown, or entire lines may be left out.

  Args:




                            <----SKIPPED LINES---->




  this_flight = flights[-1]
  this_hour = int(DisplayTime(this_flight, '%-H'))
  this_minute = int(DisplayTime(this_flight, '%-M'))
  this_date = DisplayTime(this_flight, '%x')

  # Flights that we've already seen in the last few hours we do not expect to
  # see again for another few hours, so let's exclude them from the calculation
  exclude_flights_hours = 12
  flight_numbers_seen_in_last_n_hours = [
      f['flight_number'] for f in flights
      if f['now'] > this_flight['now'] - exclude_flights_hours*SECONDS_IN_HOUR
      and 'flight_number' in f]
  still_to_come_flights = [
      f for f in flights[:-1]
      if f.get('flight_number') not in flight_numbers_seen_in_last_n_hours
      and this_date != DisplayTime(f, '%x')]

  # exclude flights that would be filtered out by altitude or distance
  still_to_come_flights = [
      f for f in still_to_come_flights
      if FlightMeetsDisplayCriteria(f, configuration)[0]]

  # exclude flights more than 30 days in the past
  now = time.time()
  still_to_come_flights = [
      f for f in still_to_come_flights
      if now - f['now'] < MAX_INSIGHT_HORIZON_DAYS * SECONDS_IN_DAY]

  minimum_minutes_next_flight = {}  # min minutes to next flight by day
  for flight in still_to_come_flights:
    date = DisplayTime(flight, '%x')
    hour = int(DisplayTime(flight, '%-H'))
    minutes = int(DisplayTime(flight, '%-M'))
    minutes_after = (hour - this_hour) * MINUTES_IN_HOUR + (
        minutes - this_minute)
    if minutes_after < 0:
      minutes_after += MINUTES_IN_DAY
    minimum_minutes_next_flight[date] = min(
        minimum_minutes_next_flight.get(date, minutes_after), minutes_after)

  minutes = list(minimum_minutes_next_flight.values())




                            <----SKIPPED LINES---->




  global LOGFILE
  LOGFILE = PrependFileName(LOGFILE, SIMULATION_PREFIX)
  ClearFile(LOGFILE)

  global ROLLING_LOGFILE
  ROLLING_LOGFILE = PrependFileName(ROLLING_LOGFILE, SIMULATION_PREFIX)
  ClearFile(ROLLING_LOGFILE)

  global ROLLING_MESSAGE_FILE
  ROLLING_MESSAGE_FILE = PrependFileName(
      ROLLING_MESSAGE_FILE, SIMULATION_PREFIX)
  ClearFile(ROLLING_MESSAGE_FILE)

  global PICKLE_FLIGHTS
  PICKLE_FLIGHTS = PrependFileName(PICKLE_FLIGHTS, SIMULATION_PREFIX)
  filenames = UnpickleObjectFromFile(
      PICKLE_FLIGHTS, True, max_days=None, filenames=True)
  for file in filenames:
    ClearFile(file)

  global PICKLE_SYSTEM_DASHBOARD
  PICKLE_SYSTEM_DASHBOARD = PrependFileName(
      PICKLE_SYSTEM_DASHBOARD, SIMULATION_PREFIX)
  filenames = UnpickleObjectFromFile(
      PICKLE_SYSTEM_DASHBOARD, True, max_days=None, filenames=True)
  for file in filenames:
    ClearFile(file)

  global PICKLE_FLIGHT_DASHBOARD
  PICKLE_FLIGHT_DASHBOARD = PrependFileName(
      PICKLE_FLIGHT_DASHBOARD, SIMULATION_PREFIX)
  filenames = UnpickleObjectFromFile(
      PICKLE_FLIGHT_DASHBOARD, True, max_days=None, filenames=True)
  for file in filenames:
    ClearFile(file)

def SimulationEnd(message_queue, flights, screens):
  """Clears message buffer, exercises histograms, and other simulation wrap up.

  Args:
    message_queue: List of flight messages that have not yet been printed.
    flights: List of flights dictionaries.
    screens: List of past screens displayed to splitflap screen.
  """
  if flights:
    histogram = {
        'type': 'both',
        'histogram':'all',
        'histogram_history':'30d',
        'histogram_max_screens': '_2',
        'histogram_data_summary': 'on'}
    message_queue.extend(TriggerHistograms(flights, histogram))

    while message_queue:
      ManageMessageQueue(message_queue, 0, {'setting_delay': 0}, screens)




                            <----SKIPPED LINES---->




    s,
    subscription_id=SUBSCRIPTION_ID,
    key=KEY,
    secret=SECRET,
    timeout=5):
  """Publishes a text string to a Vestaboard.

  The message is pushed to the vestaboard splitflap display by way of its
  web services; see https://docs.vestaboard.com/introduction for more details.

  TODO: rewrite to use the easier-to-follow requests library, more in line
  with PublishMessageLocal.

  Args:
    s: String to publish.
    subscription_id: string subscription id from Vestaboard.
    key: string key from Vestaboard.
    secret: string secret from Vestaboard.
    timeout: Max duration in seconds that we should wait to establish a
      connection.

  Returns:
    Text string indicating how the message was displayed (web or local
    api), and / or any error messages encountered.
  """
  error_code = False
  curl = pycurl.Curl()

  # See https://stackoverflow.com/questions/31826814/
  # curl-post-request-into-pycurl-code
  # Set URL value
  curl.setopt(
      pycurl.URL,
      'https://platform.vestaboard.com/subscriptions/%s/message'
      % subscription_id)
  curl.setopt(pycurl.HTTPHEADER, [
      'X-Vestaboard-Api-Key:%s' % key, 'X-Vestaboard-Api-Secret:%s' % secret])
  curl.setopt(pycurl.TIMEOUT_MS, timeout*1000)
  curl.setopt(pycurl.POST, 1)

  curl.setopt(pycurl.WRITEFUNCTION, lambda x: None) # to keep stdout clean

  # preparing body the way pycurl.READDATA wants it
  body_as_dict = {'characters': StringToCharArray(s)}
  body_as_json_string = json.dumps(body_as_dict) # dict to json
  body_as_file_object = io.StringIO(body_as_json_string)

  # prepare and send. See also: pycurl.READFUNCTION to pass function instead
  curl.setopt(pycurl.READDATA, body_as_file_object)
  curl.setopt(pycurl.POSTFIELDSIZE, len(body_as_json_string))
  failure_message = ''
  try:
    curl.perform()
    status = 'Web service published'
  except pycurl.error as e:
    timing_message = CurlTimingDetailsToString(curl)
    failure_message = (
        'curl.perform() failed with message %s; timing details: %s' %
        (e, timing_message))
    # Using the remote webservice failed, but maybe the local API will
    # be more successful?  If this succeeds, then we should not indicate
    # a failure on the status light / dashboard, but we should still log
    # the remote failure
    Log(failure_message)
    error_code = PublishMessageLocal(s, timeout=timeout, update_dashboard=False)
    if not error_code:
      status = (
          'Local service published because web service failed with %s'
          % failure_message)
    else:
      status = (
          'Local service failed with %s after web service failed with %s' %
          (error_code, failure_message))
  else:
    # you may want to check HTTP response code, e.g.
    timing_message = CurlTimingDetailsToString(curl)
    status_code = curl.getinfo(pycurl.RESPONSE_CODE)
    if status_code != 200:
      failure_message = (
          'Server returned HTTP status code %d for message %s; '
          'timing details: %s' % (status_code, s, timing_message))
      Log(failure_message)
      error_code = PublishMessageLocal(
          s, timeout=timeout, update_dashboard=False)
      if not error_code:
        status = (
            'Local service published because web service failed with %s'
            % failure_message)
      else:
        status = (
            'Local service failed with %s after web service failed with %s' %
            (error_code, failure_message))

  # We've logged the error code from the external web service, but the
  # Vestaboard local API was able to recover from the error, so we need not
  # log the failure message / error code in the dashboard.
  if not error_code:
    failure_message = ''

  curl.close()
  UpdateStatusLight(
      GPIO_ERROR_VESTABOARD_CONNECTION, error_code, failure_message)

  return status


def PublishMessageLocal(
    s,
    local_key=LOCAL_KEY,
    local_address=LOCAL_VESTABOARD_ADDRESS,
    timeout=5,
    update_dashboard=True):
  """Publishes a text string to a Vestaboard via local API.

  The message is pushed to the vestaboard splitflap display by way of its
  local API; see https://docs.vestaboard.com/local for more details.

  Args:
    s: String to publish.
    local_key: string key from Vestaboard for local API access.
    local_address: the address and port to the local Vestaboard service.
    timeout: Max duration in seconds that we should wait to establish a
      connection.
    update_dashboard: Boolean indicating whether this method should update the




                            <----SKIPPED LINES---->





  return personal_message


def ManageMessageQueue(
    message_queue, next_message_time, configuration, screens):
  """Check time & if appropriate, display next message from queue.

  Args:
    message_queue: FIFO list of message tuples of (message type,
      message string).
    next_message_time: epoch at which next message should be displayed
    configuration: dictionary of configuration attributes.
    screens: List of past screens displayed to splitflap screen.

  Returns:
    Next_message_time, potentially updated if a message has been displayed,
    or unchanged if no message was displayed.
  """
  if message_queue and (time.time() >= next_message_time or SIMULATION):
    status = ''

    if SIMULATION:  # drain the queue because the messages come so fast
      messages_to_display = list(message_queue)
      # passed by reference, so clear it out since we drained it to the display
      del message_queue[:]
    else:  # display only one message, being mindful of the display timing
      messages_to_display = [message_queue.pop(0)]

    for message in messages_to_display:
      # we cannot just unpack the tuple because messages of type
      # FLAG_MSG_FLIGHT are 3-tuples (with the third element being the flight
      # dictionary) whereas other message types are 2-tuples
      message_type = message[0]
      message_text = message[1]

      # There may be one or several insight messages that were added to the
      # message queue along with the flight at a time when the screen was
      # enabled, but by the time it comes to display them, the screen is now
      # disabled.  These should not be displayed.  Note that this check only
      # needs to be done for insight messages because other message types
      # are user initiated and so presumably should be displayed irrespective
      # of when the user triggered it to be displayed.
      if message_type == FLAG_MSG_INSIGHT and not MessageMeetsDisplayCriteria(
          configuration):
        status = 'Message purged as no longer meets display criteria'
        Log('Message %s purged' % message_text)

      else:
        if isinstance(message_text, str):
          message_text = textwrap.wrap(
              message_text,
              width=SPLITFLAP_CHARS_PER_LINE)
        display_message = Screenify(message_text, False)
        Log(display_message, file=ALL_MESSAGE_FILE)

        # Saving this to disk allows us to identify
        # persistently whats currently on the screen
        PickleObjectToFile(message, PICKLE_SCREENS, True)
        screens.append(message)

        MaintainRollingWebLog(display_message, 25)
        if not SIMULATION:
          splitflap_message = Screenify(message_text, True)
          status = PublishMessageWeb(splitflap_message)

      if message_type in (FLAG_MSG_INSIGHT, FLAG_MSG_FLIGHT):
        flight = message[2]
        # We record flight number, time stamp, message, and status
        # to a pickle file so that we may construct a flight-centric
        # status report
        #
        # Specifically, we record a 3-tuple:
        # - flight number
        # - time stamp of the recording
        # - a dictionary of elements
        data = (
            DisplayFlightNumber(flight),
            time.time(),
            {
                'message_text': message_text,
                'status': status,
                'message_type': message_type})
        PickleObjectToFile(data, PICKLE_FLIGHT_DASHBOARD, True)


    next_message_time = time.time() + configuration['setting_delay']
  return next_message_time


def DeleteMessageTypes(q, types_to_delete):
  """Delete messages from the queue if type is in the iterable types."""
  if VERBOSE:
    messages_to_delete = [m for m in q if m[0] in types_to_delete]
    if messages_to_delete:
      Log('Deleting messages from queue due to new-found plane: %s'
          % messages_to_delete)
  updated_q = [m for m in q if m[0] not in types_to_delete]
  return updated_q


def BootstrapInsightList(full_path=PICKLE_FLIGHTS):
  """(Re)populate flight pickle files with flight insight distributions.

  The set of insights generated for each flight is created at the time




                            <----SKIPPED LINES---->




  Args:
    value: Boolean indicating whether a failure has occurred (True) or
      system is nominal (False).
    subsystem: A tuple describing the system; though that description may
      have multiple attributes, the 0th element is the numeric identifier
      of that system.  monitoring.py depends on other attributes of that
      tuple being present as well.  Since the overall system does not have
      a tuple defined for it, it gets a default identifier of 0.
    failure_message: an (optional) message describing why the system /
      subsystem is being disabled or failing.
    iteration: integer indicating how many times the main loop has been
      completed.
  """
  versions = (VERSION_MESSAGEBOARD, VERSION_ARDUINO)
  if subsystem:
    subsystem = subsystem[0]
  PickleObjectToFile((
      time.time(), subsystem, value, versions,
      failure_message, INSTANCE_START_TIME,
      iteration, gpiozero.CPUTemperature().temperature),
      PICKLE_SYSTEM_DASHBOARD, True)


def RemoveFile(file):
  """Removes a file, returning a boolean indicating if it had existed."""
  if os.path.exists(file):

    try:
      os.remove(file)
    except PermissionError:
      return False
    return True

  return False


def ConfirmNewFlight(flight, flights):
  """Replaces last-seen flight with new flight if identifiers overlap.

  Flights are identified by the radio over time by a tuple of identifiers:
  flight_number and squawk.  Due to unknown communication issues, one or the




                            <----SKIPPED LINES---->




    configuration = new_configuration

    ResetLogs(configuration)  # clear the logs if requested
    UpdateRollingLogSize(configuration)

    # if this is a SIMULATION, then process every diff dump. But if it
    # isn't a simulation, then only read & do related processing for the
    # next dump if the last-modified timestamp indicates the file has been
    # updated since it was last read.
    tmp_timestamp = 0
    if not SIMULATION:
      dump_json_exists = os.path.exists(DUMP_JSON_FILE)
      if dump_json_exists:
        tmp_timestamp = os.path.getmtime(DUMP_JSON_FILE)
    if (SIMULATION and DumpJsonChanges()) or (
        not SIMULATION and dump_json_exists and
        tmp_timestamp > last_dump_json_timestamp):

      last_dump_json_timestamp = tmp_timestamp

      (persistent_nearby_aircraft, flight, now, json_desc_dict,


       persistent_path, flight_aware_error_message) = ScanForNewFlights(
           persistent_nearby_aircraft,
           persistent_path,
           configuration.get('log_jsons', False),
           flights)

      # Logging: As part of the memory instrumentation, let's track
      # the length of these data structures
      if not iteration % 1000:
        lengths = [len(flights), len(screen_history)]
        Log('Iteration: %d: object lengths: %s' % (iteration, lengths))

      # because this might just be an updated instance of the previous
      # flight as more identifier information (squawk and or flight number)
      # comes in, we only want to process this if its a truly new flight
      new_flight_flag = ConfirmNewFlight(flight, flights)

      if new_flight_flag:
        time_new_flight_found = time.time()
        flights.append(flight)
        remote, servo = RefreshArduinos(
            remote, servo,
            to_remote_q, to_servo_q, to_main_q, shutdown,
            flights, json_desc_dict, configuration, screen_history)

        flight_meets_display_criteria, reason_flight_fails_criteria = (
            FlightMeetsDisplayCriteria(flight, configuration, log=True))

        # Initialize variables for saving in case details not populated by
        # later code prior to saving in pickle
        flight_aware_error_message = ''
        time_flight_message_inserted = 0
        time_insight_message_inserted = 0

        if flight_meets_display_criteria:
          personal_message = None  # Any personal message displayed now cleared
          flight_message = (
              FLAG_MSG_FLIGHT, CreateMessageAboutFlight(flight), flight)

          # display the next message about this flight now!
          next_message_time = time.time()
          message_queue.insert(0, flight_message)
          time_flight_message_inserted = time.time()
          # and delete any queued insight messages about other flights that have
          # not yet displayed, since a newer flight has taken precedence
          message_queue = DeleteMessageTypes(message_queue, (FLAG_MSG_INSIGHT,))

          # Though we also manage the message queue outside this conditional
          # as well, because it can take a half second to generate the flight
          # insights, this allows this message to start displaying on the
          # board immediately, so it's up there when it's most relevant
          next_message_time = ManageMessageQueue(
              message_queue, next_message_time, configuration, screen_history)

          insight_messages = CreateFlightInsights(
              flights,
              configuration.get('insights'),
              insight_message_distribution)
          if configuration.get('next_flight', 'off') == 'on':
            next_flight_text = FlightInsightNextFlight(flights, configuration)
            if next_flight_text:
              insight_messages.insert(0, next_flight_text)

          insight_messages = [(FLAG_MSG_INSIGHT, m) for m in insight_messages]

          time_insight_message_inserted = time.time()
          for insight_message in insight_messages:
            message_queue.insert(0, insight_message)

        else:  # flight didn't meet display criteria
          flight['insight_types'] = []

        PickleObjectToFile(
            flight, PICKLE_FLIGHTS, True, timestamp=flight['now'])

        # We record flight number, flight_meets_display_criteria,
        # reason_flight_fails_criteria, flight_aware_error_message, and
        # time stamps of when we confirmed new flight; when we generated
        # message; when we generated insight messages (if any), to a
        # to a pickle file so that we may construct a flight-centric
        # status report
        #
        # Specifically, we record a 3-tuple:
        # - flight number
        # - time stamp of the recording
        # - a dictionary of elements
        data = (
            DisplayFlightNumber(flight),
            time.time(),
            {
                'reason_flight_fails_criteria': reason_flight_fails_criteria,
                'flight_aware_error_message': flight_aware_error_message,
                'time_new_flight_found': time_new_flight_found,
                'time_flight_message_inserted': time_flight_message_inserted,
                'time_insight_message_inserted': time_insight_message_inserted})
        PickleObjectToFile(data, PICKLE_FLIGHT_DASHBOARD, True)

      else:
        remote, servo = RefreshArduinos(
            remote, servo,
            to_remote_q, to_servo_q, to_main_q, shutdown,
            flights, json_desc_dict, configuration, screen_history)

    message_queue, next_message_time = ProcessArduinoCommmands(
        to_main_q, flights, configuration, message_queue, next_message_time)

    personal_message = PersonalMessage(
        configuration, message_queue, personal_message)

    # MEMORY MANAGEMENT
    # now that we've saved the flight, we have no more need to keep the
    # memory-hogging persistent_path key of that flight in live memory
    if 'persistent_path' in flights[-1]:
      del flights[-1]['persistent_path']
    # it turns out we only need the last screen in the screen history, not
    # the entire history, so we can purge all the rest from active memory




                            <----SKIPPED LINES---->