#!/usr/bin/python3

import datetime
import io
import json
import math

import numbers
import os
import pickle

import shutil
import signal
import statistics
import sys
import textwrap
import time

import bs4
import dateutil.relativedelta

import numpy
import matplotlib
import matplotlib.pyplot
import psutil
import pycurl
import pytz
import requests
import tzlocal
import unidecode










SIMULATION = False
SIMULATION_COUNTER = 0
SIMULATION_PREFIX = 'SIM_'
PICKLE_DUMP_JSON_FILE = 'dump_json.pk'
PICKLE_FA_JSON_FILE = 'fa_json.pk'
DUMP_JSONS = None  # loaded only if in simulation mode
FA_JSONS = None  # loaded only if in simulation mode

HOME_LAT = 37.64406
HOME_LON = -122.43463
HOME = (HOME_LAT, HOME_LON) # lat / lon tuple of antenna
HOME_ALT = 29  #altitude in meters
RADIUS = 6371.0e3  # radius of earth in meters

FEET_IN_METER = 3.28084
FEET_IN_MILE = 5280
METERS_PER_SECOND_IN_KNOTS = 0.514444

MIN_METERS = 5000/FEET_IN_METER # only planes within this distance will be detailed
# planes not seen within MIN_METERS in PERSISTENCE_SECONDS seconds will be dropped from
# the nearby list
PERSISTENCE_SECONDS = 10
TRUNCATE = 50  # max number of keys to include in a histogram image file
# number of seconds to pause between each radio poll / command processing loop
LOOP_DELAY_SECONDS = 1

# This file is where the radio drops its json file
DUMP_JSON_FILE = '/run/readsb/aircraft.json'

# This is the directory that stores all the ancillary messageboard configuration files
# that do not need to be exposed via the webserver
MESSAGEBOARD_PATH = '/home/pi/splitflap/'
# This is the directory of the webserver; files placed here are available at
# http://adsbx-custom.local/; files placed in this directly are visible via a browser
WEBSERVER_PATH = '/var/www/html/'

# At the time a flight is first identified as being of interest (in that it falls
# within MIN_METERS meters of HOME), it - and core attributes derived from FlightAware,
# if any - is appended to the end of this pickle file. However, since this file is
# cached in working memory, flights older than 30 days are flushed from this periodically.
PICKLE_FLIGHTS_30D = 'flights_30d.pk' #pickled list of up to about 30d of flights
# This is the same concept as the 30d pickle file, except it is not auto-flushed, and
# so will grow indefinitely.
PICKLE_FLIGHTS_ARCHIVE = 'flights_archive.pk' #pickled list of all flights

CACHED_ELEMENT_PREFIX = 'cached_'

# This web-exposed file is used for non-error messages that might highlight data or
# code logic to check into. It is only cleared out manually.
LOGFILE = 'log.txt'

# Identical to the LOGFILE, except it includes just the most recent n lines. Newest
# lines are at the end.
ROLLING_LOGFILE = 'rolling_log.txt' #file for error messages

# Users can trigger .png histograms analogous to the text ones from the web interface;
# this is the folder (within WEBSERVER_PATH) where those files are placed
WEBSERVER_IMAGE_FOLDER = 'images/'
# Multiple histograms can be generated, i.e. for airline, aircraft, day of week, etc.
# The output files are named by the prefix & suffix, i.e.: prefix + type + . + suffix,
# as in histogram_aircraft.png. These names match up to the names expected by the html
# page that displays the images. Also, note that the suffix is interpreted by matplotlib
# to identify the image format to create.
HISTOGRAM_IMAGE_PREFIX = 'histogram_'
HISTOGRAM_IMAGE_SUFFIX = 'png'
# For those of the approximately ten different types of histograms _not_ generated,
# an empty image is copied into the location expected by the webpage instead; this is
# the location of that "empty" image file.
HISTOGRAM_EMPTY_IMAGE_FILE = 'empty.png'

# This file indicates a pending request for histograms - either png, text-based, or




                            <----SKIPPED LINES---->




FLAG_INSIGHT_GROUNDSPEED = 3
FLAG_INSIGHT_ALTITUDE = 4
FLAG_INSIGHT_VERTRATE = 5
FLAG_INSIGHT_FIRST_DEST = 6
FLAG_INSIGHT_FIRST_ORIGIN = 7
FLAG_INSIGHT_FIRST_AIRLINE = 8
FLAG_INSIGHT_FIRST_AIRCRAFT = 9
FLAG_INSIGHT_LONGEST_DELAY = 10
FLAG_INSIGHT_FLIGHT_DELAY_FREQUENCY = 11
FLAG_INSIGHT_FLIGHT_DELAY_TIME = 12
FLAG_INSIGHT_AIRLINE_DELAY_FREQUENCY = 13
FLAG_INSIGHT_AIRLINE_DELAY_TIME = 14
FLAG_INSIGHT_DESTINATION_DELAY_FREQUENCY = 15
FLAG_INSIGHT_DESTINATION_DELAY_TIME = 16
FLAG_INSIGHT_HOUR_DELAY_FREQUENCY = 17
FLAG_INSIGHT_HOUR_DELAY_TIME = 18
FLAG_INSIGHT_DATE_DELAY_FREQUENCY = 19
FLAG_INSIGHT_DATE_DELAY_TIME = 20
INSIGHT_TYPES = 21














#if running on raspberry, then need to prepend path to file names
if psutil.sys.platform.title() == 'Linux':
  PICKLE_FLIGHTS_30D = MESSAGEBOARD_PATH + PICKLE_FLIGHTS_30D
  PICKLE_FLIGHTS_ARCHIVE = MESSAGEBOARD_PATH + PICKLE_FLIGHTS_ARCHIVE
  LOGFILE = MESSAGEBOARD_PATH + LOGFILE
  PICKLE_DUMP_JSON_FILE = MESSAGEBOARD_PATH + PICKLE_DUMP_JSON_FILE
  PICKLE_FA_JSON_FILE = MESSAGEBOARD_PATH + PICKLE_FA_JSON_FILE
  LAST_FLIGHT_FILE = MESSAGEBOARD_PATH + LAST_FLIGHT_FILE
  ARDUINO_FILE = MESSAGEBOARD_PATH + ARDUINO_FILE

  HISTOGRAM_CONFIG_FILE = WEBSERVER_PATH + HISTOGRAM_CONFIG_FILE
  CONFIG_FILE = WEBSERVER_PATH + CONFIG_FILE
  HOURLY_IMAGE_FILE = WEBSERVER_PATH + WEBSERVER_IMAGE_FOLDER + HOURLY_IMAGE_FILE
  ROLLING_MESSAGE_FILE = WEBSERVER_PATH + ROLLING_MESSAGE_FILE
  HISTOGRAM_IMAGE_PREFIX = (
      WEBSERVER_PATH + WEBSERVER_IMAGE_FOLDER + HISTOGRAM_IMAGE_PREFIX)
  HISTOGRAM_EMPTY_IMAGE_FILE = (
      WEBSERVER_PATH + WEBSERVER_IMAGE_FOLDER + HISTOGRAM_EMPTY_IMAGE_FILE)
  ALL_MESSAGE_FILE = WEBSERVER_PATH + ALL_MESSAGE_FILE
  ROLLING_LOGFILE = WEBSERVER_PATH + ROLLING_LOGFILE
  STDERR_FILE = WEBSERVER_PATH + STDERR_FILE
  BACKUP_FILE = WEBSERVER_PATH + BACKUP_FILE
  SERVICE_VERIFICATION_FILE = WEBSERVER_PATH + SERVICE_VERIFICATION_FILE






TIMEZONE = 'US/Pacific' # timezone of display
TZ = pytz.timezone(TIMEZONE)

KNOWN_AIRPORTS = ('SJC', 'SFO', 'OAK')  # iata codes that we don't need to expand

SPLITFLAP_CHARS_PER_LINE = 22
SPLITFLAP_LINE_COUNT = 6

DIRECTIONS_4 = ['N', 'E', 'S', 'W']
DIRECTIONS_8 = ['N', 'NE', 'E', 'SE', 'S', 'SW', 'W', 'NW']
DIRECTIONS_16 = ['N', 'NNE', 'NE', 'ENE', 'E', 'ESE', 'SE', 'SSE',
                 'S', 'SSW', 'SW', 'WSW', 'W', 'WNW', 'NW', 'NNW']

HOURS = ['12a', ' 1a', ' 2a', ' 3a', ' 4a', ' 5a', ' 6a', ' 7a',
         ' 8a', ' 9a', '10a', '11a', '12p', ' 1p', ' 2p', ' 3p',
         ' 4p', ' 5p', ' 6p', ' 7p', ' 8p', ' 9p', '10p', '11p']

SECONDS_IN_MINUTE = 60
MINUTES_IN_HOUR = 60




                            <----SKIPPED LINES---->




aircraft_length['Airbus A320 (twin-jet)'] = 37.57
aircraft_length['Airbus A320neo (twin-jet)'] = 37.57
aircraft_length['Airbus A321 (twin-jet)'] = 44.51
aircraft_length['Airbus A321neo (twin-jet)'] = 44.51
aircraft_length['Airbus A330-200 (twin-jet)'] = 58.82
aircraft_length['Airbus A330-300 (twin-jet)'] = 63.67
aircraft_length['Airbus A340-300 (quad-jet)'] = 63.69
aircraft_length['Airbus A350-1000 (twin-jet)'] = 73.79
aircraft_length['Airbus A350-900 (twin-jet)'] = 66.8
aircraft_length['Airbus A380-800 (quad-jet)'] = 72.72
aircraft_length['Boeing 737-400 (twin-jet)'] = 36.4
aircraft_length['Boeing 737-700 (twin-jet)'] = 33.63
aircraft_length['Boeing 737-800 (twin-jet)'] = 39.47
aircraft_length['Boeing 737-900 (twin-jet)'] = 42.11
aircraft_length['Boeing 747-400 (quad-jet)'] = 36.4
aircraft_length['Boeing 747-8 (quad-jet)'] = 76.25
aircraft_length['Boeing 757-200 (twin-jet)'] = 47.3
aircraft_length['Boeing 757-300 (twin-jet)'] = 54.4
aircraft_length['Boeing 767-200 (twin-jet)'] = 48.51
aircraft_length['Boeing 767-300 (twin-jet)'] = 54.94

aircraft_length['Boeing 777-200 (twin-jet)'] = 63.73
aircraft_length['Boeing 777-200LR/F (twin-jet)'] = 63.73
aircraft_length['Boeing 777-300ER (twin-jet)'] = 73.86
aircraft_length['Boeing 787-10 (twin-jet)'] = 68.28
aircraft_length['Boeing 787-8 (twin-jet)'] = 56.72
aircraft_length['Boeing 787-9 (twin-jet)'] = 62.81
aircraft_length['Canadair Regional Jet CRJ-200 (twin-jet)'] = 26.77
aircraft_length['Canadair Regional Jet CRJ-700 (twin-jet)'] = 32.3
aircraft_length['Canadair Regional Jet CRJ-900 (twin-jet)'] = 36.2
aircraft_length['Canadair Challenger 350 (twin-jet)'] = 20.9
aircraft_length['Bombardier Challenger 300 (twin-jet)'] = 20.92
aircraft_length['Embraer 170/175 (twin-jet)'] = (29.90 + 31.68) / 2
aircraft_length['EMBRAER 175 (long wing) (twin-jet)'] = 31.68
aircraft_length['Embraer ERJ-135 (twin-jet)'] = 26.33
aircraft_length['Cessna Caravan (single-turboprop)'] = 11.46
aircraft_length['Cessna Citation II (twin-jet)'] = 14.54
aircraft_length['Cessna Citation V (twin-jet)'] = 14.91

aircraft_length['Cessna Skyhawk (piston-single)'] = 8.28
aircraft_length['Cessna Skylane (piston-single)'] = 8.84
aircraft_length['Cessna Citation Sovereign (twin-jet)'] = 19.35
aircraft_length['Cessna T206 Turbo Stationair (piston-single)'] = 8.61
aircraft_length['Beechcraft Bonanza (33) (piston-single)'] = 7.65
aircraft_length['Beechcraft Super King Air 200 (twin-turboprop)'] = 13.31
aircraft_length['Beechcraft Super King Air 350 (twin-turboprop)'] = 14.22
aircraft_length['Beechcraft King Air 90 (twin-turboprop)'] = 10.82
aircraft_length['Pilatus PC-12 (single-turboprop)'] = 14.4


def CheckIfProcessRunning():
  """Returns proc id if process with identically-named python file running; else None."""
  this_process_id = os.getpid()
  this_process_name = os.path.basename(sys.argv[0])
  for proc in psutil.process_iter():
    try:
      # Check if process name contains this_process_name.
      commands = proc.as_dict(attrs=['cmdline', 'pid'])
      if commands['cmdline']:
        command_running = any(
            [this_process_name in s for s in commands['cmdline']])
        if command_running and commands['pid'] != this_process_id:
          return commands['pid']
    except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
      pass
  return None


def LogMessage(message, file=None):
  """Write a message to a logfile along with a timestamp.

  Args:
      message: string message to write
      file: string representing file name and, if needed, path to the file to write to
  """
  # can't define as a default parameter because LOGFILE name is potentially
  # modified based on SIMULATION flag
  if not file:
    file = LOGFILE





  try:
    with open(file, 'a') as f:
      if not SIMULATION:  # by excluding the timestamp, file diffs become easy between runs
        f.write('='*80+'\n')
        f.write(str(datetime.datetime.now(TZ))+'\n')
        f.write('\n')
      f.write(str(message)+'\n')
  except IOError:
    LogMessage('Unable to append to ' + file)





  existing_log_lines = ReadFile(LOGFILE).splitlines()
  with open(ROLLING_LOGFILE, 'w') as f:
    f.write('\n'.join(existing_log_lines[-1000:]))


def MaintainRollingWebLog(message, max_count, filename=None):
  """Maintains a rolling text file of at most max_count printed messages.

  Newest data at top and oldest data at the end, of at most max_count messages,
  where the delimiter between each message is identified by a special fixed string.

  Args:
    message: text message to prepend to the file.
    max_count: maximum number of messages to keep in the file; the max_count+1st message
        is deleted.
    filename: the file to update.
  """
  # can't define as a default parameter because ROLLING_MESSAGE_FILE name is potentially
  # modified based on SIMULATION flag
  if not filename:
    filename = ROLLING_MESSAGE_FILE
  rolling_log_header = '='*(SPLITFLAP_CHARS_PER_LINE + 2)
  existing_file = ReadFile(filename)
  log_message_count = existing_file.count(rolling_log_header)
  if log_message_count >= max_count:
    message_start_list = [i for i in range(0, len(existing_file))
                          if existing_file[i:].startswith(rolling_log_header)]
    existing_file_to_keep = existing_file[:message_start_list[max_count - 1]]
  else:
    existing_file_to_keep = existing_file

  t = datetime.datetime.now(TZ).strftime('%m/%d/%Y, %H:%M:%S')
  new_message = (
      '\n'.join([rolling_log_header, t, '', message])
      + '\n' + existing_file_to_keep)
  try:
    with open(filename, 'w') as f:
      f.write(new_message)
  except IOError:
    LogMessage('Unable to maintain rolling log at ' + filename)


def UtcToLocalTimeDifference(timezone=TIMEZONE):
  """Calculates number of seconds between UTC and given timezone.

  Returns number of seconds between UTC and given timezone; if no timezone given, uses
  TIMEZONE defined in global variable.

  Args:
    timezone: string representing a valid pytz timezone in pytz.all_timezones.

  Returns:
    Integer number of seconds.
  """
  utcnow = pytz.timezone('utc').localize(datetime.datetime.utcnow())
  home_time = utcnow.astimezone(pytz.timezone(timezone)).replace(tzinfo=None)
  system_time = utcnow.astimezone(tzlocal.get_localzone()).replace(tzinfo=None)

  offset = dateutil.relativedelta.relativedelta(home_time, system_time)
  offset_seconds = offset.hours * SECONDS_IN_HOUR




                            <----SKIPPED LINES---->




  max_time = flights[-1]['now']
  delta_hours = (max_time - min_time) / SECONDS_IN_HOUR
  return round(delta_hours)


def ReadFile(filename, log_exception=False):
  """Returns text from the given file name if available, empty string if not available.

  Args:
    filename: string of the filename to open, potentially also including the full path.
    log_exception: boolean indicating whether to log an exception if file not found.

  Returns:
    Return text string of file contents.
  """
  try:
    with open(filename, 'r') as content_file:
      file_contents = content_file.read()
  except IOError:
    if log_exception:
      LogMessage('Unable to read '+filename)
    return ''
  return file_contents

# because reading is ~25x more expensive than getmtime, we will only read & parse if
# the getmtime is more recent than last call for this file. So this dict stores the
# a tuple, the last time read & the resulting parsed return value
CACHED_FILES = {}
def ReadAndParseSettings(filename):
  """Reads given filename and then parses the resulting key-value pairs into a dict."""
  global CACHED_FILES
  (last_read_time, settings) = CACHED_FILES.get(filename, (0, {}))
  if os.path.exists(filename):
    last_modified = os.path.getmtime(filename)
    if last_modified > last_read_time:
      setting_str = ReadFile(filename)
      settings = ParseSettings(setting_str)
      CACHED_FILES[filename] = (last_modified, settings)
    return settings

  # File does not - or at least no longer - exists; so remove the cache




                            <----SKIPPED LINES---->





  return settings_dict


def WriteFile(filename, text, log_exception=False):
  """Writes the text to the file, returning boolean indicating success.

  Args:
    filename: string of the filename to open, potentially also including the full path.
    text: the text to write
    log_exception: boolean indicating whether to log an exception if file not found.

  Returns:
    Boolean indicating whether the write was successful.
  """
  try:
    with open(filename, 'w') as content_file:
      content_file.write(text)
  except IOError:
    if log_exception:
      LogMessage('Unable to write to '+filename)
    return False
  return True


def TruncatePickledDictionaries(file, days=30):
  """Truncate the pickled flights file to have at most some number of days of history.




  Delete old data from the given pickle repository; the data must be sequentially
  pickled dictionaries where each dictionary has (at minimum) a key named 'now', which
  is the timestamp of the record.

  Args:
    file: name (potentially including path) of the pickled file
    days: maximum number of days (measured from now) to include in the file

  Returns:
    Data remaining after any truncation.
  """
  flights = UnpickleObjectFromFile(file)
  flights_passing_criteria = []
  now = datetime.datetime.now(TZ)

  # We want to be cautious, not overwriting the original file until we're sure this has
  # completed; this is why we use the tmp file
  tmp_file = file+'.tmp'
  for flight in flights:
    if HoursSinceFlight(now, flight['now']) <= days*HOURS_IN_DAY:
      PickleObjectToFile(flight, tmp_file)
      flights_passing_criteria.append(flight)

  shutil.move(tmp_file, file)

  return flights_passing_criteria


def UnpickleObjectFromFile(file):
  """Load a repository of pickled flight data into memory.

  Args:
    file: name (potentially including path) of the pickled file







  Returns:
    Return a list of all the flights, in the same sequence as written to the file.
  """













  data = []
  if os.path.exists(file):
    try:
      with open(file, 'rb') as f:
        while True:
          data.append(pickle.load(f))
    except (EOFError, pickle.UnpicklingError):
      pass

  return data


def PickleObjectToFile(data, file):
  """Append one pickled flight to the end of binary file.

  Args:
    data: data to pickle
    file: name (potentially including path) of the pickled file



  """



  try:
    with open(file, 'ab') as f:
      f.write(pickle.dumps(data))

  except IOError:
    LogMessage('Unable to append pickle ' + file)


def UpdateAircraftList(persistent_nearby_aircraft, current_nearby_aircraft, now):
  """Identifies newly seen aircraft and removes aircraft that haven't been seen recently.

  Updates persistent_nearby_aircraft as follows: flights that have been last seen more
  than PERSISTENCE_SECONDS seconds ago are removed; new flights in current_nearby_aircraft
  are added. Also identifies newly-seen aircraft and updates the last-seen timestamp of
  flights that have been seen again.

  Args:
    persistent_nearby_aircraft: dictionary where keys are flight numbers, and the values
      are the time the flight was last seen.
    current_nearby_aircraft: dictionary where keys are flight numbers, and the values are
      themselves dictionaries with key-value pairs about that flight, with at least one of
      the kv-pairs being the time the flight was seen.
    now: the timestamp of the flights in the current_nearby_aircraft.

  Returns:
    A list of newly-nearby flight numbers.
  """
  newly_nearby_flight_numbers = []
  for flight_number in current_nearby_aircraft:
    if flight_number not in persistent_nearby_aircraft:
      newly_nearby_flight_numbers.append(flight_number)
    persistent_nearby_aircraft[flight_number] = now
  flights_to_delete = []
  for flight_number in persistent_nearby_aircraft:
    if (flight_number not in current_nearby_aircraft
        and (now - persistent_nearby_aircraft[flight_number]) > PERSISTENCE_SECONDS):
      flights_to_delete.append(flight_number)
  for flight_number in flights_to_delete:
    del persistent_nearby_aircraft[flight_number]
  return newly_nearby_flight_numbers


def ScanForNewFlights(persistent_nearby_aircraft, persistent_path):
  """Determines if there are any new aircraft in the radio message.

  The radio is continuously dumping new json messages to the Raspberry pi with all the
  flights currently observed. This function picks up the latest radio json, and for
  any new nearby flights - there should generally be at most one new flight on each
  pass through - gets additional flight data from FlightAware and augments the flight
  definition with the relevant fields to keep.

  Args:
    persistent_nearby_aircraft: dictionary where keys are flight numbers, and the values
      are the time the flight was last seen.
    persistent_path: dictionary where keys are flight numbers, and the values are a
      sequential list of the location-attributes in the json file; allows for tracking
      the flight path over time.


  Returns:
    A tuple:
    - updated persistent_nearby_aircraft
    - (possibly empty) dictionary of flight attributes of the new flight upon its
      first observation.
    - the time of the radio observation if present; None if no radio dump
    - a dictionary of attributes about the dump itself (i.e.: # of flights; furthest
      observed flight, etc.)
    - persistent_path, a data structure containing past details of a flight's location
      as described in ParseDumpJson
  """
  flight_details = {}
  now = time.time()
  if SIMULATION:
    (dump_json, json_time) = DUMP_JSONS[SIMULATION_COUNTER]
  else:
    dump_json = ReadFile(DUMP_JSON_FILE, log_exception=True)

  json_desc_dict = {}
  current_nearby_aircraft = {}
  if dump_json:
    (current_nearby_aircraft,
     now,
     json_desc_dict,
     persistent_path) = ParseDumpJson(dump_json, persistent_path)

    if not SIMULATION:
      PickleObjectToFile((dump_json, now), PICKLE_DUMP_JSON_FILE)

    newly_nearby_flight_numbers = UpdateAircraftList(
        persistent_nearby_aircraft, current_nearby_aircraft, now)

    if newly_nearby_flight_numbers:

      if len(newly_nearby_flight_numbers) > 1:
        newly_nearby_flight_numbers_str = ', '.join(newly_nearby_flight_numbers)
        newly_nearby_flight_details_str = '\n'.join(
            [str(current_nearby_aircraft[f]) for f in newly_nearby_flight_numbers])
        LogMessage('Multiple newly-nearby flights: %s\n%s' % (
            newly_nearby_flight_numbers_str, newly_nearby_flight_details_str))
      flight_number = newly_nearby_flight_numbers[0]

      flight_aware_json = {}
      if SIMULATION:
        json_times = [j[1] for j in FA_JSONS]
        if json_time in json_times:
          flight_aware_json = FA_JSONS[json_times.index(json_time)][0]
      elif flight_number:
        flight_aware_json = GetFlightAwareJson(flight_number)

        if not flight_aware_json:
          LogMessage('No json returned from Flightaware for flight: %s' % flight_number)


      flight_details = {}
      if flight_aware_json:
        flight_details = ParseFlightAwareJson(flight_aware_json)

      if not SIMULATION:
        PickleObjectToFile((flight_aware_json, now), PICKLE_FA_JSON_FILE)

      # Augment FlightAware details with radio / radio-derived details
      flight_details.update(current_nearby_aircraft[flight_number])

      # Augment with the past location data
      flight_details['persistent_path'] = persistent_path[flight_number][1]

  return (
      persistent_nearby_aircraft,
      flight_details,
      now,
      json_desc_dict,
      persistent_path)


def DescribeDumpJson(parsed):
  """Generates a dictionary with descriptive attributes about the dump json file.

  Args:
    parsed: The parsed json file.




                            <----SKIPPED LINES---->




  for aircraft in parsed['aircraft']:
    simplified_aircraft = {}

    simplified_aircraft['now'] = now

    # flight_number
    flight_number = aircraft.get('flight')
    if flight_number:
      flight_number = flight_number.strip()
      if flight_number:
        simplified_aircraft['flight_number'] = flight_number

    if 'lat' in aircraft and 'lon' in aircraft:
      lat = aircraft['lat']
      lon = aircraft['lon']
      if isinstance(lat, numbers.Number) and isinstance(lon, numbers.Number):
        simplified_aircraft['lat'] = lat
        simplified_aircraft['lon'] = lon

        altitude = aircraft.get('altitude', aircraft.get('alt_baro'))
        if altitude is not None:
          simplified_aircraft['altitude'] = altitude

        speed = aircraft.get('speed', aircraft.get('gs'))
        if speed is not None:
          simplified_aircraft['speed'] = speed

        vert_rate = aircraft.get('vert_rate', aircraft.get('baro_rate'))
        if vert_rate is not None:
          simplified_aircraft['vert_rate'] = vert_rate

        if aircraft.get('squawk') is not None:
          simplified_aircraft['squawk'] = aircraft.get('squawk')

        track = aircraft.get('track')
        if isinstance(track, numbers.Number):
          min_meters = MinMetersToHome((lat, lon), track)
          simplified_aircraft['track'] = track
          simplified_aircraft['min_feet'] = min_meters * FEET_IN_METER

        if HaversineDistanceMeters(HOME, (lat, lon)) < MIN_METERS:




                            <----SKIPPED LINES---->





  return (nearby_aircraft, now, json_desc_dict, persistent_path)


def GetFlightAwareJson(flight_number):
  """Scrapes the text json message from FlightAware for a given flight number.

  Given a flight number, loads the corresponding FlightAware webpage for that flight and
  extracts the relevant script that contains all the flight details from that page.

  Args:
    flight_number: text flight number (i.e.: SWA1234)

  Returns:
    Text representation of the json message from FlightAware.
  """
  url = 'https://flightaware.com/live/flight/' + flight_number
  try:
    response = requests.get(url)
  except requests.exceptions.RequestException as e:
    LogMessage('Unable to query FA for URL due to %s: %s' % (e, url))
    return ''
  soup = bs4.BeautifulSoup(response.text, "html.parser")
  l = soup.findAll('script')
  flight_script = None
  for script in l:
    if "trackpollBootstrap" in script:
      flight_script = script
      break
  if not flight_script:
    LogMessage('Unable to find trackpollBootstrap script in page: ' + response.text)
    return ''
  first_open_curly_brace = flight_script.find('{')

  flight_json = flight_script[first_open_curly_brace:-1]
  return flight_json


def Unidecode(s):
  """Convert a special unicode characters to closest ASCII representation."""
  if s is not None:
    s = unidecode.unidecode(s)
  return s


def ParseFlightAwareJson(flight_json):
  """Strips relevant data about the flight from FlightAware feed.

  The FlightAware json has hundreds of fields about a flight, only a fraction of which
  are relevant to extract. Note that some of the fields are inconsistently populated
  (i.e.: scheduled and actual times for departure and take-off).

  Args:
    flight_json: Text representation of the FlightAware json about a single flight.





                            <----SKIPPED LINES---->




    flight['scheduled_departure_time'] = gate_departure_time.get('scheduled')
    flight['actual_departure_time'] = gate_departure_time.get('actual')

  gate_arrival_time = parsed_flight_details.get('gateArrivalTimes')
  if gate_arrival_time:
    flight['scheduled_arrival_time'] = gate_arrival_time.get('scheduled')
    flight['estimated_arrival_time'] = gate_arrival_time.get('estimated')

  landing_time = parsed_flight_details.get('landingTimes')
  if landing_time:
    flight['scheduled_landing_time'] = landing_time.get('scheduled')
    flight['estimated_landing_time'] = landing_time.get('estimated')

  airline = parsed_flight_details.get('airline')
  if airline:
    flight['airline_call_sign'] = Unidecode(airline.get('callsign'))
    flight['airline_short_name'] = Unidecode(airline.get('shortName'))
    flight['airline_full_name'] = Unidecode(airline.get('fullName'))

  if len(parsed_json['flights'].keys()) > 1:
    LogMessage('There are multiple flights in the FlightAware json: ' + parsed_json)

  return flight


def EpochDisplayTime(epoch, format_string='%Y-%m-%d %H:%M:%S.%f%z'):
  """Converts epoch in seconds to formatted time string."""
  return datetime.datetime.fromtimestamp(epoch, TZ).strftime(format_string)


def DisplayTime(flight, format_string='%Y-%m-%d %H:%M:%S.%f%z'):
  """Converts flight 'now' to formatted time string, caching results on flight."""
  cached_key = CACHED_ELEMENT_PREFIX + 'now-' + format_string
  cached_time = flight.get(cached_key)
  if cached_time:
    return cached_time

  epoch_display_time = EpochDisplayTime(flight['now'], format_string)
  flight[cached_key] = epoch_display_time
  return epoch_display_time





                            <----SKIPPED LINES---->




    remaining_seconds = flight['now'] - arrival
  else:
    remaining_seconds = None
  return remaining_seconds


def FlightMeetsDisplayCriteria(flight, configuration, display_all_hours=False, log=False):
  """Returns boolean indicating whether the screen is currently accepting new flight data.

  Based on the configuration file, determines whether the flight data should be displayed.
  Specifically, the configuration:
  - may include 'enabled' indicating whether screen should be driven at all
  - should include 'on' & 'off' parameters indicating minute (from midnight) of operation
  - should include altitude & elevation parameters indicating max values of interest

  Args:
    flight: dictionary of flight attributes.
    configuration: dictionary of configuration attributes.
    display_all_hours: a boolean indicating whether we should ignore whether the
      screen is turned off (either via the enabling, or via the hour settings)



  Returns:
    Boolean as described.
  """
  flight_altitude = flight.get('altitude', float('inf'))
  config_max_altitude = configuration['setting_max_altitude']

  flight_meets_criteria = True
  if flight_altitude > config_max_altitude:
    flight_meets_criteria = False
    if log:
      LogMessage(
          '%s not displayed because it fails altitude criteria - flight altitude: '
          '%.0f; required altitude: %.0f' % (
              DisplayFlightNumber(flight), flight_altitude, config_max_altitude))
  else:
    flight_distance = flight.get('min_feet', float('inf'))
    config_max_distance = configuration['setting_max_distance']
    if flight_distance > config_max_distance:
      flight_meets_criteria = False
      if log:
        LogMessage(
            '%s not displayed because it fails distance criteria - flight distance: '
            '%.0f; required distance: %.0f' % (
                DisplayFlightNumber(flight), flight_distance, config_max_distance))

  if not display_all_hours and flight_meets_criteria:
    flight_timestamp = flight['now']
    dt = datetime.datetime.fromtimestamp(flight_timestamp, TZ)
    minute_of_day = dt.hour * MINUTES_IN_HOUR + dt.minute
    if minute_of_day <= configuration['setting_on_time']:
      flight_meets_criteria = False
      if log:
        LogMessage(
            '%s not displayed because it occurs too early - minute_of_day: '
            '%d; setting_on_time: %d' % (
                DisplayFlightNumber(flight), minute_of_day,
                configuration['setting_on_time']))
    elif minute_of_day > configuration['setting_off_time'] + 1:
      flight_meets_criteria = False
      if log:
        LogMessage(
            '%s not displayed because it occurs too late - minute_of_day: '
            '%d; setting_off_time: %d' % (
                DisplayFlightNumber(flight), minute_of_day,
                configuration['setting_off_time']))
    elif configuration.get('setting_screen_enabled', 'off') == 'off':
      flight_meets_criteria = False
      if log:
        LogMessage(
            '%s not displayed because screen disabled' % DisplayFlightNumber(flight))

  return flight_meets_criteria


def IdentifyFlightDisplayed(flights, configuration, display_all_hours=False):
  """Finds the most recent flight in flights that meet the display criteria.

  Args:
    flights: list of flight dictionaries.
    configuration: dictionary of settings.
    display_all_hours: boolean indicating whether we should ignore the time constraints
      (i.e.: whether the screen is enabled, and its turn-on or turn-off times) in
      identifying the most recent flight. That is, if False, then this will only return
      flights that would have been displayed in the ordinarily usage, vs. if True,
      a flight irrespective of the time it would be displayed.

  Returns:
    A flight dictionary if one can be found; None otherwise.
  """
  for n in range(len(flights)-1, -1, -1):  # traverse the flights in reverse
    if FlightMeetsDisplayCriteria(
        flights[n], configuration, display_all_hours=display_all_hours):
      return n
  return None


def FlightMessageTestHarness(flights=None, display=True):
  """Simulates what flight messages might be displayed by replaying past flights."""
  if flights is None:
    flights = UnpickleObjectFromFile(PICKLE_FLIGHTS_30D)

  messages = []
  for flight in flights:

    flight_message = Screenify(CreateMessageAboutFlight(flight), False)
    if display:
      print(flight_message)
    messages.append(flight_message)

  return messages


def CreateMessageAboutFlight(flight):
  """Creates a message to describe interesting attributes about a single flight.

  Generates a multi-line description of a flight. A typical message might look like:
  UAL300 - UNITED        <- Flight number and airline
  BOEING 777-200 (TWIN)  <- Aircraft type
  SFO-HNL HONOLULU       <- Origin & destination
  DEP 02:08 ER REM 5:14  <- Time details: departure time; early / late / ontime; remaining




                            <----SKIPPED LINES---->





  for unused_n in range(SPLITFLAP_LINE_COUNT-len(lines)):
    lines.append('')
  lines = [
      border_character + line.ljust(SPLITFLAP_CHARS_PER_LINE).upper() + border_character
      for line in lines]

  if not splitflap:
    lines.insert(0, divider)
    lines.append(divider)

  return append_character.join(lines)


def FlightInsightsTestHarness(
    flights=None,
    display=True,
    flight_insights_enabled_string='all'):
  """Simulates what insightful messages might be displayed by replaying past flights."""
  if flights is None:
    flights = UnpickleObjectFromFile(PICKLE_FLIGHTS_30D)

  distribution = {}
  messages = []
  for (n, flight) in enumerate(flights):

    flight_message = CreateMessageAboutFlight(flight)
    if display:
      print('='*25)
      print(Screenify(flight_message, False))
    messages.append(flight_message)

    insights = CreateFlightInsights(
        flights[:n+1], flight_insights_enabled_string, distribution)

    FlightInsightNextFlight(flights[:n+1])

    if display:
      for insight in insights:
        print(Screenify(insight, False))
    messages.extend(insights)

  return messages


def FlightInsightLastSeen(flights, days_ago=2):
  """Generates string indicating when flight was last seen.

  Generates text of the following form.
  - KAL214 was last seen 2d0h ago

  Args:
    flights: the list of the raw data from which the insights will be generated,
        where the flights are listed in order of observation - i.e.: flights[0] was the
        earliest seen, and flights[-1] is the most recent flight for which we are
        attempting to generate an insight.
    days_ago: the minimum time difference for which a message should be generated -
        i.e.: many flights are daily, and so we are not necessarily interested to see
        about every daily flight that it was seen yesterday. However, more infrequent
        flights might be of interest.

  Returns:
    Printable string message; if no message or insights to generate, then an empty string.
  """
  message = ''
  this_flight = flights[-1]
  this_flight_number = DisplayFlightNumber(this_flight)
  this_timestamp = flights[-1]['now']
  last_seen = [f for f in flights[:-1] if DisplayFlightNumber(f) == this_flight_number]
  if last_seen and 'flight_number' in this_flight:
    last_timestamp = last_seen[-1]['now']
    if this_timestamp - last_timestamp > days_ago*SECONDS_IN_DAY:
      message = '%s was last seen %s ago' % (
          this_flight_number, SecondsToDdHh(this_timestamp - last_timestamp))
  return message


def FlightInsightDifferentAircraft(flights, percent_size_difference=0.1):
  """Generates string indicating changes in aircraft for the most recent flight.

  Generates text of the following form for the "focus" flight in the data.
  - Last time ASA1964 was seen on Mar 16, it was with a much larger plane (Airbus A320
    (twin-jet) @ 123ft vs. Airbus A319 (twin-jet) @ 111ft)
  - Last time ASA743 was seen on Mar 19, it was with a different type of airpline
    (Boeing 737-900 (twin-jet) vs. Boeing 737-800 (twin-jet))

  Args:
    flights: the list of the raw data from which the insights will be generated,
        where the flights are listed in order of observation - i.e.: flights[0] was the
        earliest seen, and flights[-1] is the most recent flight for which we are
        attempting to generate an insight.
    percent_size_difference: the minimum size (i.e.: length) difference for the insight
        to warrant including the size details.

  Returns:
    Printable string message; if no message or insights to generate, then an empty string.
  """
  message = ''
  this_flight = flights[-1]
  this_flight_number = DisplayFlightNumber(this_flight)
  last_seen = [f for f in flights[:-1] if DisplayFlightNumber(f) == this_flight_number]

  # Last time this same flight flew a materially different type of aircraft
  if last_seen and 'flight_number' in this_flight:
    last_flight = last_seen[-1]

    last_aircraft = last_flight.get('aircraft_type_friendly')
    last_aircraft_length = aircraft_length.get(last_aircraft, 0)

    this_aircraft = this_flight.get('aircraft_type_friendly')
    this_aircraft_length = aircraft_length.get(this_aircraft, 0)

    this_likely_commercial_flight = (
        this_flight.get('origin_iata') and this_flight.get('destination_iata'))
    if this_likely_commercial_flight and this_aircraft and not this_aircraft_length:
      LogMessage('%s used in a flight with defined origin & destination but yet is '
                 'missing length details' % this_aircraft, file=LOGFILE)

    likely_same_commercial_flight = (
        last_flight.get('origin_iata') == this_flight.get('origin_iata') and
        last_flight.get('destination_iata') == this_flight.get('destination_iata') and
        last_flight.get('airline_call_sign') == this_flight.get('airline_call_sign'))

    this_aircraft_bigger = False
    last_aircraft_bigger = False
    if (likely_same_commercial_flight and
        this_aircraft_length > last_aircraft_length * (1 + percent_size_difference)):
      this_aircraft_bigger = True
      comparative_text = 'larger'
    elif (likely_same_commercial_flight and
          last_aircraft_length > this_aircraft_length * (1 + percent_size_difference)):
      last_aircraft_bigger = True
      comparative_text = 'smaller'

    last_flight_time_string = DisplayTime(last_flight, '%b %-d')
    if this_aircraft and last_aircraft:
      if this_aircraft_bigger or last_aircraft_bigger:




                            <----SKIPPED LINES---->




                       RemoveParentheticals(last_aircraft),
                       last_aircraft_length*FEET_IN_METER))
      elif last_aircraft and this_aircraft and last_aircraft != this_aircraft:
        message = (
            '%s used a different aircraft today compared with last, on %s (%s vs. %s)' % (
                this_flight_number, last_flight_time_string, this_aircraft, last_aircraft))

  return message


def FlightInsightNthFlight(flights, hours=1, min_multiple_flights=2):
  """Generates string about seeing many flights to the same destination in a short period.

  Generates text of the following form for the "focus" flight in the data.
  - ASA1337 was the 4th flight to PHX in the last 53 minutes, served by Alaska Airlines,
    American Airlines, Southwest and United
  - SWA3102 was the 2nd flight to SAN in the last 25 minutes, both with Southwest

  Args:
    flights: the list of the raw data from which the insights will be generated,
        where the flights are listed in order of observation - i.e.: flights[0] was the
        earliest seen, and flights[-1] is the most recent flight for which we are
        attempting to generate an insight.
    hours: the time horizon over which to look for flights with the same destination.
    min_multiple_flights: the minimum number of flights to that same destination to
        warrant generating an insight.

  Returns:
    Printable string message; if no message or insights to generate, then an empty string.
  """
  message = ''
  this_flight = flights[-1]
  this_flight_number = this_flight.get('flight_number', 'This')
  this_destination = this_flight.get('destination_iata', '')
  this_airline = this_flight.get('airline_short_name', KEY_NOT_PRESENT_STRING)
  if not this_airline:
    this_airline = KEY_NOT_PRESENT_STRING # in case airline was stored as, say, ''
  this_timestamp = this_flight['now']
  if this_destination and this_destination not in ['SFO', 'LAX']:
    similar_flights = [f for f in flights[:-1] if
                       this_timestamp - f['now'] < SECONDS_IN_HOUR*hours and
                       this_destination == f.get('destination_iata', '')]
    similar_flights_count = len(similar_flights) + 1  # +1 for this_flight
    similar_flights_airlines = list(
        {f.get('airline_short_name', KEY_NOT_PRESENT_STRING) for f in similar_flights})





                            <----SKIPPED LINES---->




  return message


def FlightInsightSuperlativeAttribute(
    flights,
    key,
    label,
    units,
    absolute_list,
    insight_min=True,
    insight_max=True,
    hours=HOURS_IN_DAY):
  """Generates string about a numeric attribute of the flight being an extreme value.

  Generates text of the following form for the "focus" flight in the data.
  - N5286C has the slowest groundspeed (113mph vs. 163mph) in last 24 hours
  - CKS828 has the highest altitude (40000ft vs. 16575ft) in last 24 hours

  Args:
    flights: the list of the raw data from which the insights will be generated,
        where the flights are listed in order of observation - i.e.: flights[0] was the
        earliest seen, and flights[-1] is the most recent flight for which we are
        attempting to generate an insight.
    key: the key of the attribute of interest - i.e.: 'speed'.
    label: the human-readable string that should be displayed in the message - i.e.:
        'groundspeed'.
    units: the string units that should be used to label the value of the key - i.e.:
        'MPH'.
    absolute_list: a 2-tuple of strings that is used to label the min and the max - i.e.:
        ('lowest', 'highest'), or ('slowest', 'fastest').
    insight_min: boolean indicating whether to generate an insight about the min value.
    insight_max: boolean indicating whether to generate an insight about the max value.
    hours: the time horizon over which to look for superlative flights.

  Returns:
    Printable string message; if no message or insights to generate, then an empty string.
  """
  message = ''
  this_flight = flights[-1]
  this_flight_number = this_flight.get('flight_number', 'The last flight')
  first_timestamp = flights[0]['now']
  last_timestamp = flights[-1]['now']
  included_seconds = last_timestamp - first_timestamp

  if included_seconds > SECONDS_IN_HOUR * hours:
    relevant_flights = [
        f for f in flights[:-1]
        if last_timestamp - f['now'] < SECONDS_IN_HOUR * hours]
    value_min = min(
        [f.get(key) for f in relevant_flights if isinstance(f.get(key), numbers.Number)])




                            <----SKIPPED LINES---->




      else:
        superlative = False

      if superlative:
        message = '%s has the %s %s (%d%s vs. %d%s) in last %d hours' % (
            this_flight_number, absolute_string, label,
            this_value, units, other_value, units, hours)

  return message


def FlightInsightNextFlight(flights):
  """Generates string about estimated wait until next flight.

  Generates text of the following form for the "focus" flight in the data.
  - Last flight at 2:53a; avg wait is 1h58m & median is 42m, but could be as long as
    8h43m, based on last 20 days

  Args:
    flights: the list of the raw data from which the insights will be generated,
        where the flights are listed in order of observation - i.e.: flights[0] was the
        earliest seen, and flights[-1] is the most recent flight for which we are
        attempting to generate an insight.

  Returns:
    Printable string message; if no message because not enough history, then an
    empty string.
  """
  msg = ''
  # m = min of day of this flight
  # find minute of day of prior flights st
  # -- that flight not seen in last 12 hrs
  # -- that min of day >= this
  this_flight = flights[-1]
  this_hour = int(DisplayTime(this_flight, '%-H'))
  this_minute = int(DisplayTime(this_flight, '%-M'))
  this_date = DisplayTime(this_flight, '%x')

  # Flights that we've already seen in the last few hours we do not expect to see
  # again for another few hours, so let's exclude them from the calculation
  exclude_flights_hours = 12
  flight_numbers_seen_in_last_n_hours = [
      f['flight_number'] for f in flights




                            <----SKIPPED LINES---->




    group_label,
    value_label,
    filter_function=lambda this, other: True,
    min_days=1,
    lookback_days=30,
    min_this_group_size=0,
    min_comparison_group_size=0,
    min_group_qty=0,
    percentile_low=float('-inf'),
    percentile_high=float('inf')):
  """Generates a string about extreme values of groups of flights.

  Generates text of the following form for the "focus" flight in the data.
  - flight SIA31 (n=7) has a delay frequency in the 95th %tile, with 100% of flights
    delayed an average of 6m over the last 4d1h
  - flight UAL300 (n=5) has a delay time in the 1st %tile, with an average delay of 0m
  over the last 4d5h

  Args:
    flights: the list of the raw data from which the insights will be generated,
        where the flights are listed in order of observation - i.e.: flights[0] was the
        earliest seen, and flights[-1] is the most recent flight for which we are
        attempting to generate an insight.
    group_function: function that, when called with a flight, returns the grouping key.
        That is, for example, group_function(flight) = 'B739'
    value_function: function that, when called with a list of flights, returns the
        value to be used for the comparison to identify min / max. Typically, the count,
        but could also be a sum, standard deviation, etc. - for perhaps the greatest
        range in flight altitude. If the group does not have a valid value and so
        should be excluded from comparison - i.e.: average delay of a group of flights
        which did not have a calculable_delay on any flight, this function should
        return None.
    value_string_function: function that, when called with the two parameters flights
        and value, returns a string (inclusive of units and label) that should be
        displayed to describe the quantity. For instance, if value_function returns
        seconds, value_string_function could convert that to a string '3h5m'. Or if
        value_function returns an altitude range, value_string_function could return
        a string 'altitude range of 900ft (1100ft - 2000ft)'.
    group_label: string to identify the group type - i.e.: 'aircraft' or 'flight' in
        the examples above.
    value_label: string to identify the value - i.e.: 'flights' in the examples above,
        but might also be i.e.: longest *delay*, or other quantity descriptor.
    filter_function: an optional function that, when called with the most recent flight
        and another flight filter_function(flights[-1], flight[n]), returns a value
        interpreted as a boolean indicating whether flight n should be included in
        determining the percentile.
    min_days: the minimum amount of history required to start generating insights
        about delays.
    lookback_days: the maximum amount of history which will be considered in generating
        insights about delays.
    min_this_group_size: even if this group has, say, the maximum average delay, if its
        a group of size 1, that is not necessarily very interesting. This sets the
        minimum group size for the focus flight.
    min_comparison_group_size: similarly, comparing the focus group to groups of size
        one does not necessarily produce a meaningful comparison; this sets to minimum
        size for the other groups.
    min_group_qty: when generating a percentile, if there are only 3 or 4 groups among
        which to generate a percentile (i.e.: only a handful of destinations have been
        seen so far, etc.) then it is not necessarily very interesting to generate a
        message; this sets the minimum quantity of groups necessary (including the
        focus group) to generate a message.
    percentile_low: number [0, 100] inclusive that indicates the percentile that the focus
        flight group must equal or be less than for the focus group to trigger an insight.
    percentile_high: number [0, 100] inclusive that indicates the percentile that the
        focus flight group must equal or be greater than for the focus group to trigger an
        insight.

  Returns:
    Printable string message; if no message or insights to generate, then an empty string.
  """
  debug = False
  message = ''
  this_flight = flights[-1]
  first_timestamp = flights[0]['now']
  last_timestamp = this_flight['now']
  included_seconds = last_timestamp - first_timestamp

  if (included_seconds > SECONDS_IN_DAY * min_days and
      group_function(this_flight) != KEY_NOT_PRESENT_STRING):

    relevant_flights = [
        f for f in flights if
        last_timestamp - f['now'] < SECONDS_IN_DAY * lookback_days and
        filter_function(this_flight, f)]

    grouped_flights = {}




                            <----SKIPPED LINES---->




    value_label,
    absolute_list,
    min_days=1,
    lookback_days=30,
    min_this_group_size=0,
    min_comparison_group_size=0,
    insight_min=True,
    insight_max=True):
  """Generates a string about extreme values of groups of flights.

  Generates text of the following form for the "focus" flight in the data.
  - aircraft B739 (n=7) is tied with B738 and A303 for the most flights at 7 flights
    over the last 3d7h amongst aircraft with a least 5 flights
  - aircraft B739 (n=7) is tied with 17 others for the most flights at 7 flights over
    the last 3d7h amongst aircraft with a least 5 flights
  - flight UAL1075 (n=12) has the most flights with 12 flights; the next most flights
    is 11 flights over the last 7d5h

  Args:
    flights: the list of the raw data from which the insights will be generated,
        where the flights are listed in order of observation - i.e.: flights[0] was the
        earliest seen, and flights[-1] is the most recent flight for which we are
        attempting to generate an insight.
    group_function: function that, when called with a flight, returns the grouping key.
        That is, for example, group_function(flight) = 'B739'
    value_function: function that, when called with a list of flights, returns the
        value to be used for the comparison to identify min / max. Typically, the count,
        but could also be a sum, standard deviation, etc. - for perhaps the greatest
        range in flight altitude. If the group does not have a valid value and so
        should be excluded from comparison - i.e.: average delay of a group of flights
        which did not have a calculable_delay on any flight, this function should
        return None.
    value_string_function: function that, when called with the two parameters flights
        and value, returns a string (inclusive of units and label) that should be
        displayed to describe the quantity. For instance, if value_function returns
        seconds, value_string_function could convert that to a string '3h5m'. Or if
        value_function returns an altitude range, value_string_function could return
        a string 'altitude range of 900ft (1100ft - 2000ft)'.
    group_label: string to identify the group type - i.e.: 'aircraft' or 'flight' in
        the examples above.
    value_label: string to identify the value - i.e.: 'flights' in the examples above,
        but might also be i.e.: longest *delay*, or other quantity descriptor.
    absolute_list: a 2-tuple of strings that is used to label the min and the max - i.e.:
        ('most', 'least'), or ('lowest average', 'highest average').
    min_days: the minimum amount of history required to start generating insights
        about delays.
    lookback_days: the maximum amount of history which will be considered in generating
        insights about delays.
    min_this_group_size: even if this group has, say, the maximum average delay, if its
        a group of size 1, that is not necessarily very interesting. This sets the
        minimum group size for the focus flight.
    min_comparison_group_size: similarly, comparing the focus group to groups of size
        one does not necessarily produce a meaningful comparison; this sets to minimum
        size for the other groups.
    insight_min: boolean indicating whether to possibly generate insight based on the
        occurrence of the min value.
    insight_max: boolean indicating whether to possibly generate insight based on the
        occurrence of the max value.

  Returns:
    Printable string message; if no message or insights to generate, then an empty string.
  """
  message = ''
  first_timestamp = flights[0]['now']
  last_timestamp = flights[-1]['now']
  included_seconds = last_timestamp - first_timestamp

  if included_seconds > SECONDS_IN_DAY * min_days:

    relevant_flights = [
        f for f in flights if
        last_timestamp - f['now'] < SECONDS_IN_DAY * lookback_days]

    grouped_flights = {}
    for flight in relevant_flights:
      group = group_function(flight)
      grouping = grouped_flights.get(group, [])
      grouping.append(flight)




                            <----SKIPPED LINES---->




  if calculable_delay_seconds:
    percent_delay = delay_count / len(calculable_delay_seconds)
  return percent_delay


def FlightInsightFirstInstance(
    flights,
    key,
    label,
    days=7,
    additional_descriptor_fcn=''):
  """Generates string indicating the flight has the first instance of a particular key.

  Generates text of the following form for the "focus" flight in the data.
  - N311CG is the first time aircraft GLF6 (Gulfstream Aerospace Gulfstream G650
    (twin-jet)) has been seen since at least 7d5h ago
  - PCM8679 is the first time airline Westair Industries has been seen since 9d0h ago

  Args:
    flights: the list of the raw data from which the insights will be generated,
        where the flights are listed in order of observation - i.e.: flights[0] was the
        earliest seen, and flights[-1] is the most recent flight for which we are
        attempting to generate an insight.
    key: the key of the attribute of interest - i.e.: 'destination_iata'.
    label: the human-readable string that should be displayed in the message - i.e.:
        'destination'.
    days: the minimum time of interest for an insight - i.e.: we probably see LAX every
        hour, but we are only interested in particular attributes that have not been
        seen for at least some number of days. Note, however, that the code will go back
        even further to find the last time that attribute was observed, or if never
        observed, indicating "at least".
    additional_descriptor_fcn: a function that, when passed a flight, returns an
        additional parenthetical notation to include about the attribute or flight
        observed - such as expanding the IATA airport code to its full name, etc.

  Returns:
    Printable string message; if no message or insights to generate, then an empty string.
  """
  message = ''
  this_flight = flights[-1]
  this_flight_number = DisplayFlightNumber(this_flight)
  first_timestamp = flights[0]['now']
  last_timestamp = flights[-1]['now']
  included_seconds = last_timestamp - first_timestamp

  if included_seconds > SECONDS_IN_DAY * days:
    this_instance = this_flight.get(key)
    matching = [f for f in flights[:-1] if f.get(key) == this_instance]

    last_potential_observation_sec = included_seconds
    if matching:
      last_potential_observation_sec = last_timestamp - matching[-1]['now']

    if this_instance and last_potential_observation_sec > SECONDS_IN_DAY * days:




                            <----SKIPPED LINES---->




            last_potential_observation_string)

  return message


def FlightInsightSuperlativeVertrate(flights, hours=HOURS_IN_DAY):
  """Generates string about the climb rate of the flight being an extreme value.

  Generates text of the following form for the "focus" flight in the data.
  - UAL631   has the fastest ascent rate (5248fpm, 64fpm faster than next fastest) in
    last 24 hours
  - CKS1820 has the fastest descent rate (-1152fpm, -1088fpm faster than next fastest)
    in last 24 hours

  While this is conceptually similar to the more generic FlightInsightSuperlativeVertrate
  function, vert_rate - because it can be either positive or negative, with different
  signs requiring different labeling and comparisons - it needs its own special handling.

  Args:
    flights: the list of the raw data from which the insights will be generated,
        where the flights are listed in order of observation - i.e.: flights[0] was the
        earliest seen, and flights[-1] is the most recent flight for which we are
        attempting to generate an insight.
    hours: the time horizon over which to look for superlative flights.

  Returns:
    Printable string message; if no message or insights to generate, then an empty string.
  """
  message = ''
  this_flight = flights[-1]
  this_flight_number = this_flight.get('flight_number')
  first_timestamp = flights[0]['now']
  last_timestamp = flights[-1]['now']
  sufficient_data = (last_timestamp - first_timestamp) > SECONDS_IN_HOUR * hours
  pinf = float('inf')
  ninf = float('-inf')

  if sufficient_data:
    relevant_flights = [
        f for f in flights[:-1]
        if last_timestamp - f['now'] < SECONDS_IN_HOUR * hours]

    def AscentRate(f, default):




                            <----SKIPPED LINES---->






def FlightInsightDelays(
    flights,
    min_days=1,
    lookback_days=30,
    min_late_percentage=0.75,
    min_this_delay_minutes=0,
    min_average_delay_minutes=0):
  """Generates string about the delays this flight has seen in the past.

  Only if this flight has a caclculable delay itself, this will generate text of the
  following form for the "focus" flight in the data.
  - This 8m delay is the longest UAL1175 has seen in the last 9 days (avg delay is 4m);
    overall stats: 1 early; 9 late; 10 total
  - With todays delay of 7m, UAL1175 is delayed 88% of the time in the last 8 days for
    avg delay of 4m; overall stats: 1 early; 8 late; 9 total

  Args:
    flights: the list of the raw data from which the insights will be generated,
        where the flights are listed in order of observation - i.e.: flights[0] was the
        earliest seen, and flights[-1] is the most recent flight for which we are
        attempting to generate an insight.
    min_days: the minimum amount of history required to start generating insights
        about delays.
    lookback_days: the maximum amount of history which will be considered in generating
        insights about delays.
    min_late_percentage: flights that are not very frequently delayed are not
        necessarily very interesting to generate insights about; this specifies the
        minimum percentage the flight must be late to generate a message that focuses
        on the on-time percentage.
    min_this_delay_minutes: a delay of 1 minute is not necessarily interesting; this
        specifies the minimum delay time this instance of the flight must be late to
        generate a message that focuses on this flight's delay.
    min_average_delay_minutes: an average delay of only 1 minute, even if it happens
        every day, is not necessarily very interesting; this specifies the minimum
        average delay time to generate either type of delay message.

  Returns:
    Printable string message; if no message or insights to generate, then an empty string.
  """
  message = ''
  this_flight = flights[-1]
  this_flight_number = this_flight.get('flight_number', '')
  first_timestamp = flights[0]['now']
  last_timestamp = flights[-1]['now']
  included_seconds = last_timestamp - first_timestamp

  if (included_seconds > SECONDS_IN_DAY * min_days
      and DisplayDepartureTimes(this_flight)['calculable_delay']):
    this_delay_seconds = DisplayDepartureTimes(this_flight)['delay_seconds']
    relevant_flights = [
        f for f in flights if
        last_timestamp - f['now'] < SECONDS_IN_DAY * lookback_days and
        this_flight_number == f.get('flight_number', '')]

    if (




                            <----SKIPPED LINES---->




        # it's just been delayed frequently!
        message = (
            'With today''s delay of %s, %s is delayed %d%% of the time in the last %d '
            'days for avg delay of %s; overall stats: %s' % (
                SecondsToHhMm(this_delay_seconds),
                this_flight_number,
                int(100 * late_percentage),
                days_history,
                SecondsToHhMm(delay_late_avg_sec),
                overall_stats_text))
  return message


def FlightInsights(flights):
  """Identifies all the insight messages about the most recently seen flight.

  Generates a possibly-empty list of messages about the flight.

  Args:
    flights: List of all flights where the last flight in the list is the focus flight
        for which we are trying to identify something interesting.

  Returns:
    List of 2-tuples, where the first element in the tuple is a flag indicating the type
    of insight message, and the second selement is the printable strings (with embedded
    new line characters) for something interesting about the flight; if there isn't
    anything interesting, returns an empty list.
  """
  messages = []

  def AppendMessageType(message_type, message):
    if message:
      messages.append((message_type, message))

  # This flight number was last seen x days ago
  AppendMessageType(FLAG_INSIGHT_LAST_SEEN, FlightInsightLastSeen(flights, days_ago=2))

  # Yesterday this same flight flew a materially different type of aircraft
  AppendMessageType(
      FLAG_INSIGHT_DIFF_AIRCRAFT,
      FlightInsightDifferentAircraft(flights, percent_size_difference=0.1))




                            <----SKIPPED LINES---->




    for message_tuple in insight_messages:
      if message_tuple[0] == t:
        naked_messages.append(message_tuple[1])
        this_flight_insights.append(t)
        break

  # Save the distribution displayed for this flight so we needn't regen it in future
  flights[-1]['insight_types'] = this_flight_insights

  return naked_messages


def MessageboardHistogramsTestHarness(
    flights=None,
    hist_type='day_of_month',
    hist_history='30d',
    max_screens='all',
    summary=False):
  """Test harness to generate messageboard histograms."""
  if flights is None:
    flights = UnpickleObjectFromFile(PICKLE_FLIGHTS_30D)

  messages = MessageboardHistograms(
      flights, hist_type, hist_history, max_screens, summary)

  for message in messages:
    print(message)

  return messages


def FlightCriteriaHistogramPng(
    flights,
    max_distance_feet,
    max_altitude_feet,
    max_days,
    filename=HOURLY_IMAGE_FILE,
    last_max_distance_feet=None,
    last_max_altitude_feet=None):
  """Saves as a png file the histogram of the hourly flight data for the given filters.

  Generates a png histogram of the count of flights by hour that meet the specified
  criteria: max altitude, max distance, and within the last number of days. Also
  optionally generates as a separate data series in same chart a histogram with a
  different max altitude and distance. Saves this histogram to disk.

  Args:
    flights: list of the flights.
    max_distance_feet: max distance for which to include flights in the histogram.
    max_altitude_feet: max altitude for which to include flights in the histogram.
    max_days: maximum number of days as described.
    filename: file into which to save the csv.
    last_max_distance_feet: if provided, along with last_max_altitude_feet, generates
        a second data series with different criteria for distance and altitude, for
        which the histogram data will be plotted alongside the first series.
    last_max_altitude_feet: see above.
  """
  (values, keys, unused_filtered_data) = GenerateHistogramData(
      flights,
      HourString,
      HOURS,
      hours=max_days*HOURS_IN_DAY,
      max_distance_feet=max_distance_feet,
      max_altitude_feet=max_altitude_feet,
      normalize_factor=max_days,
      exhaustive=True)

  comparison = last_max_distance_feet is not None and last_max_altitude_feet is not None
  if comparison:
    (last_values, unused_last_keys, unused_filtered_data) = GenerateHistogramData(
        flights,
        HourString,
        HOURS,
        hours=max_days*HOURS_IN_DAY,
        max_distance_feet=last_max_distance_feet,




                            <----SKIPPED LINES---->




  matplotlib.pyplot.close()


def GenerateHistogramData(
    data,
    keyfunction,
    sort_type,
    truncate=float('inf'),
    hours=float('inf'),
    max_distance_feet=float('inf'),
    max_altitude_feet=float('inf'),
    normalize_factor=0,
    exhaustive=False):
  """Generates sorted data for a histogram from a description of the flights.

  Given an iterable describing the flights, this function generates the label (or key),
  and the frequency (or value) from which a histogram can be rendered.

  Args:
    data: the iterable of the raw data from which the histogram will be generated;
        each element of the iterable is a dictionary, that contains at least the key
        'now', and depending on other parameters, also potentially
        'min_feet' amongst others.
    keyfunction: the function that determines how the key or label of the histogram
        should be generated; it is called for each element of the data iterable. For
        instance, to simply generate a histogram on the attribute 'heading',
        keyfunction would be lambda a: a['heading'].
    sort_type: determines how the keys (and the corresponding values) are sorted:
        'key': the keys are sorted by a simple comparison operator between them, which
            sorts strings alphabetically and numbers numerically.
        'value': the keys are sorted by a comparison between the values, which means
            that more frequency-occurring keys are listed first.
        list: if instead of the strings a list is passed, the keys are then sorted in
            the sequence enumerated in the list. This is useful for, say, ensuring that
            the days of the week (Tues, Wed, Thur, ...) are listed in sequence. Keys
            that are generated by keyfunction but that are not in the given list are
            sorted last (and then amongst those, alphabetically).
    truncate: integer indicating the maximum number of keys to return; if set to 0, or if
        set to a value larger than the number of keys, no truncation occurs. But if set
        to a value less than the number of keys, then the keys with the lowest frequency
        are combined into one key named OTHER_STRING so that the number of keys
        in the resulting histogram (together with OTHER_STRING) is equal to truncate.
    hours: integer indicating the number of hours of history to include. Flights with a
        calcd_display_time more than this many hours in the past are excluded from the
        histogram generation. Note that this is timezone aware, so that if the histogram
        data is generated on a machine with a different timezone than that that recorded
        the original data, the correct number of hours is still honored.
    max_distance_feet: number indicating the geo fence outside of which flights should
        be ignored for the purposes of including the flight data in the histogram.
    max_altitude_feet: number indicating the maximum altitude outside of which flights
        should be ignored for the purposes of including the flight data in the histogram.
    normalize_factor: divisor to apply to all the values, so that we can easily
        renormalize the histogram to display on a percentage or daily basis; if zero,
        no renormalization is applied.
    exhaustive: boolean only relevant if sort_type is a list, in which case, this ensures
        that the returned set of keys (and matching values) contains all the elements in
        the list, including potentially those with a frequency of zero, within the
        restrictions of truncate.

  Returns:
    2-tuple of lists cut and sorted as indicated by parameters above:
    - list of values (or frequency) of the histogram elements
    - list of keys (or labels) of the histogram elements
  """
  histogram_dict = {}
  filtered_data = []

  def IfNoneReturnInf(f, key):
    value = f.get(key)
    if not value:
      value = float('inf')
    return value

  # get timezone & now so that we can generate a timestamp for comparison just once
  if hours:
    now = datetime.datetime.now(TZ)
  for element in data:
    if (




                            <----SKIPPED LINES---->




    keys = []
  return (values, keys, filtered_data)


def SortByValues(values, keys, ignore_sort_at_end_strings=False):
  """Sorts the list of values in descending sequence, applying same resorting to keys.

  Given a list of keys and values representing a histogram, returns two new lists that
  are sorted so that the values occur in descending sequence and the keys are moved
  around in the same way. This allows the printing of a histogram with the largest
  keys listed first - i.e.: top five airlines.

  Keys identified by SORT_AT_END_STRINGS - such as, perhaps, 'Other' - will optionally
  be placed at the end of the sequence. And where values are identical, the secondary
  sort is based on the keys.

  Args:
    values: list of values for the histogram to be used as the primary sort key.
    keys: list of keys for the histogram that will be moved in the same way as the values.
    ignore_sort_at_end_strings: boolean indicating whether specially-defined keys will be
        sorted at the end.

  Returns:
    2-tuple of (values, keys) lists sorted as described above
  """
  if ignore_sort_at_end_strings:
    sort_at_end_strings = []
  else:
    sort_at_end_strings = SORT_AT_END_STRINGS

  return SortZipped(
      values, keys, True,
      lambda a: (
          False, False, a[1]) if a[1] in sort_at_end_strings else (True, a[0], a[1]))


def SortByKeys(values, keys, ignore_sort_at_end_strings=False):
  """Sorts the list of keys in ascending sequence, applying same resorting to values.

  Given a list of keys and values representing a histogram, returns two new lists that
  are sorted so that the keys occur in ascending alpha sequence and the values are moved
  around in the same way. This allows the printing of a histogram with the first keys
  alphabetically listed first - i.e.: 7am, 8am, 9am.

  Keys identified by SORT_AT_END_STRINGS - such as, perhaps, 'Other' - will optionally
  be placed at the end of the sequence.

  Args:
    values: list of values for the histogram that will be moved in the same way as the
        keys.
    keys: list of keys for the histogram to be used as the primary sort key.
    ignore_sort_at_end_strings: boolean indicating whether specially-defined keys will be
        sorted at the end.

  Returns:
    2-tuple of (values, keys) lists sorted as described above
  """
  if ignore_sort_at_end_strings:
    sort_at_end_strings = []
  else:
    sort_at_end_strings = SORT_AT_END_STRINGS

  return SortZipped(
      values, keys, False,
      lambda a: (True, a[1]) if a[1] in sort_at_end_strings else (False, a[1]))


def SortByDefinedList(values, keys, sort_sequence):
  """Sorts the keys in user-enumerated sequence, applying same resorting to values.

  Given a list of keys and values representing a histogram, returns two new lists that
  are sorted so that the keys occur in the specific sequence identified in the list
  sort_sequence, while the values are moved around in the same way. This allows the
  printing of a histogram with the keys occurring in a canonical order - i.e.: Tuesday,
  Wednesday, Thursday. Keys present in keys but not existing in sort_sequence are then
  sorted at the end, but amongst them, sorted based on the value.

  Args:
    values: list of values for the histogram that will be moved in the same way as the
        keys.
    keys: list of keys for the histogram to be used as the primary sort key.
    sort_sequence: list - which need not be exhaustive - of the keys in their desired
        order.

  Returns:
    2-tuple of (values, keys) lists sorted as described above
  """
  return SortZipped(
      values, keys, False,
      lambda a: (
          False, sort_sequence.index(a[1])) if a[1] in sort_sequence else (True, a[0]))


def SortZipped(x, y, reverse, key):
  """Sorts lists x & y via the function defined in key.

  Applies the same reordering to the two lists x and y, where the reordering is given
  by the function defined in the key applied to the tuple (x[n], y[n]). That is, suppose
  - x = [3, 2, 1]
  - y = ['b', 'c', 'a']
  Then the sort to both lists is done based on how the key is applied to the tuples:
  - [(3, 'b'), (2, 'c'), (1, 'a')]

  If key = lambda a: a[0], then the sort is done based on 3, 2, 1, so the sorted lists are
  - x = [1, 2, 3]
  - y = ['a', 'c', 'b']

  If key = lambda a: a[1], then the sort is done based on ['b', 'c', 'a'], so the sorted
  lists are
  - x = [1, 3, 2]
  - y = ['a', 'b', 'c']

  Args:
    x: First list
    y: Second list
    reverse: Boolean indicating whether the sort should be ascending (True) or descending
        (False)
    key: function applied to the 2-tuple constructed by taking the corresponding values
        of the lists x & y, used to generate the key on which the sort is applied

  Returns:
    2-tuple of (x, y) lists sorted as described above
  """
  zipped_xy = zip(x, y)
  sorted_xy = sorted(zipped_xy, reverse=reverse, key=key)
  # unzip
  (x, y) = list(zip(*sorted_xy))
  return (x, y)


def CreateSingleHistogramChart(
    data,
    keyfunction,
    sort_type,
    title,
    position=None,
    truncate=0,
    hours=float('inf'),
    max_distance_feet=float('inf'),
    max_altitude_feet=float('inf'),
    normalize_factor=0,
    exhaustive=False,
    figsize_inches=(9, 6)):
  """Creates matplotlib.pyplot of histogram that can then be saved or printed.

  Args:
    data: the iterable (i.e.: list) of flight details, where each element in the list is
        a dictionary of the flight attributes.
    keyfunction: a function that when applied to a single flight (i.e.:
        keyfunction(data[0]) returns the key to be used for the histogram.
    data: the iterable of the raw data from which the histogram will be generated;
        each element of the iterable is a dictionary, that contains at least the key
        'now', and depending on other parameters, also potentially
        'min_feet' amongst others.
    keyfunction: the function that determines how the key or label of the histogram
        should be generated; it is called for each element of the data iterable. For
        instance, to simply generate a histogram on the attribute 'heading',
        keyfunction would be lambda a: a['heading'].
    title: the "base" title to include on the histogram; it will additionally be
        augmented with the details about the date range.
    position: Either a 3-digit integer or an iterable of three separate integers
        describing the position of the subplot. If the three integers are nrows, ncols,
        and index in order, the subplot will take the index position on a grid with nrows
        rows and ncols columns. index starts at 1 in the upper left corner and increases
        to the right.
    sort_type: determines how the keys (and the corresponding values) are sorted:
        'key': the keys are sorted by a simple comparison operator between them, which
            sorts strings alphabetically and numbers numerically.
        'value': the keys are sorted by a comparison between the values, which means
            that more frequency-occurring keys are listed first.
        list: if instead of the strings a list is passed, the keys are then sorted in
            the sequence enumerated in the list. This is useful for, say, ensuring that
            the days of the week (Tues, Wed, Thur, ...) are listed in sequence. Keys
            that are generated by keyfunction but that are not in the given list are
            sorted last (and then amongst those, alphabetically).
    truncate: integer indicating the maximum number of keys to return; if set to 0, or if
        set to a value larger than the number of keys, no truncation occurs. But if set
        to a value less than the number of keys, then the keys with the lowest frequency
        are combined into one key named OTHER_STRING so that the number of keys
        in the resulting histogram (together with OTHER_STRING) is equal to truncate.
    hours: integer indicating the number of hours of history to include. Flights with a
        calcd_display_time more than this many hours in the past are excluded from the
        histogram generation. Note that this is timezone aware, so that if the histogram
        data is generated on a machine with a different timezone than that that recorded
        the original data, the correct number of hours is still honored.
    max_distance_feet: number indicating the geo fence outside of which flights should
        be ignored for the purposes of including the flight data in the histogram.
    max_altitude_feet: number indicating the maximum altitude outside of which flights
        should be ignored for the purposes of including the flight data in the histogram.
    normalize_factor: divisor to apply to all the values, so that we can easily
        renormalize the histogram to display on a percentage or daily basis; if zero,
        no renormalization is applied.
    exhaustive: boolean only relevant if sort_type is a list, in which case, this ensures
        that the returned set of keys (and matching values) contains all the elements in
        the list, including potentially those with a frequency of zero, within the
        restrictions of truncate.
    figsize_inches: a 2-tuple of width, height indicating the size of the histogram.
  """
  (values, keys, filtered_data) = GenerateHistogramData(
      data,
      keyfunction,
      sort_type,
      truncate=truncate,
      hours=hours,
      max_distance_feet=max_distance_feet,
      max_altitude_feet=max_altitude_feet,
      normalize_factor=normalize_factor,
      exhaustive=exhaustive)
  if position:
    matplotlib.pyplot.subplot(*position)
  matplotlib.pyplot.figure(figsize=figsize_inches)
  values_coordinates = numpy.arange(len(keys))
  matplotlib.pyplot.bar(values_coordinates, values)

  earliest_flight_time = int(filtered_data[0]['now'])
  last_flight_time = int(filtered_data[-1]['now'])




                            <----SKIPPED LINES---->






def HistogramSettingsHours(how_much_history):
  """Extracts the desired history (in hours) from the histogram configuration string.

  Args:
    how_much_history: string from the histogram config file.

  Returns:
    Number of hours of history to include in the histogram.
  """
  if how_much_history == 'today':
    hours = HoursSinceMidnight()
  elif how_much_history == '24h':
    hours = HOURS_IN_DAY
  elif how_much_history == '7d':
    hours = 7 * HOURS_IN_DAY
  elif how_much_history == '30d':
    hours = 30 * HOURS_IN_DAY
  else:
    LogMessage(
        'Histogram form has invalid value for how_much_history: %s' % how_much_history)
    hours = 7 * HOURS_IN_DAY
  return hours


def HistogramSettingsScreens(max_screens):
  """Extracts the desired number of text screens from the histogram configuration string.

  Args:
    max_screens: string from the histogram config file.

  Returns:
    Number of maximum number of screens to display for a splitflap histogram.
  """
  if max_screens == '_1':
    screen_limit = 1
  elif max_screens == '_2':
    screen_limit = 2
  elif max_screens == '_5':
    screen_limit = 5
  elif max_screens == 'all':
    screen_limit = 0  # no limit on screens
  else:
    LogMessage('Histogram form has invalid value for max_screens: %s' % max_screens)
    screen_limit = 1
  return screen_limit


def HistogramSettingsKeySortTitle(which, hours, max_altitude=45000):
  """Provides the arguments necessary to generate a histogram from the config string.

  The same parameters are used to generate either a splitflap text or web-rendered
  histogram in terms of the histogram title, the keyfunction, and how to sort the keys.
  For a given histogram name (based on the names defined in the histogram config file),
  this provides those parameters.

  Args:
    which: string from the histogram config file indicating the histogram to provide
        settings for.
    hours: how many hours of histogram data have been requested.
    max_altitude: indicates the maximum altitude that should be included on the
        altitude labels.

  Returns:
    A 4-tuple of the parameters used by either CreateSingleHistogramChart or
    MessageboardHistogram, of the keyfunction, sort, title, and hours.
  """
  def DivideAndFormat(dividend, divisor):
    if dividend is None:
      return KEY_NOT_PRESENT_STRING
    if isinstance(dividend, numbers.Number):
      return '%2d' % round(dividend / divisor)
    return dividend[:2]

  if which == 'destination':
    key = lambda k: k.get('destination_iata', KEY_NOT_PRESENT_STRING)
    sort = 'value'
    title = 'Destination'
  elif which == 'origin':
    key = lambda k: k.get('origin_iata', KEY_NOT_PRESENT_STRING)
    sort = 'value'
    title = 'Origin'




                            <----SKIPPED LINES---->




    sort = ['%2d'%x for x in range(0, round((MIN_METERS*FEET_IN_METER)/100)+1)]
    title = 'Min Dist (100ft)'
  elif which == 'day_of_week':
    key = lambda k: DisplayTime(k, '%a')
    sort = DAYS_OF_WEEK
    title = 'Day of Week'
    # if less than one week, as requested; if more than one week, in full week multiples
    hours_in_week = 7 * HOURS_IN_DAY
    weeks = hours / hours_in_week
    if weeks > 1:
      hours = hours_in_week * int(hours / hours_in_week)
  elif which == 'day_of_month':
    key = lambda k: DisplayTime(k, '%-d').rjust(2)
    today_day = datetime.datetime.now(TZ).day
    days = list(range(today_day, 0, -1))  # today down to the first of the month
    days.extend(range(31, today_day, -1))  # 31st of the month down to day after today
    days = [str(d).rjust(2) for d in days]
    sort = days
    title = 'Day of Month'
  else:
    LogMessage(
        'Histogram form has invalid value for which_histograms: %s' % which)
    return HistogramSettingsKeySortTitle(
        'destination', hours, max_altitude=max_altitude)

  return (key, sort, title, hours)


def ImageHistograms(
    flights,
    which_histograms,
    how_much_history,
    filename_prefix=HISTOGRAM_IMAGE_PREFIX,
    filename_suffix=HISTOGRAM_IMAGE_SUFFIX):
  """Generates multiple split histogram images.

  Args:
    flights: the iterable of the raw data from which the histogram will be generated;
        each element of the iterable is a dictionary, that contains at least the key
        'now', and depending on other parameters, also potentially
        'min_feet' amongst others.
    which_histograms: string paramater indicating which histogram(s) to generate, which
        can be either the special string 'all', or a string linked to a specific
        histogram.
    how_much_history: string parameter taking a value among ['today', '24h', '7d', '30d].
    filename_prefix: this string indicates the file path and name prefix for the images
        that are created. File names are created in the form [prefix]name.[suffix], i.e.:
        if the prefix is histogram_ and the suffix is png, then the file name might be
        histogram_aircraft.png.
    filename_suffix: see above; also interpreted by savefig to generate the correct
        format.

  Returns:
    List of the names of the histograms generated.
  """
  hours = HistogramSettingsHours(how_much_history)

  histograms_to_generate = []
  if which_histograms in ['destination', 'all']:
    histograms_to_generate.append({'generate': 'destination'})
  if which_histograms in ['origin', 'all']:
    histograms_to_generate.append({'generate': 'origin'})
  if which_histograms in ['hour', 'all']:
    histograms_to_generate.append({'generate': 'hour'})
  if which_histograms in ['airline', 'all']:
    histograms_to_generate.append({'generate': 'airline', 'truncate': int(TRUNCATE/2)})
  if which_histograms in ['aircraft', 'all']:
    histograms_to_generate.append({'generate': 'aircraft'})
  if which_histograms in ['altitude', 'all']:
    histograms_to_generate.append({'generate': 'altitude', 'exhaustive': True})
  if which_histograms in ['bearing', 'all']:




                            <----SKIPPED LINES---->




        hours=hours,
        exhaustive=histogram.get('exhaustive', False))
    filename = filename_prefix + histogram['generate'] + '.' + filename_suffix
    matplotlib.pyplot.savefig(filename)
    matplotlib.pyplot.close()

  histograms_generated = [h['generate'] for h in histograms_to_generate]
  return histograms_generated


def MessageboardHistograms(
    flights,
    which_histograms,
    how_much_history,
    max_screens,
    data_summary):
  """Generates multiple split flap screen histograms.

  Args:
    flights: the iterable of the raw data from which the histogram will be generated;
        each element of the iterable is a dictionary, that contains at least the key
        'now', and depending on other parameters, also potentially
        'min_feet' amongst others.
    which_histograms: string paramater indicating which histogram(s) to generate, which
        can be either the special string 'all', or a string linked to a specific
        histogram.
    how_much_history: string parameter taking a value among ['today', '24h', '7d', '30d].
    max_screens: string parameter taking a value among ['_1', '_2', '_5', or 'all'].
    data_summary: parameter that evaluates to a boolean indicating whether the data
        summary screen in the histogram should be displayed.

  Returns:
    Returns a list of printable strings (with embedded new line characters) representing
    the histogram, for each screen in the histogram.
  """
  messages = []

  hours = HistogramSettingsHours(how_much_history)
  screen_limit = HistogramSettingsScreens(max_screens)

  histograms_to_generate = []
  if which_histograms in ['destination', 'all']:
    histograms_to_generate.append({
        'generate': 'destination',
        'suppress_percent_sign': True,
        'columns': 3})
  if which_histograms in ['origin', 'all']:
    histograms_to_generate.append({
        'generate': 'origin',
        'suppress_percent_sign': True,




                            <----SKIPPED LINES---->





  return messages


def MessageboardHistogram(
    data,
    keyfunction,
    sort_type,
    title,
    screen_limit=1,
    columns=2,
    column_divider=' ',
    data_summary=False,
    hours=0,
    suppress_percent_sign=False,
    absolute=False):
  """Generates a text representation of one histogram that can be rendered on the display.

  Args:
    data: the iterable of the raw data from which the histogram will be generated;
        each element of the iterable is a dictionary, that contains at least the key
        'now', and depending on other parameters, also potentially
        'min_feet' amongst others.
    keyfunction: the function that determines how the key or label of the histogram
        should be generated; it is called for each element of the data iterable. For
        instance, to simply generate a histogram on the attribute 'heading',
        keyfunction would be lambda a: a['heading'].
    sort_type: determines how the keys (and the corresponding values) are sorted; see
        GenerateHistogramData docstring for details
    title: string title, potentially truncated to fit, to be displayed for the histogram
    screen_limit: maximum number of screens to be displayed for the histogram; a value
        of zero is interpreted to mean no limit on screens.
    columns: number of columns of data to be displayed for the histogram; note that the
        keys of the histogram may need to be truncated in length to fit the display
        as more columns are squeezed into the space
    column_divider: string for the character(s) to be used to divide the columns
    data_summary: boolean indicating whether to augment the title with a second header
        line about the data presented in the histogram
    hours: integer indicating the oldest data to be included in the histogram
    suppress_percent_sign: boolean indicating whether to suppress the percent sign
        in the data (but to add it to the title) to reduce the amount of string
        truncation potentially necessary for display of the keys
    absolute: boolean indicating whether to values should be presented as percentage or
        totals; if True, suppress_percent_sign is irrelevant.

  Returns:
    Returns a list of printable strings (with embedded new line characters) representing
    the histogram.
  """
  title_lines = 1
  if data_summary:
    title_lines += 1
  available_entries_per_screen = (SPLITFLAP_LINE_COUNT - title_lines) *  columns
  available_entries_total = available_entries_per_screen * screen_limit
  (values, keys, unused_filtered_data) = GenerateHistogramData(
      data, keyfunction, sort_type, truncate=available_entries_total, hours=hours)

  screen_count = math.ceil(len(keys) / available_entries_per_screen)

  column_width = int(
      (SPLITFLAP_CHARS_PER_LINE - len(column_divider)*(columns - 1)) / columns)
  leftover_space = SPLITFLAP_CHARS_PER_LINE - (
      column_width*columns + len(column_divider)*(columns - 1))
  extra_divider_chars = math.floor(leftover_space / (columns - 1))




                            <----SKIPPED LINES---->




  for flight in flights:
    if 'altitude' in flight and 'min_feet' in flight:
      this_altitude = flight['altitude']
      this_distance = flight['min_feet']
      hour = HourString(flight)
      for altitude in [a for a in altitudes if a >= this_altitude]:
        for distance in [d for d in distances if d >= this_distance]:
          d[(altitude, distance, hour)] = d.get((altitude, distance, hour), 0) + 1
  for altitude in altitudes:
    for distance in distances:
      line_elements = [str(altitude), str(distance)]
      for hour in HOURS:
        line_elements.append(str(d.get((altitude, distance, hour), 0)))
      line = ','.join(line_elements)
      lines.append(line)
  try:
    with open(filename, 'w') as f:
      for line in lines:
        f.write(line+'\n')
  except IOError:
    LogMessage('Unable to write hourly histogram data file ' + filename)


def SaveFlightsToCSV(flights=None, filename='flights.csv'):
  """Saves all the attributes about the flight to a CSV, including on-the-fly attributes.

  Args:
    flights: dictionary of flight attributes; if not provided, loaded from
      PICKLE_FLIGHTS_30D.
    filename: name of desired csv file; if not provided, defaults to flights.csv.
  """
  if not flights:
    flights = UnpickleObjectFromFile(PICKLE_FLIGHTS_30D)

  print('='*80)
  print('Number of flights to save to %s: %d' % (filename, len(flights)))

  # list of functions in 2-tuple, where second element is a function that generates
  # something about the flight, and the first element is the name to give that value
  # when extended into the flight definition
  functions = [
      ('display_flight_number', DisplayFlightNumber),
      ('display_airline', DisplayAirline),
      ('display_aircraft', DisplayAircraft),
      ('display_origin_iata', DisplayOriginIata),
      ('display_destination_iata', DisplayDestinationIata),
      ('display_origin_friendly', DisplayOriginFriendly),
      ('display_destination_friendly', DisplayDestinationFriendly),
      ('display_origin_destination_pair', DisplayOriginDestinationPair),
      ('display_seconds_remaining', DisplaySecondsRemaining),
      ('now_datetime', DisplayTime),
      ('now_date', lambda flight: DisplayTime(flight, '%x')),
      ('now_time', lambda flight: DisplayTime(flight, '%X'))]




                            <----SKIPPED LINES---->




      'altitude_degrees_00s', 'altitude_degrees_10s', 'altitude_degrees_20s',
      'ground_distance_feet_00s', 'ground_distance_feet_10s', 'ground_distance_feet_20s',
      'crow_distance_feet_00s', 'crow_distance_feet_10s', 'crow_distance_feet_20s']
  for key in all_keys:
    if key not in keys_logical_order:
      keys_logical_order.append(key)

  f = open(filename, 'w')
  f.write(','.join(keys_logical_order)+'\n')
  for flight in flights:
    f.write(','.join(['"'+str(flight.get(k))+'"' for k in keys_logical_order])+'\n')
  f.close()


def SimulationSetup():
  """Updates global variable file names and loads in JSON data for simulation runs."""
  global SIMULATION
  SIMULATION = True

  global DUMP_JSONS
  DUMP_JSONS = UnpickleObjectFromFile(PICKLE_DUMP_JSON_FILE)

  global FA_JSONS
  FA_JSONS = UnpickleObjectFromFile(PICKLE_FA_JSON_FILE)

  global ALL_MESSAGE_FILE
  ALL_MESSAGE_FILE = SIMULATION_PREFIX + ALL_MESSAGE_FILE
  if os.path.exists(ALL_MESSAGE_FILE):
    os.remove(ALL_MESSAGE_FILE)

  global LOGFILE
  LOGFILE = SIMULATION_PREFIX + LOGFILE
  if os.path.exists(LOGFILE):
    os.remove(LOGFILE)

  global ROLLING_LOGFILE
  ROLLING_LOGFILE = SIMULATION_PREFIX + ROLLING_LOGFILE
  if os.path.exists(ROLLING_LOGFILE):
    os.remove(ROLLING_LOGFILE)


  global ROLLING_MESSAGE_FILE
  ROLLING_MESSAGE_FILE = SIMULATION_PREFIX + ROLLING_MESSAGE_FILE
  if os.path.exists(ROLLING_MESSAGE_FILE):
    os.remove(ROLLING_MESSAGE_FILE)

  global PICKLE_FLIGHTS_30D
  PICKLE_FLIGHTS_30D = SIMULATION_PREFIX + PICKLE_FLIGHTS_30D
  if os.path.exists(PICKLE_FLIGHTS_30D):
    os.remove(PICKLE_FLIGHTS_30D)

  global PICKLE_FLIGHTS_ARCHIVE
  PICKLE_FLIGHTS_ARCHIVE = SIMULATION_PREFIX + PICKLE_FLIGHTS_ARCHIVE
  if os.path.exists(PICKLE_FLIGHTS_ARCHIVE):
    os.remove(PICKLE_FLIGHTS_ARCHIVE)

  if os.path.exists(ARDUINO_FILE):
    os.remove(ARDUINO_FILE)


def SimulationEnd(message_queue, flights):
  """Clears message buffer, exercises histograms, and other misc test & status code.

  Args:
    message_queue: List of flight messages that have not yet been printed.
    flights: List of flights dictionaries.
  """
  if flights:
    histogram = {
        'type': 'both',
        'histogram':'all',
        'histogram_history':'30d',
        'histogram_max_screens': '_2',
        'histogram_data_summary': 'on'}
    histogram_messages = TriggerHistograms(flights, histogram)
    histogram_messages = [(FLAG_MSG_HISTOGRAM, m) for m in histogram_messages]
    message_queue.extend(histogram_messages)

    while message_queue:
      ManageMessageQueue(message_queue, 0, {'setting_delay': 0})
    SaveFlightsByAltitudeDistanceCSV(flights)
    SaveFlightsToCSV(flights)

    # repickle to a new .pk with full track info
  file_parts = PICKLE_FLIGHTS_30D.split('.')
  new_pickle_file = '.'.join([file_parts[0] + '_full_path', file_parts[1]])
  if os.path.exists(new_pickle_file):
    os.remove(new_pickle_file)
  for flight in flights:
    PickleObjectToFile(flight, new_pickle_file)

  print('Simulation complete after %s dump json messages processed' % len(DUMP_JSONS))


def DumpJsonChanges():
  """Identifies if sequential dump json files changes, for simulation optimization.

  If we are logging the radio output faster than it is updating, then there will be
  sequential log files in the json list that are identical; we only need to process the
  first of these, and can ignore subsequent ones, without any change of output in the
  simulation results. This function identifies whether the current active json changed
  from the prior one.

  Returns:
    Boolean - True if different (and processing needed), False if identical
  """
  if SIMULATION_COUNTER == 0:
    return True
  (this_json, unused_now) = DUMP_JSONS[SIMULATION_COUNTER]
  (last_json, unused_now) = DUMP_JSONS[SIMULATION_COUNTER - 1]
  return this_json != last_json





































def UpdateArduinoFile(flights, json_desc_dict):
  """Saves a file that can be read by arduino.py with necessary flight attributes.

  The independently-running arduino python modules need basic information about the
  flight and radio in order to send useful information to be displayed by the digital
  alphanumeric displays.

  Args:
    flights: list of the flight dictionaries.
    json_desc_dict: dict with details about the radio range and similar radio details.

  Returns:
    String that is also written to disk.
  """
  # Start with radio_range_miles & radio_range_flights
  d = json_desc_dict

  if flights:




                            <----SKIPPED LINES---->




    d['now'] = flight['now']

    requested_time = time.time()
    requested_elapsed_seconds = requested_time - flight['now']
    updated_loc = ClosestKnownLocation(flight, requested_elapsed_seconds)
    actual_elapsed_seconds = requested_elapsed_seconds - updated_loc[1]
    d['speed'] = updated_loc[0]['speed']
    d['lat'] = updated_loc[0]['lat']
    d['lon'] = updated_loc[0]['lon']
    d['track'] = updated_loc[0]['track']
    d['altitude'] = updated_loc[0]['altitude']
    d['vertrate'] = updated_loc[0]['vertrate']
    d['flight_loc_now'] = flight['now'] + actual_elapsed_seconds

  today = datetime.datetime.now(TZ).strftime('%x')
  flight_count_today = len([1 for f in flights if DisplayTime(f, '%x') == today])

  d['flight_count_today'] = flight_count_today

  settings_string = BuildSettings(d)

  if not SIMULATION:
    if os.path.exists(ARDUINO_FILE):
      existing_data = ReadAndParseSettings(ARDUINO_FILE)
      if d != existing_data:
        WriteFile(ARDUINO_FILE, settings_string)
    else:
      WriteFile(ARDUINO_FILE, settings_string)

  return settings_string


def PublishMessage(
    s,
    subscription_id='12fd73cd-75ef-4cae-bbbf-29b2678692c1',
    key='c5f62d44-e30d-4c43-a43e-d4f65f4eb399',
    secret='b00aeb24-72f3-467c-aad2-82ba5e5266ca',
    timeout=3):
  """Publishes a text string to a Vestaboard.

  The message is pushed to the vestaboard splitflap display by way of its web services;
  see https://docs.vestaboard.com/introduction for more details.

  Args:
    s: String to publish.
    subscription_id: string subscription id from Vestaboard.
    key: string key from Vestaboard.
    secret: string secret from Vestaboard.
    timeout: Max duration in seconds that we should wait to establish a connection.
  """

  # See https://docs.vestaboard.com/characters: any chars needing to be replaced
  special_characters = ((u'\u00b0', '{62}'),)  # degree symbol '°'

  for special_character in special_characters:
    s = s.replace(*(special_character))
  curl = pycurl.Curl()

  # See https://stackoverflow.com/questions/31826814/curl-post-request-into-pycurl-code
  # Set URL value
  curl.setopt(
      pycurl.URL,
      'https://platform.vestaboard.com/subscriptions/%s/message' % subscription_id)
  curl.setopt(pycurl.HTTPHEADER, [
      'X-Vestaboard-Api-Key:%s' % key, 'X-Vestaboard-Api-Secret:%s' % secret])
  curl.setopt(pycurl.TIMEOUT_MS, timeout*1000)
  curl.setopt(pycurl.POST, 1)

  curl.setopt(pycurl.WRITEFUNCTION, lambda x: None) # to keep stdout clean

  # preparing body the way pycurl.READDATA wants it
  body_as_dict = {'text': s}
  body_as_json_string = json.dumps(body_as_dict) # dict to json
  body_as_file_object = io.StringIO(body_as_json_string)

  # prepare and send. See also: pycurl.READFUNCTION to pass function instead
  curl.setopt(pycurl.READDATA, body_as_file_object)
  curl.setopt(pycurl.POSTFIELDSIZE, len(body_as_json_string))
  try:
    curl.perform()
  except pycurl.error as e:
    LogMessage('curl.perform() failed with message %s' % e)


  else:
    # you may want to check HTTP response code, e.g.
    status_code = curl.getinfo(pycurl.RESPONSE_CODE)
    if status_code != 200:
      LogMessage('Server returned HTTP status code %d for message %s' % (status_code, s))



  curl.close()




def ManageMessageQueue(message_queue, next_message_time, configuration):
  """Check time & if appropriate, display next message from queue.

  Args:
    message_queue: FIFO list of message tuples of (message type, message string).
    next_message_time: epoch at which next message should be displayed
    configuration: dictionary of configuration attributes.

  Returns:
    Next_message_time, potentially updated if a message has been displayed, or unchanged
    if no message was displayed.
  """
  if message_queue and (time.time() >= next_message_time or SIMULATION):

    if SIMULATION:  # drain the queue because the messages come so fast
      messages_to_display = list(message_queue)
      # passed by reference, so clear it out since we drained it to the display
      del message_queue[:]
    else:  # display only one message, being mindful of the display timing
      messages_to_display = [message_queue.pop(0)]

    for message in messages_to_display:
      message_text = message[1]
      if isinstance(message_text, str):
        message_text = textwrap.wrap(message_text, width=SPLITFLAP_CHARS_PER_LINE)
      display_message = Screenify(message_text, False)
      LogMessage(display_message, file=ALL_MESSAGE_FILE)
      MaintainRollingWebLog(display_message, 25)
      if not SIMULATION:
        splitflap_message = Screenify(message_text, True)
        PublishMessage(splitflap_message)

    next_message_time = time.time() + configuration['setting_delay']
  return next_message_time


def BootstrapInsightList(filename_tuple=(PICKLE_FLIGHTS_30D, PICKLE_FLIGHTS_ARCHIVE)):
  """(Re)populate flight pickle files with flight insight distributions.

  The set of insights generated for each flight is created at the time the flight was
  first identified, and saved on the flight pickle. This saving allows the current
  running distribution to be recalculated very quickly, but it means that as code
  enabling new insights gets added, those historical distributions may not necessarily
  be considered correct.

  They are "correct" in the sense that that new insight was not available at the time
  that older flight was seen, but it is not correct in the sense that, because this new
  insight is starting out with an incidence in the historical data of zero, this
  new insight may be reported more frequently than desired until it "catches up".

  So this method replays the flight history with the latest insight code, regenerating
  the insight distribution for each flight.
  """
  for filename in filename_tuple:




    print('Bootstrapping %s' % filename)
    configuration = ReadAndParseSettings(CONFIG_FILE)
    flights = []
    tmp_filename = filename + 'tmp'

    if os.path.exists(tmp_filename):
      os.remove(tmp_filename)

    if os.path.exists(filename):
      mtime = os.path.getmtime(filename)
      flights = UnpickleObjectFromFile(filename)
      for (n, flight) in enumerate(flights):
        if n/25 == int(n/25):
          print(' - %d' % n)
        CreateFlightInsights(flights[:n+1], configuration.get('insights', 'hide'), {})
        PickleObjectToFile(flight, tmp_filename)

      if mtime == os.path.getmtime(filename):
        shutil.move(tmp_filename, filename)
      else:
        print('Failed to bootstrap %s: file changed while in process' % filename)



def ResetLogs(config):
  """Clears the non-scrolling logs if reset_logs in config."""
  if 'reset_logs' in config:
    LogMessage('Reset logs')

    if os.path.exists(STDERR_FILE):
      os.remove(STDERR_FILE)
      LogMessage('', STDERR_FILE)
    if os.path.exists(BACKUP_FILE):
      os.remove(BACKUP_FILE)
      open(BACKUP_FILE, 'a').close()
    if os.path.exists(SERVICE_VERIFICATION_FILE):
      os.remove(SERVICE_VERIFICATION_FILE)
      open(SERVICE_VERIFICATION_FILE, 'a').close()
    config.pop('reset_logs')
    config = BuildSettings(config)
    WriteFile(CONFIG_FILE, config)
  return config


def main():
  """Traffic cop between incoming radio flight messages, configuration, and messageboard.

  This is the main logic, checking for new flights, augmenting the radio signal with
  additional web-scraped data, and generating messages in a form presentable to the
  messageboard.
  """

  already_running_id = CheckIfProcessRunning()
  if already_running_id:

    os.kill(already_running_id, signal.SIGKILL)

  LogMessage('Starting up')
  if '-s' in sys.argv:
    global SIMULATION_COUNTER
    SimulationSetup()

  # Redirect any errors to a log file instead of the screen, and add a datestamp
  if not SIMULATION:
    sys.stderr = open(STDERR_FILE, 'a')
    sys.stdout = open(STDERR_FILE, 'a')
    LogMessage('', STDERR_FILE)






  configuration = ReadAndParseSettings(CONFIG_FILE)
  last_distance = configuration.get('distance')
  last_altitude = configuration.get('altitude')
  startup_time = time.time()
  json_desc_dict = {}

  # If we're displaying just a single insight message, we want it to be something
  # unique, to the extent possible; this dict holds a count of the diff types of messages
  # displayed so far
  insight_message_distribution = {}

  flights = []
  if os.path.exists(PICKLE_FLIGHTS_30D):
    flights = TruncatePickledDictionaries(PICKLE_FLIGHTS_30D)

    # Clear the loaded flight of any cached data since code fixes may change
    # the values for some of those cached elements
    for flight in flights:
      for key in list(flight.keys()):
        if key[:len(CACHED_ELEMENT_PREFIX)] == CACHED_ELEMENT_PREFIX:
          flight.pop(key)

    # bootstrap the flight insights distribution
    for (n, flight) in enumerate(flights):
      if 'insight_types' in flight:
        distribution = flight['insight_types']
        for key in distribution:
          insight_message_distribution[key] = (
              insight_message_distribution.get(key, 0) + 1)
      # # - but if it doesn't exist, we still have a path forward
      else:
        CreateFlightInsights(
            flights[:n+1],
            configuration.get('insights', 'hide'),
            insight_message_distribution)

  # used in simulation to print the hour of simulation once per simulated hour
  prev_simulated_hour = ''

  persistent_nearby_aircraft = {} # key = flight number; value = last seen
  persistent_path = {}
  histogram = {}

  # Next up to print is element 0; this is a list of tuples:
  # Element#1: flag indicating the type of message that this is
  # Element#2: the message itself
  message_queue = []
  next_message_time = time.time()

  # We repeat the loop every x seconds; this ensures that if the processing time is long,
  # we don't wait another x seconds after processing completes
  next_loop_time = time.time() + LOOP_DELAY_SECONDS

  # These files are read only if the version on disk has been modified more recently
  # than the last time it was read
  last_dump_json_timestamp = 0



  LogMessage('Finishing initialization; starting radio polling loop')
  while not SIMULATION or SIMULATION_COUNTER < len(DUMP_JSONS):

    new_configuration = ReadAndParseSettings(CONFIG_FILE)

    n_c = new_configuration
    c = configuration
    if (n_c.get('setting_max_distance') != c.get('setting_max_distance') or
        n_c.get('setting_max_altitude') != c.get('setting_max_altitude')):
      last_distance = configuration.get('setting_max_distance')
      last_altitude = configuration.get('setting_max_altitude')
      FlightCriteriaHistogramPng(
          flights,
          new_configuration['setting_max_distance'],
          new_configuration['setting_max_altitude'],
          7,
          last_max_distance_feet=last_distance,
          last_max_altitude_feet=last_altitude)

    if (n_c.get('setting_max_distance') != c.get('setting_max_distance') or
        n_c.get('setting_max_altitude') != c.get('setting_max_altitude') or
        n_c.get('setting_off_time') != c.get('setting_off_time') or
        n_c.get('setting_on_time') != c.get('setting_on_time')):
      next_flight_message = FlightInsightNextFlight(flights)
      if next_flight_message:
        message_queue.append((FLAG_MSG_INTERESTING, next_flight_message))

    configuration = new_configuration

    ResetLogs(configuration)  # clear the logs if requested

    # if this is a SIMULATION, then process every diff dump. But if it isn't a simulation,
    # then only read & do related processing for the next dump if the last-modified
    # timestamp indicates the file has been updated since it was last read.
    tmp_timestamp = 0
    if not SIMULATION:


      tmp_timestamp = os.path.getmtime(DUMP_JSON_FILE)
    if (SIMULATION and DumpJsonChanges()) or (
        not SIMULATION and tmp_timestamp > last_dump_json_timestamp):

      last_dump_json_timestamp = tmp_timestamp

      # a command might request info about flight to be (re)displayed, irrespective of
      # whether the screen is on; if so, let's put that message at the front of the message
      # queue, and delete any subsequent messages in queue because presumably the button
      # was pushed either a) when the screen was off (so no messages in queue), or b)
      # because the screen was on, but the last flight details got lost after other screens
      if os.path.exists(LAST_FLIGHT_FILE):
        messageboard_flight_index = IdentifyFlightDisplayed(
            flights, configuration, display_all_hours=True)
        if messageboard_flight_index is not None:
          message_queue = [m for m in message_queue if m[0] != FLAG_MSG_INTERESTING]
          flight_message = CreateMessageAboutFlight(flights[messageboard_flight_index])
          message_queue = [FLAG_MSG_FLIGHT, flight_message]
          next_message_time = time.time()
        os.remove(LAST_FLIGHT_FILE)

      (persistent_nearby_aircraft,
       flight, now,
       json_desc_dict,
       persistent_path) = ScanForNewFlights(persistent_nearby_aircraft, persistent_path)




      if flight:
        flights.append(flight)
        UpdateArduinoFile(
            flights, json_desc_dict)


        flight_meets_display_criteria = FlightMeetsDisplayCriteria(
            flight, configuration, log=True)
        if flight_meets_display_criteria:
          flight_message = (FLAG_MSG_FLIGHT, CreateMessageAboutFlight(flight))

          # display the next message about this flight now!
          next_message_time = time.time()
          message_queue.insert(0, flight_message)
          # and delete any queued insight messages about other flights that have
          # not yet displayed, since a newer flight has taken precedence
          messages_to_delete = [m for m in message_queue if m[0] == FLAG_MSG_INTERESTING]
          if messages_to_delete:
            LogMessage(
                'Deleting messages from queue due to new-found plane: %s' %
                messages_to_delete)
          message_queue = [m for m in message_queue if m[0] != FLAG_MSG_INTERESTING]

          # Though we also manage the message queue outside this conditional as well,
          # because it can take a half second to generate the flight insights, this allows
          # this message to start displaying on the board immediately, so it's up there
          # when it's most relevant
          next_message_time = ManageMessageQueue(
              message_queue, next_message_time, configuration)

          insight_messages = CreateFlightInsights(
              flights, configuration.get('insights'), insight_message_distribution)
          if configuration.get('next_flight', 'off') == 'on':
            next_flight_text = FlightInsightNextFlight(flights)
            if next_flight_text:
              insight_messages.insert(0, next_flight_text)

          insight_messages = [(FLAG_MSG_INTERESTING, m) for m in insight_messages]

          for insight_message in insight_messages:
            message_queue.insert(0, insight_message)

        else:  # flight didn't meet display criteria
          flight['insight_types'] = []

        PickleObjectToFile(flight, PICKLE_FLIGHTS_30D)
        PickleObjectToFile(flight, PICKLE_FLIGHTS_ARCHIVE)

      else:
        UpdateArduinoFile(flights, json_desc_dict)



    if SIMULATION:
      if now:
        simulated_hour = EpochDisplayTime(now, '%Y-%m-%d %H:00%z')
      if simulated_hour != prev_simulated_hour:
        print(simulated_hour)
        prev_simulated_hour = simulated_hour

    histogram = ReadAndParseSettings(HISTOGRAM_CONFIG_FILE)
    if os.path.exists(HISTOGRAM_CONFIG_FILE):
      os.remove(HISTOGRAM_CONFIG_FILE)

    # We also need to make sure there are flights on which to generate a histogram! Why
    # might there not be any flights? Primarily during a simulation, if there's a
    # lingering histogram file at the time of history restart.


    if histogram and flights:
      histogram_messages = TriggerHistograms(flights, histogram)
      histogram_messages = [(FLAG_MSG_HISTOGRAM, m) for m in histogram_messages]
      message_queue.extend(histogram_messages)

    # check time & if appropriate, display next message from queue
    next_message_time = ManageMessageQueue(message_queue, next_message_time, configuration)

    # if we've been running a long time, and everything else is quiet, reboot
    running_hours = (time.time() - startup_time) / SECONDS_IN_HOUR
    if (
        running_hours >= HOURS_IN_DAY and
        not message_queue and
        not json_desc_dict.get('radio_range_flights')):
      LogMessage('About to reboot after running for %.2f hours' % running_hours)
      os.system('sudo reboot')
      return  # exit execution

    if not SIMULATION:
      time.sleep(max(0, next_loop_time - time.time()))
      next_loop_time = time.time() + LOOP_DELAY_SECONDS
    else:
      SIMULATION_COUNTER += 1



  if SIMULATION:
    SimulationEnd(message_queue, flights)

































































































































if __name__ == "__main__":


  if '-i' in sys.argv:
    BootstrapInsightList()
  else:
    main()