#!/usr/bin/python3

import datetime

import json
import random
import math
import numbers
import os
import pickle

import signal
import shutil

import textwrap
import time

import bs4
import dateutil.relativedelta
import numpy
import matplotlib
import matplotlib.pyplot
import psutil

import pytz
import requests
import tzlocal










HOME_LAT = 37.64406
HOME_LON = -122.43463

HOME_ALT = 29  #altitude in meters
RADIUS = 6371.0e3  # radius of earth in meters
HOME = (HOME_LAT, HOME_LON) # lat / lon tuple of antenna
MIN_METERS = 1000 # only planes within this distance will be detailed
# planes not seen within MIN_METERS in 5 seconds will be dropped from the nearby list
PERSISTENCE_SECONDS = 5
FEET_IN_METER = 3.28084

METERS_PER_SECOND_IN_KNOTS = 0.514444





TRUNCATE = 50  # max number of keys to include in a histogram image file




DUMP_JSON_FILE = '/run/dump1090-mutability/aircraft.json'



MESSAGEBOARD_PATH = '/home/pi/splitflap/'








PICKLEFILE_30D = 'flights_30d.pk' #pickled list of up to about 30d of flights


PICKLEFILE_ARCHIVE = 'flights_archive.pk' #pickled list of all flights





LOGFILE = 'log.txt'




WEBSERVER_PATH = '/var/www/html/'

WEBSERVER_IMAGE_FOLDER = 'images/'
HOURLY_HISTOGRAM_FILE = 'histogram.txt'
CONFIG_FILE = 'settings.txt'
HOURLY_IMAGE_FILE = 'hours.png'
HOURLY_DATA_FILE = 'hours.txt'
ALL_MESSAGE_FILE = 'all_messages.txt'  #enumeration of all messages sent to board
ROLLING_MESSAGE_FILE = 'rolling_messages.txt'
HISTOGRAM_IMAGE_PREFIX = 'histogram_'
HISTOGRAM_IMAGE_SUFFIX = 'png'



HISTOGRAM_EMPTY_IMAGE_FILE = 'empty.png'
ROLLING_LOGFILE = 'rolling_log.txt' #file for error messages







































FLAG_MSG_FLIGHT = 1  # basic flight details
FLAG_MSG_INTERESTING = 2  # random tidbit about a flight
FLAG_MSG_HISTOGRAM = 3 # histogram message
























#if running on raspberry, then need to prepend path to file names
if psutil.sys.platform.title() == 'Linux':
  PICKLEFILE_30D = MESSAGEBOARD_PATH + PICKLEFILE_30D
  PICKLEFILE_ARCHIVE = MESSAGEBOARD_PATH + PICKLEFILE_ARCHIVE
  LOGFILE = MESSAGEBOARD_PATH + LOGFILE





  HOURLY_HISTOGRAM_FILE = WEBSERVER_PATH + HOURLY_HISTOGRAM_FILE
  CONFIG_FILE = WEBSERVER_PATH + CONFIG_FILE
  HOURLY_IMAGE_FILE = WEBSERVER_PATH + WEBSERVER_IMAGE_FOLDER + HOURLY_IMAGE_FILE
  HOURLY_DATA_FILE = WEBSERVER_PATH + HOURLY_DATA_FILE
  ROLLING_MESSAGE_FILE = WEBSERVER_PATH + ROLLING_MESSAGE_FILE
  HISTOGRAM_IMAGE_PREFIX = (
      WEBSERVER_PATH + WEBSERVER_IMAGE_FOLDER + HISTOGRAM_IMAGE_PREFIX)
  HISTOGRAM_EMPTY_IMAGE_FILE = (
      WEBSERVER_PATH + WEBSERVER_IMAGE_FOLDER + HISTOGRAM_EMPTY_IMAGE_FILE)
  ALL_MESSAGE_FILE = WEBSERVER_PATH + ALL_MESSAGE_FILE
  ROLLING_LOGFILE = WEBSERVER_PATH + ROLLING_LOGFILE

TIMEZONE = 'US/Pacific' # timezone of display
TZ = pytz.timezone(TIMEZONE)

SCREENIFY_SPLIT_FLAP = True

SPLITFLAP_CHARS_PER_LINE = 23
SPLITFLAP_LINE_COUNT = 7
SPLITFLAP_HEADER = '+' + '-'*SPLITFLAP_CHARS_PER_LINE + '+'
if SCREENIFY_SPLIT_FLAP:
  BORDER_CHARACTER = '|'
else:
  BORDER_CHARACTER = ''

DIRECTIONS_4 = ['N', 'E', 'S', 'W']
DIRECTIONS_8 = ['N', 'NE', 'E', 'SE', 'S', 'SW', 'W', 'NW']
DIRECTIONS_16 = ['N', 'NNE', 'NE', 'ENE', 'E', 'ESE', 'SE', 'SSE',
                 'S', 'SSW', 'SW', 'WSW', 'W', 'WNW', 'NW', 'NNW']

HOURS = ['12a', ' 1a', ' 2a', ' 3a', ' 4a', ' 5a', ' 6a', ' 7a',
         ' 8a', ' 9a', '10a', '11a', '12p', ' 1p', ' 2p', ' 3p',
         ' 4p', ' 5p', ' 6p', ' 7p', ' 8p', ' 9p', '10p', '11p']

SECONDS_IN_MINUTE = 60
MINUTES_IN_HOUR = 60
HOURS_IN_DAY = 24
SECONDS_IN_HOUR = SECONDS_IN_MINUTE * MINUTES_IN_HOUR

SECONDS_IN_DAY = SECONDS_IN_HOUR*HOURS_IN_DAY

#debug: still need to validate vert_rate units!
CLIMB_RATE_UNITS = 'fpm'  # need to confirm

#speed units from tracker are knots, based on dump-1090/track.c
#https://github.com/SDRplay/dump1090/blob/master/track.c
SPEED_UNITS = 'kn'
DISTANCE_UNITS = 'ft'

# For displaying histograms
# If a key is not present, how should it be displayed in histograms?
KEY_NOT_PRESENT_STRING = 'Unknown'
OTHER_STRING = 'Other' # What key strings should be listed last in sequence?
# What key strings should be listed last in sequence?
SORT_AT_END_STRINGS = [OTHER_STRING, KEY_NOT_PRESENT_STRING]
# What is the sorted sequence of keys for days of week?
DAYS_OF_WEEK = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat']

aircraft_length = {} # in meters
aircraft_length['Airbus A220-100 (twin-jet)'] = 35

aircraft_length['Airbus A319 (twin-jet)'] = 33.84
aircraft_length['Airbus A320 (twin-jet)'] = 37.57
aircraft_length['Airbus A320neo (twin-jet)'] = 37.57
aircraft_length['Airbus A321 (twin-jet)'] = 44.51
aircraft_length['Airbus A321neo (twin-jet)'] = 44.51
aircraft_length['Airbus A330-200 (twin-jet)'] = 58.82
aircraft_length['Airbus A330-300 (twin-jet)'] = 63.67
aircraft_length['Airbus A340-300 (quad-jet)'] = 63.69
aircraft_length['Airbus A350-1000 (twin-jet)'] = 73.79
aircraft_length['Airbus A350-900 (twin-jet)'] = 66.8
aircraft_length['Airbus A380-800 (quad-jet)'] = 72.72
aircraft_length['Boeing 737-400 (twin-jet)'] = 36.4
aircraft_length['Boeing 737-700 (twin-jet)'] = 33.63
aircraft_length['Boeing 737-800 (twin-jet)'] = 39.47
aircraft_length['Boeing 737-900 (twin-jet)'] = 42.11
aircraft_length['Boeing 747-400 (quad-jet)'] = 36.4
aircraft_length['Boeing 747-8 (quad-jet)'] = 76.25
aircraft_length['Boeing 757-200 (twin-jet)'] = 47.3
aircraft_length['Boeing 757-300 (twin-jet)'] = 54.4
aircraft_length['Boeing 767-200 (twin-jet)'] = 48.51
aircraft_length['Boeing 767-300 (twin-jet)'] = 54.94
aircraft_length['Boeing 777-200 (twin-jet)'] = 63.73
aircraft_length['Boeing 777-200LR/F (twin-jet)'] = 63.73
aircraft_length['Boeing 777-300ER (twin-jet)'] = 73.86
aircraft_length['Boeing 787-10 (twin-jet)'] = 68.28
aircraft_length['Boeing 787-8 (twin-jet)'] = 56.72
aircraft_length['Boeing 787-9 (twin-jet)'] = 62.81
aircraft_length['Canadair Regional Jet CRJ-200 (twin-jet)'] = 26.77
aircraft_length['Canadair Regional Jet CRJ-700 (twin-jet)'] = 32.3


aircraft_length['Bombardier Challenger 300 (twin-jet)'] = 20.92
aircraft_length['EMBRAER 175 (long wing) (twin-jet)'] = 31.68














def CheckIfProcessRunning(processName):
  #  Check if there is any running process that contains the given name processName.
  this_process_id = os.getpid()

  for proc in psutil.process_iter():
    try:
      # Check if process name contains the given name string.

      commands = proc.as_dict(attrs=['cmdline', 'pid'])
      if commands['cmdline']:
        command_running = any([processName.lower() in s.lower() for s in commands['cmdline']])

        if command_running and commands['pid'] != this_process_id:
          return commands['pid']
    except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
      pass
  return None


def LogMessage(message, file=LOGFILE):
  """Write a message to a logfile along with a timestamp.

  Args:
      message: string message to write
      file: string representing file name and, if needed, path to the file to write to
  """





  try:
    with open(file, 'a') as f:

      f.write('='*80+'\n')
      f.write(str(datetime.datetime.now(TZ))+'\n')
      f.write('\n')
      f.write(message+'\n')
  except IOError:
    LogMessage('Unable to append to ' + file)

  existing_log_lines = ReadFile(LOGFILE).splitlines()
  with open(ROLLING_LOGFILE, 'w') as f:
    f.write('\n'.join(existing_log_lines[-1000:]))







































def UtcToLocalTimeDifference(timezone=TIMEZONE):
  """Calculates number of seconds between UTC and given timezone.

  Returns number of seconds between UTC and given timezone; if no timezone given, uses
  TIMEZONE defined in global variable.

  Args:
    timezone: string representing a valid pytz timezone in pytz.all_timezones.

  Returns:
    Integer number of seconds.
  """
  utcnow = pytz.timezone('utc').localize(datetime.datetime.utcnow())
  home_time = utcnow.astimezone(pytz.timezone(timezone)).replace(tzinfo=None)
  system_time = utcnow.astimezone(tzlocal.get_localzone()).replace(tzinfo=None)

  offset = dateutil.relativedelta.relativedelta(home_time, system_time)
  offset_seconds = offset.hours * SECONDS_IN_HOUR




                            <----SKIPPED LINES---->




  lambda3 = lambda1 + dlambda13
  intersection = (degrees(phi3), degrees(lambda3))
  return intersection


def ConvertBearingToCompassDirection(bearing, length=3, pad=False):
  """Converts a bearing (in degrees) to a compass dir of 1, 2, or 3 chars (N, NW, NNW).

  Args:
    bearing: degrees to be converted
    length: if 1, 2, or 3, converts to one of 4, 8, or 16 headings:
      - 1: N, S, E, W
      - 2: SE, SW, etc. also valid
      - 3: NWN, ESE, etc. also valid
    pad: boolean indicating whether the direction should be right-justified to length
      characters

  Returns:
    String representation of the compass heading.
  """



  divisions = 2**(length+1)  # i.e.: 4, 8, or 16
  division_size = 360 / divisions  # i.e.: 90, 45, or 22.5
  bearing_number = round(bearing / division_size)

  if length == 1:
    directions = DIRECTIONS_4
  elif length == 2:
    directions = DIRECTIONS_8
  else:
    directions = DIRECTIONS_16

  direction = directions[bearing_number%divisions]
  if pad:
    direction = direction.rjust(length)
  return direction


def HaversineDistanceMeters(pos1, pos2):
  """Calculate the distance between two points on a sphere (e.g. Earth).





                            <----SKIPPED LINES---->




    pos2: a 2-tuple defining (lat, lon) in decimal degrees

  Returns:
    Distance between two points in meters.
  """
  is_numeric = [isinstance(x, numbers.Number) for x in (*pos1, *pos2)]
  if False in is_numeric:
    return None

  lat1, lon1, lat2, lon2 = [math.radians(x) for x in (*pos1, *pos2)]
  hav = (math.sin((lat2 - lat1) / 2.0)**2
         + math.cos(lat1) * math.cos(lat2) * math.sin((lon2 - lon1) / 2.0)**2)
  distance = 2 * RADIUS * math.asin(math.sqrt(hav))

  # Note: though pyproj has this, having trouble installing on rpi
  #az12, az21, distance = g.inv(lon1, lat1, lon2, lat2)

  return distance






























































































































def Angles(pos1, elevation1, pos2, elevation2):


















  sin = math.sin
  cos = math.cos
  atan2 = math.atan2
  atan = math.atan
  sqrt = math.sqrt
  radians = math.radians
  degrees = math.degrees





  distance = HaversineDistanceMeters(pos1, pos2)  # from home to plumb line of plane
  lat1, lon1, lat2, lon2 = [radians(x) for x in (*pos1, *pos2)]
  d_lon = lon2 - lon1
  # azimuth calc from https://www.omnicalculator.com/other/azimuth
  az = atan2((sin(d_lon)*cos(lat2)), (cos(lat1)*sin(lat2)-sin(lat1)*cos(lat2)*cos(d_lon)))
  az_degrees = degrees(az)
  elevation = elevation2 - elevation1
  alt = atan(elevation / distance)
  alt_degrees = degrees(alt)
  crow_distance = sqrt(elevation**2 + distance**2)  # from home to the plane

  return (az_degrees, alt_degrees, distance, crow_distance)




def TrajectoryLatLon(pos, distance, track):










  #distance in meters
  #track in degrees
  sin = math.sin
  cos = math.cos
  atan2 = math.atan2
  asin = math.asin
  radians = math.radians
  degrees = math.degrees

  track = radians(track)
  lat1 = radians(pos[0])
  lon1 = radians(pos[1])

  d_div_R = distance/RADIUS
  lat2 = asin(sin(lat1)*cos(d_div_R) + cos(lat1)*sin(d_div_R)*cos(track))
  lon2 = lon1 + atan2(sin(track)*sin(d_div_R)*cos(lat1), cos(d_div_R)-sin(lat1)*sin(lat2))

  lat2_degrees = degrees(lat2)
  lon2_degrees = degrees(lon2)
  return (lat2_degrees, lon2_degrees)




                            <----SKIPPED LINES---->




  is_numeric = [isinstance(x, numbers.Number) for x in (*pos, bearing)]
  if False in is_numeric:
    return None

  # To find the minimum distance, we must first find the point at which the minimum
  # distance will occur, which in turn is accomplished by finding the intersection
  # between that trajectory and a trajectory orthogonal (+90 degrees, or -90 degrees)
  # to it but intersecting HOME.
  potential_intersection1 = IntersectionForTwoPaths(pos, bearing, HOME, bearing + 90)
  potential_intersection2 = IntersectionForTwoPaths(pos, bearing, HOME, bearing - 90)
  potential_distance1 = HaversineDistanceMeters(potential_intersection1, HOME)
  potential_distance2 = HaversineDistanceMeters(potential_intersection2, HOME)

  # Since one of those two potential intersection points (i.e.: +90 or -90 degrees) will
  # create an irrational result, and given the strong locality to HOME that is expected
  # from the initial position, the "correct" result is identified by simply taking the
  # minimum distance of the two candidate.
  return min(potential_distance1, potential_distance2)


def SecondsToHhMm(seconds):
  """Converts integer number of seconds to xhym string (i.e.: 7h17m).

  Args:
    seconds: number of seconds


  Returns:
    String representation of hours and minutes.
  """
  minutes = int(abs(seconds) / SECONDS_IN_MINUTE)
  if minutes > MINUTES_IN_HOUR:
    hours = int(minutes / MINUTES_IN_HOUR)
    minutes = minutes % MINUTES_IN_HOUR



    text = str(hours) + 'h' + str(minutes) + 'm'
  else:



    text = str(minutes) + 'm'
  return text


def SecondsToHours(seconds):
  """Converts integer number of seconds to xh string (i.e.: 7h).

  Args:
    seconds: number of seconds

  Returns:
    String representation of hours.
  """
  minutes = int(abs(seconds) / SECONDS_IN_MINUTE)
  hours = round(minutes / MINUTES_IN_HOUR)
  return hours


def SecondsToDdHh(seconds):
  """Converts integer number of seconds to xdyh string (i.e.: 7d17h).

  Args:
    seconds: number of seconds

  Returns:
    String representation of days and hours.
  """
  days = int(abs(seconds) / SECONDS_IN_DAY)
  hours = SecondsToHours(seconds - days*SECONDS_IN_DAY)
  if hours == HOURS_IN_DAY:
    hours = 0
    days += 1
  text = '%dd%dh' % (days, hours)
  return text






















def HoursSinceMidnight(timezone=TIMEZONE):
  """Returns the float number of hours elapsed since midnight in the given timezone."""
  tz = pytz.timezone(timezone)
  now = datetime.datetime.now(tz)
  seconds_since_midnight = (
      now - now.replace(hour=0, minute=0, second=0, microsecond=0)).total_seconds()
  hours = seconds_since_midnight / SECONDS_IN_HOUR
  return hours


def HoursSinceFlight(now, then):
  """Returns the number of hours between a timestamp and a flight.

  Args:
    now: timezone-aware datetime representation of timestamp
    then: string representation of a timestamp in format '%Y-%m-%d %H:%M:%S.%f%z' (i.e.:
        '2020-03-22 00:05:44.026630-07:00'

  Returns:
    Number of hours between now and then (i.e.: now - then; a positive return value
    means now occurred after then).
  """
  then_time = datetime.datetime.strptime(then, '%Y-%m-%d %H:%M:%S.%f%z')
  delta = now - then_time
  delta_hours = delta.days * HOURS_IN_DAY + delta.seconds / SECONDS_IN_HOUR
  return delta_hours


def DataHistoryHours(flights):
  """Calculates the number of hours between the earliest flight in data and now.

  flights: List of all flights in sequential order, so that the first in list is earliest
      in time.

  Returns:
    Return time difference in hours between the first flight in data and now.
  """
  min_time_string = flights[0]['calcd_display_time']
  min_time = datetime.datetime.strptime(min_time_string, '%Y-%m-%d %H:%M:%S.%f%z')
  tz = min_time.tzinfo
  now = datetime.datetime.now(tz)
  delta = now - min_time
  delta_hours = delta.days * HOURS_IN_DAY + delta.seconds / SECONDS_IN_HOUR
  return round(delta_hours)


def DictGetReplaceNone(d, key, default_value=''):
  """d[key] if key exists and is not None; default_value otherwise."""
  value = d.get(key)
  if value is None:
    value = default_value
































































































  return value






































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































def SortByValues(values, keys, ignore_sort_at_end_strings=False):
  """Sorts the list of values in descending sequence, applying same resorting to keys.

  Given a list of keys and values representing a histogram, returns two new lists that
  are sorted so that the values occur in descending sequence and the keys are moved
  around in the same way. This allows the printing of a histogram with the largest
  keys listed first - i.e.: top five airlines.

  Keys identified by SORT_AT_END_STRINGS - such as, perhaps, 'Other' - will optionally
  be placed at the end of the sequence. And where values are identical, the secondary
  sort is based on the keys.

  Args:
    values: list of values for the histogram to be used as the primary sort key.
    keys: list of keys for the histogram that will be moved in the same way as the values.
    ignore_sort_at_end_strings: boolean indicating whether specially-defined keys will be
        sorted at the end.





                            <----SKIPPED LINES---->




    keyfunction,
    sort_type,
    title,
    position=None,
    truncate=0,
    hours=float('inf'),
    max_distance_feet=float('inf'),
    max_altitude_feet=float('inf'),
    normalize_factor=0,
    exhaustive=False,
    figsize_inches=(9, 6)):
  """Creates matplotlib.pyplot of histogram that can then be saved or printed.

  Args:
    data: the iterable (i.e.: list) of flight details, where each element in the list is
        a dictionary of the flight attributes.
    keyfunction: a function that when applied to a single flight (i.e.:
        keyfunction(data[0]) returns the key to be used for the histogram.
    data: the iterable of the raw data from which the histogram will be generated;
        each element of the iterable is a dictionary, that contains at least the key
        'calcd_display_time', and depending on other parameters, also potentially
        'calcd_min_feet' amongst others.
    keyfunction: the function that determines how the key or label of the histogram
        should be generated; it is called for each element of the data iterable. For
        instance, to simply generate a histogram on the attribute 'heading',
        keyfunction would be lambda a: a['heading'].
    title: the "base" title to include on the histogram; it will additionally be
        augmented with the details about the date range.
    position: Either a 3-digit integer or an iterable of three separate integers
        describing the position of the subplot. If the three integers are nrows, ncols,
        and index in order, the subplot will take the index position on a grid with nrows
        rows and ncols columns. index starts at 1 in the upper left corner and increases
        to the right.
    sort_type: determines how the keys (and the corresponding values) are sorted:
        'key': the keys are sorted by a simple comparison operator between them, which
            sorts strings alphabetically and numbers numerically.
        'value': the keys are sorted by a comparison between the values, which means
            that more frequency-occurring keys are listed first.
        list: if instead of the strings a list is passed, the keys are then sorted in
            the sequence enumerated in the list. This is useful for, say, ensuring that
            the days of the week (Tues, Wed, Thur, ...) are listed in sequence. Keys
            that are generated by keyfunction but that are not in the given list are




                            <----SKIPPED LINES---->




        the list, including potentially those with a frequency of zero, within the
        restrictions of truncate.
    figsize_inches: a 2-tuple of width, height indicating the size of the histogram.
  """
  (values, keys, filtered_data) = GenerateHistogramData(
      data,
      keyfunction,
      sort_type,
      truncate=truncate,
      hours=hours,
      max_distance_feet=max_distance_feet,
      max_altitude_feet=max_altitude_feet,
      normalize_factor=normalize_factor,
      exhaustive=exhaustive)
  if position:
    matplotlib.pyplot.subplot(*position)
  matplotlib.pyplot.figure(figsize=figsize_inches)
  values_coordinates = numpy.arange(len(keys))
  matplotlib.pyplot.bar(values_coordinates, values)

  earliest_flight_time = int(filtered_data[0]['calcd_timestamp'])
  last_flight_time = int(filtered_data[-1]['calcd_timestamp'])
  date_range_string = ' (%d flights over last %s hours)' % (
      sum(values), SecondsToDdHh(last_flight_time - earliest_flight_time))

  matplotlib.pyplot.title(title + date_range_string)

  matplotlib.pyplot.subplots_adjust(bottom=0.15, left=0.09, right=0.99, top=0.92)

  matplotlib.pyplot.xticks(
      values_coordinates, keys, rotation='vertical', wrap=True,
      horizontalalignment='right',
      verticalalignment='center')


def SaveTimeOfDayDataFile(flights, max_days, filename=HOURLY_DATA_FILE, precision=100):
  """Extracts hourly histogram into text file for a variety of altitudes and distances.

  Generates a csv with 26 columns:
  - col#1: altitude (in feet)
  - col#2: distance (in feet)
  - cols#3-26: hour of the day

  The first row is a header row; subsequent rows list the number of flights that have
  occurred in the last max_days with an altitude and distance less than that identified
  in the first two columns. Each row increments elevation or altitude by precision feet,
  up to the max determined by the max altitude and max distance amongst all the flights.

  Args:
    flights: list of the flights.
    max_days: maximum number of days as described.
    filename: file into which to save the csv.
    precision: number of feet to increment the altitude or distance.
  """
  max_altitude = int(max([flight.get('altitude', -1) for flight in flights]))
  max_distance = int(max([
      flight.get('distance_meters', -1)*FEET_IN_METER for flight in flights]))
  header_printed = False
  for flight in flights:
    flight['calcd_hour'] = ConvertHourStringTimeString(flight)
  lines = []
  for altitude in range(0, max_altitude, precision):
    for distance in range(0, max_distance, precision):
      (values, keys, unused_filtered_data) = GenerateHistogramData(
          flights,
          lambda a: a['calcd_hour'],
          HOURS,
          hours=max_days*HOURS_IN_DAY,
          max_distance_feet=distance,
          max_altitude_feet=altitude,
          normalize_factor=1, #debug: max_days
          exhaustive=True)
      if not header_printed:
        header_elements = ['altitude', 'distance', *keys]
        header_elements = [str(v) for v in header_elements]
        header = ','.join(header_elements)
        lines.append(header)
        header_printed = True

      line_elements = [altitude, distance]
      line_elements.extend(values)
      line_elements = [str(v) for v in line_elements]
      line = ','.join(line_elements)
      lines.append(line)
  try:
    with open(filename, 'w') as f:
      for line in lines:
        f.write(line+'\n')
  except IOError:
    LogMessage('Unable to write hourly histogram data file ' + filename)


def SaveTimeOfDayHistogramPng(
    flights,
    max_distance_feet,
    max_altitude_feet,
    max_days,
    filename=HOURLY_IMAGE_FILE,
    last_max_distance_feet=None,
    last_max_altitude_feet=None):
  """Saves as a png file the histogram of the hourly flight data for the given filters.

  Generates a png histogram of the count of flights by hour that meet the specified
  criteria: max altitude, max distance, and within the last number of days. Saves this
  histogram to disk, as well as potentially moving the previously saved image to a new
  location.

  Args:
    flights: list of the flights.
    max_distance_feet: max distance for which to include flights in the histogram.
    max_altitude_feet: max altitude for which to include flights in the histogram.
    max_days: maximum number of days as described.
    filename: file into which to save the csv.
    last_max_distance_feet: if provided, along with last_max_altitude_feet, generates
        a second data series with different criteria for distance and altitude, for
        which the histogram data will be plotted alongside the first series.
    last_max_altitude_feet: see above.
  """
  (values, keys, unused_filtered_data) = GenerateHistogramData(
      flights,
      ConvertHourStringTimeString,
      HOURS,
      hours=max_days*HOURS_IN_DAY,
      max_distance_feet=max_distance_feet,
      max_altitude_feet=max_altitude_feet,
      normalize_factor=max_days,
      exhaustive=True)

  comparison = last_max_distance_feet is not None and last_max_altitude_feet is not None
  if comparison:
    (last_values, unused_last_keys, unused_filtered_data) = GenerateHistogramData(
        flights,
        ConvertHourStringTimeString,
        HOURS,
        hours=max_days*HOURS_IN_DAY,
        max_distance_feet=last_max_distance_feet,
        max_altitude_feet=last_max_altitude_feet,
        normalize_factor=max_days,
        exhaustive=True)

  x = numpy.arange(len(keys))
  unused_fig, ax = matplotlib.pyplot.subplots()
  width = 0.35
  ax.bar(
      x - width/2, values, width,
      label='Current - alt: %d; dist: %d' % (max_altitude_feet, max_distance_feet))
  title = 'Daily Flights Expected: %d / day' % sum(values)
  if comparison:
    ax.bar(
        x + width/2, last_values, width,
        label='Prior - alt: %d; dist: %d' % (
            last_max_altitude_feet, last_max_distance_feet))
    title += ' (%+d)' % (round(sum(values) - sum(last_values)))

  ax.set_title(title)
  ax.set_ylabel('Average Observed Flights')
  if comparison:
    ax.legend()
  matplotlib.pyplot.xticks(
      x, keys, rotation='vertical', wrap=True,
      horizontalalignment='right',
      verticalalignment='center')

  matplotlib.pyplot.savefig(filename)
  matplotlib.pyplot.close()


def GenerateHistogramData(
    data,
    keyfunction,
    sort_type,
    truncate=float('inf'),
    hours=float('inf'),
    max_distance_feet=float('inf'),
    max_altitude_feet=float('inf'),
    normalize_factor=0,
    exhaustive=False):
  """Generates sorted data for a histogram from a description of the flights.

  Given an iterable describing the flights, this function generates the label (or key),
  and the frequency (or value) from which a histogram can be rendered.

  Args:
    data: the iterable of the raw data from which the histogram will be generated;
        each element of the iterable is a dictionary, that contains at least the key
        'calcd_display_time', and depending on other parameters, also potentially
        'calcd_min_feet' amongst others.
    keyfunction: the function that determines how the key or label of the histogram
        should be generated; it is called for each element of the data iterable. For
        instance, to simply generate a histogram on the attribute 'heading',
        keyfunction would be lambda a: a['heading'].
    sort_type: determines how the keys (and the corresponding values) are sorted:
        'key': the keys are sorted by a simple comparison operator between them, which
            sorts strings alphabetically and numbers numerically.
        'value': the keys are sorted by a comparison between the values, which means
            that more frequency-occurring keys are listed first.
        list: if instead of the strings a list is passed, the keys are then sorted in
            the sequence enumerated in the list. This is useful for, say, ensuring that
            the days of the week (Tues, Wed, Thur, ...) are listed in sequence. Keys
            that are generated by keyfunction but that are not in the given list are
            sorted last (and then amongst those, alphabetically).
    truncate: integer indicating the maximum number of keys to return; if set to 0, or if
        set to a value larger than the number of keys, no truncation occurs. But if set
        to a value less than the number of keys, then the keys with the lowest frequency
        are combined into one key named OTHER_STRING so that the number of keys
        in the resulting histogram (together with OTHER_STRING) is equal to truncate.
    hours: integer indicating the number of hours of history to include. Flights with a
        calcd_display_time more than this many hours in the past are excluded from the
        histogram generation. Note that this is timezone aware, so that if the histogram
        data is generated on a machine with a different timezone than that that recorded
        the original data, the correct number of hours is still honored.
    max_distance_feet: number indicating the geo fence outside of which flights should
        be ignored for the purposes of including the flight data in the histogram.
    max_altitude_feet: number indicating the maximum altitude outside of which flights
        should be ignored for the purposes of including the flight data in the histogram.
    normalize_factor: divisor to apply to all the values, so that we can easily
        renormalize the histogram to display on a percentage or daily basis; if zero,
        no renormalization is applied.
    exhaustive: boolean only relevant if sort_type is a list, in which case, this ensures
        that the returned set of keys (and matching values) contains all the elements in
        the list, including potentially those with a frequency of zero, within the
        restrictions of truncate.

  Returns:
    2-tuple of lists cut and sorted as indicated by parameters above:
    - list of values (or frequency) of the histogram elements
    - list of keys (or labels) of the histogram elements
  """
  histogram_dict = {}
  filtered_data = []

  # get timezone & now so that we can generate a timestamp for comparison just once
  if hours:
    now = GetNowInTimeZoneOfArbtraryFlight(data)
  for element in data:
    if (
        DictGetReplaceNone(
            element, 'calcd_min_feet', float('inf')) < max_distance_feet and
        DictGetReplaceNone(element, 'altitude', float('inf')) < max_altitude_feet and
        HoursSinceFlight(now, element['calcd_display_time']) <= hours):
      filtered_data.append(element)
      key = keyfunction(element)
      if key is None or key == '':
        key = KEY_NOT_PRESENT_STRING
      if key in histogram_dict:
        histogram_dict[key] += 1
      else:
        histogram_dict[key] = 1
  values = list(histogram_dict.values())
  keys = list(histogram_dict.keys())

  if normalize_factor:
    values = [v / normalize_factor for v in values]

  sort_by_enumerated_list = isinstance(sort_type, list)
  if exhaustive and sort_by_enumerated_list:
    missing_keys = set(sort_type).difference(set(keys))
    missing_values = [0 for unused_k in missing_keys]
    keys.extend(missing_keys)
    values.extend(missing_values)

  if keys:  # filters could potentially have removed all data
    if not truncate or len(keys) <= truncate:
      if sort_by_enumerated_list:
        (values, keys) = SortByDefinedList(values, keys, sort_type)
      elif sort_type == 'value':
        (values, keys) = SortByValues(values, keys)
      else:
        (values, keys) = SortByKeys(values, keys)
    else: #Unknown might fall in the middle, and so shouldn't be truncated
      (values, keys) = SortByValues(values, keys, ignore_sort_at_end_strings=True)

      truncated_values = list(values[:truncate-1])
      truncated_keys = list(keys[:truncate-1])
      other_value = sum(values[truncate-1:])
      truncated_values.append(other_value)
      truncated_keys.append(OTHER_STRING)
      if sort_by_enumerated_list:
        (values, keys) = SortByDefinedList(
            truncated_values, truncated_keys, sort_type)
      elif sort_type == 'value':
        (values, keys) = SortByValues(
            truncated_values, truncated_keys, ignore_sort_at_end_strings=False)
      else:
        (values, keys) = SortByKeys(truncated_values, truncated_keys)
  else:
    values = []
    keys = []
  return (values, keys, filtered_data)


def ReadFile(filename, log_exception=False):
  """Returns text from the given file name if available, empty string if not available.

  Args:
    filename: string of the filename to open, potentially also including the full path.
    log_exception: boolean indicating whether to log an exception if file not found.

  Returns:
    Return text string of file contents.
  """
  try:
    with open(filename, 'r') as content_file:
      file_contents = content_file.read()
  except IOError:
    if log_exception:
      LogMessage('Unable to read '+filename)
    return ''
  return file_contents


def ParseDumpJson(dump_json):
  """Identifies all airplanes within given distance of home from the dump1090 file.

  Since the dump1090 json will have messages from all flights that the antenna has picked
  up, we want to keep only flights that are within a relevant distance to us, and also to
  extract from the full set of data in the json to just the relevant fields for additional
  analysis.

  Args:
    dump_json: The text representation of the json message from dump1090-mutability

  Returns:
    Return 2-tuple, with first element being a dictionary of all airplanes, where keys are
    flight numbers (i.e.: 'SWA7543'), and the value is itself a dictionary of attributes.
    The included attributes are: speed, altitude, vert_rate, distance_meters, lat / lon,
    and track (or bearing / heading). The second element is the time stamp in the json
    file.
  """
  parsed = json.loads(dump_json)
  nearby_aircraft = {}
  for aircraft in parsed['aircraft']:
    flight_number = aircraft.get('flight', KEY_NOT_PRESENT_STRING).strip()
    if 'lat' in aircraft and 'lon' in aircraft:
      now = datetime.datetime.now(TZ)
      lat = aircraft.get('lat', KEY_NOT_PRESENT_STRING)
      lon = aircraft.get('lon', KEY_NOT_PRESENT_STRING)
      track = aircraft.get('track', KEY_NOT_PRESENT_STRING)
      distance_meters = round(
          HaversineDistanceMeters(HOME, (aircraft['lat'], aircraft['lon'])))
      min_meters = MinMetersToHome((lat, lon), track)
      if min_meters:
        min_feet = min_meters * FEET_IN_METER
      else:
        min_feet = None
      # simplified_aircraft is all attributes from radio or derivable from radio & math
      simplified_aircraft = {
          'distance_meters': distance_meters,
          # altitude is unknown units, but believed to be feet?
          'altitude': aircraft.get('altitude', KEY_NOT_PRESENT_STRING),
          # speed is is in knots
          'speed': aircraft.get('speed', KEY_NOT_PRESENT_STRING),
          # vert_rate in unknown units
          'vert_rate': aircraft.get('vert_rate', KEY_NOT_PRESENT_STRING),
          'lat': lat,
          'lon': lon,
          'track': track,
          'squawk': aircraft.get('squawk', KEY_NOT_PRESENT_STRING),
          'radio_now': parsed['now'],
          'dump_flight_number': flight_number,
          'calcd_display_time': now.strftime('%Y-%m-%d %H:%M:%S.%f%z'),
          'calcd_timestamp': datetime.datetime.timestamp(now),
          'calcd_min_feet': min_feet}

      if distance_meters < MIN_METERS:
        nearby_aircraft[flight_number] = simplified_aircraft
        if not flight_number:
          LogMessage('Dump JSON does not include a flight number: %s' % str(aircraft),
                     file=LOGFILE)

  return (nearby_aircraft, parsed['now'])


def GetFlightAwareJson(flight_number):
  """Scrapes the text json message from FlightAware for a given flight number.

  Given a flight number, loads the corresponding FlightAware webpage for that flight and
  extracts the relevant script that contains all the flight details from that page.

  Args:
    flight_number: text flight number (i.e.: SWA1234)

  Returns:
    Text representation of the json message from FlightAware.
  """
  url = 'https://flightaware.com/live/flight/' + flight_number
  try:
    response = requests.get(url)
  except:
    LogMessage('Unable to query FA for URL: ' + url)
    return ''
  soup = bs4.BeautifulSoup(response.text, "html.parser")
  l = soup.findAll('script')
  flight_script = None
  for script in l:
    if "trackpollBootstrap" in script.text:
      flight_script = script.text
      break
  if not flight_script:
    LogMessage('Unable to find trackpollBootstrap script in page: ' + response.text)
    return ''
  first_open_curly_brace = flight_script.find('{')
  flight_json = flight_script[first_open_curly_brace:-1]
  return flight_json


def ParseFlightAwareJson(flight_json):
  """Strips relevant data about the flight from FlightAware feed.

  The FlightAware json has hundreds of fields about a flight, only a fraction of which
  are relevant to extract. Note that some of the fields are inconsistently populated
  (i.e.: scheduled and actual times for departure and take-off).

  Args:
    flight_json: Text representation of the FlightAware json about a single flight.

  Returns:
    Dictionary of flight attributes extracted from the FlightAware json.
  """
  flight_details = {}
  parsed_json = json.loads(flight_json)
  fa_flight_number = list(parsed_json['flights'].keys())[0]
  parsed_flight_details = parsed_json['flights'][fa_flight_number]
  flight_details['fa_flight_number'] = fa_flight_number

  origin = parsed_flight_details.get('origin', '')
  if origin:
    origin_friendly = origin.get('friendlyLocation', '')
    origin_iata = origin.get('iata', '')
  else:
    origin_friendly = ''
    origin_iata = ''
  flight_details['origin_friendly'] = origin_friendly
  flight_details['origin_iata'] = origin_iata

  destination = parsed_flight_details.get('destination', '')
  if destination:
    destination_friendly = destination.get('friendlyLocation', '')
    destination_iata = destination.get('iata', '')
  else:
    destination_friendly = ''
    destination_iata = ''
  flight_details['destination_friendly'] = destination_friendly
  flight_details['destination_iata'] = destination_iata

  if origin_iata and destination_iata == origin_iata:
    LogMessage('Origin & destination both %s in FA JSON: %s'
               % (origin_iata, str(parsed_json)), file=LOGFILE)

  # perhaps interesting for private planes
  flight_details['owner_location'] = parsed_flight_details.get('ownerLocation')
  flight_details['owner'] = parsed_flight_details.get('owner')
  flight_details['tail'] = parsed_flight_details.get('tail')

  aircraft_type = parsed_flight_details.get('aircraft')
  if aircraft_type:
    aircraft_type_code = aircraft_type.get('type', '')
    aircraft_type_friendly = aircraft_type.get('friendlyType', '')
  else:
    aircraft_type_code = ''
    aircraft_type_friendly = ''
  flight_details['aircraft_type_code'] = aircraft_type_code
  flight_details['aircraft_type_friendly'] = aircraft_type_friendly

  takeoff_time = parsed_flight_details.get('takeoffTimes', '')
  if takeoff_time:
    scheduled_takeoff_time = takeoff_time.get('scheduled', '')
    actual_takeoff_time = takeoff_time.get('actual', '')
  else:
    scheduled_takeoff_time = ''
    actual_takeoff_time = ''
  flight_details['scheduled_takeofftime'] = scheduled_takeoff_time
  flight_details['actual_takeoff_time'] = actual_takeoff_time

  gate_departure_time = parsed_flight_details.get('gateDepartureTimes', '')
  if gate_departure_time:
    scheduled_departure_time = gate_departure_time.get('scheduled', '')
    actual_departure_time = gate_departure_time.get('actual', '')
  else:
    scheduled_departure_time = ''
    actual_departure_time = ''
  flight_details['scheduled_departure_time'] = scheduled_departure_time
  flight_details['actual_departure_time'] = actual_departure_time

  gate_arrival_time = parsed_flight_details.get('gateArrivalTimes', '')
  if gate_arrival_time:
    scheduled_arrival_time = gate_arrival_time.get('scheduled', '')
    estimated_arrival_time = gate_arrival_time.get('estimated', '')
  else:
    scheduled_arrival_time = ''
    estimated_arrival_time = ''
  flight_details['scheduled_arrival_time'] = scheduled_arrival_time
  flight_details['estimated_arrival_time'] = estimated_arrival_time

  landing_time = parsed_flight_details.get('landingTimes', '')
  if landing_time:
    scheduled_landing_time = landing_time.get('scheduled', '')
    estimated_landing_time = landing_time.get('estimated', '')
  else:
    scheduled_landing_time = ''
    estimated_landing_time = ''
  flight_details['scheduled_landing_time'] = scheduled_landing_time
  flight_details['estimated_landing_time'] = estimated_landing_time

  airline = parsed_flight_details.get('airline', '')
  if airline:
    airline_call_sign = airline.get('callsign', '')
    airline_short_name = airline.get('shortName', '')
    airline_full_name = airline.get('fullName', '')
  else:
    airline_call_sign = ''
    airline_short_name = ''
    airline_full_name = ''
  flight_details['airline_call_sign'] = airline_call_sign
  flight_details['airline_short_name'] = airline_short_name
  flight_details['airline_full_name'] = airline_full_name

  if len(parsed_json['flights'].keys()) > 1:
    LogMessage('There are multiple flights in the FlightAware json: ' + parsed_json)

  return flight_details


def AugmentWithDisplayableAirline(flight_details):
  """Augments flight details with display-ready airline attributes.

  Args:
    flight_details: dictionary with key-value attributes about the flight, but not yet
      containing all the necessary print-ready flight attributes.
  """
  # LINE1: UAL1425 - UNITED
  airline = DictGetReplaceNone(flight_details, 'airline_short_name')
  if not airline:
    airline = DictGetReplaceNone(flight_details, 'airline_full_name')
  if not airline:
    airline = KEY_NOT_PRESENT_STRING
  flight_details['calcd_airline'] = airline


def AugmentWithDisplayableAircraft(flight_details):
  """Augments flight details with display-ready aircraft attributes.

  Args:
    flight_details: dictionary with key-value attributes about the flight, but not yet
      containing all the necessary print-ready flight attributes.
  """
  # LINE2: Boeing 737-800 (twin-jet)
  aircraft_type = DictGetReplaceNone(flight_details, 'aircraft_type_friendly')
  aircraft_type = aircraft_type.replace('(twin-jet)', '(twin)')
  aircraft_type = aircraft_type.replace('(quad-jet)', '(quad)')
  aircraft_type = aircraft_type.replace('Regional Jet ', '')
  aircraft_type = aircraft_type[:SPLITFLAP_CHARS_PER_LINE]
  flight_details['calcd_aircraft_type'] = aircraft_type


def AugmentWithDisplayableOriginDestination(flight_details):
  """Augments flight details with display-ready origin and destination attributes.

  If the origin or destination is among a few key airports where the IATA code is
  well-known, then we can display only that code. Otherwise, we'll want to display
  both the code and a longer description of the airport. But we need to be mindful of
  the overall length of the display. So, for instance, these might be produced as
  valid origin-destination pairs:
  SFO-CLT Charlotte       <- Known origin
  Charlotte CLT-SFO       <- Known destination
  Charl CLT-SAN San Diego <- Neither origin nor destination known

  Args:
    flight_details: dictionary with key-value attributes about the flight, but not yet
      containing all the necessary print-ready flight attributes.
  """
  iata_length = 3
  known_airports = ('SJC', 'SFO', 'OAK') # Airport codes we don't need to expand
  origin = (DictGetReplaceNone(flight_details, 'origin_iata') + ' ' +
            DictGetReplaceNone(flight_details, 'origin_friendly').split(',')[0])
  destination = (DictGetReplaceNone(flight_details, 'destination_iata') + ' ' +
                 DictGetReplaceNone(flight_details, 'destination_friendly').split(',')[0])
  max_line_length_after_divider = SPLITFLAP_CHARS_PER_LINE - len('-')
  if (origin[:iata_length] not in known_airports
      and destination[:iata_length] not in known_airports):
    max_origin_length = int(max_line_length_after_divider/2)
    max_destination_length = max_line_length_after_divider - max_origin_length
    if len(origin) > max_origin_length and len(destination) > max_destination_length:
      origin_length = max_origin_length
      destination_length = max_destination_length
    elif len(origin) > max_origin_length:
      destination_length = len(destination)
      origin_length = max_line_length_after_divider - destination_length
    elif len(destination) > max_destination_length:
      origin_length = len(origin)
      destination_length = max_line_length_after_divider - origin_length
    else:
      origin_length = max_origin_length
      destination_length = max_destination_length
  elif origin[:iata_length] in known_airports:
    origin_length = iata_length
    destination_length = max_line_length_after_divider - origin_length
  elif destination[:iata_length] in known_airports:
    destination_length = iata_length
    origin_length = max_line_length_after_divider - destination_length
  else:
    destination_length = iata_length
    origin_length = iata_length

  origin = origin[:origin_length]
  destination = destination[:destination_length]
  flight_details['calcd_origin'] = origin
  flight_details['calcd_destination'] = destination

  origin_destination_pair = ''
  if origin and destination:
    origin_destination_pair = '-'.join([origin, destination])
  flight_details['calcd_origin_destination_pair'] = origin_destination_pair


def AugmentWithDisplayableDelay(flight_details):
  """Augments flight details with display-ready departure time and delay attributes.

  Args:
    flight_details: dictionary with key-value attributes about the flight, but not yet
      containing all the necessary print-ready flight attributes.
  """
  # LINE4: Dep: 08:18 (10m early)
  #        Dep: Unknown
  actual_departure = flight_details.get('actual_departure_time')
  scheduled_departure = flight_details.get('scheduled_departure_time')
  actual_takeoff_time = flight_details.get('actual_takeoff_time')
  scheduled_takeoff_time = flight_details.get('scheduled_takeofftime')
  calculable_delay = False

  scheduled = None
  delay_seconds = None
  delay_text = ''

  if actual_departure and scheduled_departure:
    actual = actual_departure
    scheduled = scheduled_departure
    departure_label = 'Dep'
    calculable_delay = True
  elif actual_takeoff_time and scheduled_takeoff_time:
    actual = actual_takeoff_time
    scheduled = scheduled_takeoff_time
    departure_label = 'T-O'
    calculable_delay = True
  elif actual_departure:
    actual = actual_departure
    departure_label = 'Act Dep'
  elif scheduled_departure:
    actual = scheduled_departure
    departure_label = 'Sch Dep'
  elif actual_takeoff_time:
    actual = actual_takeoff_time
    departure_label = 'Act T-O'
  elif scheduled_takeoff_time:
    actual = scheduled_takeoff_time
    departure_label = 'Sch T-O'
  else:
    actual = 0
    departure_time_text = 'Dep: Unknown'

  if actual:
    tz_corrected_actual = actual + UtcToLocalTimeDifference()
    departure_time_text = (
        departure_label + ': ' +
        datetime.datetime.fromtimestamp(tz_corrected_actual).strftime('%I:%M'))

    if calculable_delay:
      delay_seconds = actual - scheduled
      if int(delay_seconds / SECONDS_IN_MINUTE) == 0:
        delay_text = 'on time'
      else:
        if delay_seconds < 0:
          delay_direction = ' early'
        else:
          delay_direction = ' late'
        delay_text = SecondsToHhMm(delay_seconds) + delay_direction

      delay_text = ' (' + delay_text + ')'

  flight_details['calcd_actual_departure'] = actual
  flight_details['calcd_departure_time_text'] = departure_time_text
  flight_details['calcd_calculable_delay'] = calculable_delay
  flight_details['calcd_scheduled_departure'] = scheduled
  flight_details['calcd_delay_seconds'] = delay_seconds
  flight_details['calcd_delay_text'] = delay_text


def AugmentWithDisplayableTimeRemaining(flight_details):
  """Augments flight details with display-ready flight time remaining attributes.

  Args:
    flight_details: dictionary with key-value attributes about the flight, but not yet
      containing all the necessary print-ready flight attributes.
  """
  # LINE5: Rem: 4h18m
  #        Rem: Unk
  arrival = flight_details.get('estimated_arrival_time')
  if not arrival:
    arrival = flight_details.get('estimated_landing_time')
  if not arrival:
    arrival = flight_details.get('scheduled_arrival_time')
  if not arrival:
    arrival = flight_details.get('scheduled_landing_time')

  actual = flight_details['calcd_actual_departure']
  if actual and arrival:
    remaining_seconds = arrival - actual
  else:
    remaining_seconds = None
  flight_details['calcd_remaining_seconds'] = remaining_seconds


def Screenify(lines):
  """Transforms a list of lines to a single text string either for printing or sending.

  Given a list of lines that is a fully-formed message to send to the splitflap display,
  this function transforms the list of strings to a single string that is an
  easier-to-read and more faithful representation of how the message will be displayed.
  The transformations are to add blank lines to the message to make it consistent number
  of lines, and to add border to the sides & top / bottom of the message.

  Args:
    lines: list of strings that comprise the message

  Returns:
    String - which includes embedded new line characters, borders, etc. as described
    above, that can be printed to screen as the message.
  """
  if SCREENIFY_SPLIT_FLAP:
    for unused_n in range(SPLITFLAP_LINE_COUNT-len(lines)):
      lines.append('')
    lines = [BORDER_CHARACTER + line.ljust(SPLITFLAP_CHARS_PER_LINE) + BORDER_CHARACTER
             for line in lines]
    lines.insert(0, SPLITFLAP_HEADER)
    lines.append(SPLITFLAP_HEADER)

  return '\n'.join(lines)


def CreateMessageAboutFlight(flight_details):
  """Creates a message to describe interesting attributes about a single flight.

  Generates a multi-line description of a flight. A typical message might look like:
  UAL300 - UNITED        <- Flight number and airline
  BOEING 777-200 (TWIN)  <- Aircraft type
  SFO-HNL HONOLULU       <- Origin & destination
  DEP: 02:08 (12M EARLY) <- Departure time details
  REM: 5H14M             <- Remaining flight time
  185MPH 301DEG D:117FT  <- Trajectory details: speed; bearing; forecast min dist to HOME
  1975FT (+2368FPM)      <- Altitude details: current altitude & rate or ascent / descent

  However, not all of these details are always present, so some may be listed as unknown,
  or entire lines may be left out.

  Args:
    flight_details: dictionary of flight attributes.

  Returns:
    Printable string (with embedded new line characters)
  """
  lines = []

  # LINE1: UAL1425 - UNITED
  #        =======================
  airline = flight_details['calcd_airline']
  flight_number = DictGetReplaceNone(flight_details, 'dump_flight_number')
  line = (flight_number + ' - ' + airline)[:SPLITFLAP_CHARS_PER_LINE]
  if airline or flight_number:
    lines.append(line.upper())

  # LINE2: Boeing 737-800 (twin-jet)
  #        =======================
  aircraft_type = flight_details['calcd_aircraft_type']
  if aircraft_type:
    lines.append(aircraft_type.upper())

  # LINE3: SFO-CLT Charlotte
  #        Charlotte CLT-SFO
  #        =======================
  origin_destination_pair = flight_details.get('calcd_origin_destination_pair')
  if origin_destination_pair:
    lines.append(origin_destination_pair.upper())

  # LINE4: Dep: 08:18 (10m early)
  #        Dep: Unknown
  #        =======================
  departure_time_text = DictGetReplaceNone(flight_details, 'calcd_departure_time_text')
  if departure_time_text:
    delay_text = flight_details['calcd_delay_text']
    line = departure_time_text + delay_text
    lines.append(line.upper())

  # LINE5: Rem: 4h18m
  #        Rem: Unk
  #        =======================
  remaining_seconds = flight_details['calcd_remaining_seconds']
  if remaining_seconds is not None:
    line = 'Rem: ' + SecondsToHhMm(remaining_seconds)
  else:
    line = 'Rem: Unknown'
  lines.append(line.upper())

  # LINE6: 123mph 297deg D:1383ft
  #        =======================
  speed = flight_details.get('speed')
  heading = flight_details.get('track')

  if heading is None:
    line = str(speed) + 'mph'
  else:
    min_feet = flight_details['calcd_min_feet']
    line = '%d%s %sdeg D:%d%s' % (speed, SPEED_UNITS, heading, min_feet, DISTANCE_UNITS)
  lines.append(line.upper())

  # LINE7: Alt: 12345ft +1234fpm
  #        =======================
  altitude = flight_details.get('altitude')
  vert_rate = flight_details.get('vert_rate')

  line = ''
  if altitude:
    line += '%d%s' % (altitude, DISTANCE_UNITS)
  if vert_rate:
    line += ' (%+d%s)' % (vert_rate, CLIMB_RATE_UNITS)
  if line:
    lines.append(line.upper())

  return Screenify(lines)


def FlightInsightsTestHarness(flights, display=True):
  """Simulates what insightful messages might be displayed by replaying past flights."""

  messages = []
  for n in range(len(flights)):

    flight_message = CreateMessageAboutFlight(flights[n])
    if display:
      print(flight_message)
    messages.append(flight_message)

    interesting_things = FlightInsights(flights[:n+1])
    if display:
      for thing in interesting_things:
        print(thing)
    messages.extend(interesting_things)

  return messages


def FlightInsightLastSeen(flights, days_ago=2):
  """Generates string indicating when flight was last seen.

  Generates text of the following form.
  - KAL214 was last seen 2d0h ago

  Args:
    flights: the list of the raw data from which the insights will be generated,
        where the flights are listed in order of observation - i.e.: flights[0] was the
        earliest seen, and flights[-1] is the most recent flight for which we are
        attempting to generate an insight.
    days_ago: the minimum time difference for which a message should be generated -
        i.e.: many flights are daily, and so we are not necessarily interested to see
        about every daily flight that it was seen yesterday. However, more infrequent
        flights might be of interest.

  Returns:
    List of printable string messages; if no messages or insights to generate, then
    the list will be empty.
  """
  messages = []
  this_flight = flights[-1]
  this_flight_number = this_flight['dump_flight_number']
  this_timestamp = flights[-1]['calcd_timestamp']
  last_seen = [f for f in flights[:-1] if f['dump_flight_number'] == this_flight_number]
  messages = []
  if last_seen and this_flight_number:
    last_timestamp = last_seen[-1]['calcd_timestamp']
    if this_timestamp - last_timestamp > days_ago*SECONDS_IN_DAY:
      messages = [
          '%s was last seen %s ago' %
          (this_flight_number.strip(), SecondsToDdHh(this_timestamp - last_timestamp))]
  return messages


def FlightInsightDifferentAircraft(flights, percent_size_difference=0.1):
  """Generates string indicating changes in aircraft for the most recent flight.

  Generates text of the following form for the "focus" flight in the data.
  - Last time ASA1964 was seen on Mar 16, it was with a much larger plane (Airbus A320
    (twin-jet) @ 123ft vs. Airbus A319 (twin-jet) @ 111ft)
  - Last time ASA743 was seen on Mar 19, it was with a different type of airpline
    (Boeing 737-900 (twin-jet) vs. Boeing 737-800 (twin-jet))

  Args:
    flights: the list of the raw data from which the insights will be generated,
        where the flights are listed in order of observation - i.e.: flights[0] was the
        earliest seen, and flights[-1] is the most recent flight for which we are
        attempting to generate an insight.
    percent_size_difference: the minimum size (i.e.: length) difference for the insight
        to warrant including the size details.

  Returns:
    List of printable string messages; if no messages or insights to generate, then
    the list will be empty.
  """
  messages = []
  this_flight = flights[-1]
  this_flight_number = this_flight['dump_flight_number']
  last_seen = [f for f in flights[:-1] if f['dump_flight_number'] == this_flight_number]

  # Yesterday this same flight flew a materially different type of aircraft
  if last_seen and this_flight_number:
    last_flight = last_seen[-1]

    last_aircraft = last_flight.get('aircraft_type_friendly')
    last_aircraft_length = aircraft_length.get(last_aircraft, 0)

    this_aircraft = this_flight.get('aircraft_type_friendly')
    this_aircraft_length = aircraft_length.get(this_aircraft, 0)

    this_likely_commercial_flight = (
        this_flight.get('origin_iata') and this_flight.get('destination_iata'))
    if this_likely_commercial_flight and not this_aircraft_length:
      LogMessage('%s appears to be used in a commercial flight but yet is missing length'
                 'details' % this_aircraft, file=LOGFILE)

    likely_same_commercial_flight = (
        last_flight.get('origin_iata') == this_flight.get('origin_iata') and
        last_flight.get('destination_iata') == this_flight.get('destination_iata') and
        last_flight.get('airline_call_sign') == this_flight.get('airline_call_sign'))

    this_aircraft_bigger = False
    last_aircraft_bigger = False
    if (likely_same_commercial_flight and
        this_aircraft_length > last_aircraft_length * (1 + percent_size_difference)):
      this_aircraft_bigger = True
      comparative_text = 'larger'
    elif (likely_same_commercial_flight and
          last_aircraft_length > this_aircraft_length * (1 + percent_size_difference)):
      last_aircraft_bigger = True
      comparative_text = 'smaller'

    last_flight_time_string = datetime.datetime.fromtimestamp(
        last_flight['calcd_timestamp']).strftime('%b %-d')
    if this_aircraft_bigger or last_aircraft_bigger:
      messages = [
          'Last time %s was seen on %s, it was with a much %s '
          'plane (%s @ %dft vs. %s @ %dft)' % (
              this_flight_number.strip(), last_flight_time_string, comparative_text,
              this_aircraft, this_aircraft_length*FEET_IN_METER,
              last_aircraft, last_aircraft_length*FEET_IN_METER)]
    elif last_aircraft and this_aircraft and last_aircraft != this_aircraft:
      messages = [
          'Last time %s was seen on %s, it was with a different '
          'type of airpline (%s vs. %s)' % (
              this_flight_number.strip(), last_flight_time_string,
              last_aircraft, this_aircraft)]

  return messages


def FlightInsightNthFlight(flights, hours=1, min_multiple_flights=2):
  """Generates string about seeing many flights to the same destination in a short period.

  Generates text of the following form for the "focus" flight in the data.
  - ASA1337 was the 4th flight to PHX in the last 53 minutes, served by Alaska Airlines,
    American Airlines, Southwest and United
  - SWA3102 was the 2nd flight to SAN in the last 25 minutes, both with Southwest

  Args:
    flights: the list of the raw data from which the insights will be generated,
        where the flights are listed in order of observation - i.e.: flights[0] was the
        earliest seen, and flights[-1] is the most recent flight for which we are
        attempting to generate an insight.
    hours: the time horizon over which to look for flights with the same destination.
    min_multiple_flights: the minimum number of flights to that same destination to
        warrant generating an insight.

  Returns:
    List of printable string messages; if no messages or insights to generate, then
    the list will be empty.
  """
  messages = []
  this_flight = flights[-1]
  this_flight_number = DictGetReplaceNone(this_flight, 'dump_flight_number').strip()
  this_destination = this_flight.get('destination_iata', '')
  this_airline = DictGetReplaceNone(
      this_flight, 'airline_short_name', default_value=KEY_NOT_PRESENT_STRING)
  if not this_airline:
    this_airline = KEY_NOT_PRESENT_STRING # in case airline was stored as, say, ''
  this_timestamp = this_flight['calcd_timestamp']
  if this_destination and this_destination not in ['SFO', 'LAX']:
    similar_flights = [f for f in flights[:-1] if
                       this_timestamp - f['calcd_timestamp'] < SECONDS_IN_HOUR*hours and
                       this_destination == f.get('destination_iata', '')]
    similar_flights_count = len(similar_flights) + 1  # +1 for this_flight
    similar_flights_airlines = list(set([
        DictGetReplaceNone(f, 'airline_short_name', default_value=KEY_NOT_PRESENT_STRING)
        for f in similar_flights]))

    same_airline = [this_airline] == similar_flights_airlines

    if similar_flights_count >= min_multiple_flights:
      ordinal = lambda n: '%d%s' % (
          n, 'tsnrhtdd'[(math.floor(n/10)%10 != 1)*(n%10 < 4)*n%10::4])
      n_minutes = (
          (this_flight['calcd_timestamp'] - similar_flights[0]['calcd_timestamp'])
          / SECONDS_IN_MINUTE)
      message = ('%s was the %s flight to %s in the last %d minutes' % (
          this_flight_number, ordinal(similar_flights_count),
          this_destination, n_minutes))
      if same_airline and similar_flights_count == 2:
        message += ', both with %s' % this_airline
      elif same_airline:
        message += ', all with %s' % this_airline
      else:
        similar_flights_airlines.append(this_airline)
        similar_flights_airlines.sort()
        message += ', served by %s and %s' % (
            ', '.join(similar_flights_airlines[:-1]),
            similar_flights_airlines[-1])
      messages = [message]

  return messages


def FlightInsightSuperlativeAttribute(
    flights,
    key,
    label,
    units,
    absolute_list,
    insight_min=True,
    insight_max=True,
    hours=24):
  """Generates string about a numeric attribute of the flight being an extreme value.

  Generates text of the following form for the "focus" flight in the data.
  - N5286C has the slowest groundspeed (113mph vs. 163mph) in last 24 hours
  - CKS828 has the highest altitude (40000ft vs. 16575ft) in last 24 hours

  Args:
    flights: the list of the raw data from which the insights will be generated,
        where the flights are listed in order of observation - i.e.: flights[0] was the
        earliest seen, and flights[-1] is the most recent flight for which we are
        attempting to generate an insight.
    key: the key of the attribute of interest - i.e.: 'speed'.
    label: the human-readable string that should be displayed in the message - i.e.:
        'groundspeed'.
    units: the string units that should be used to label the value of the key - i.e.:
        'MPH'.
    absolute_list: a 2-tuple of strings that is used to label the min and the max - i.e.:
        ('lowest', 'highest'), or ('slowest', 'fastest').
    insight_min: boolean indicating whether to generate an insight about the min value.
    insight_max: boolean indicating whether to generate an insight about the max value.
    hours: the time horizon over which to look for superlative flights.

  Returns:
    List of printable string messages; if no messages or insights to generate, then
    the list will be empty.
  """
  messages = []
  this_flight = flights[-1]
  this_flight_number = DictGetReplaceNone(
      this_flight, 'dump_flight_number', 'The last flight').strip()
  first_timestamp = flights[0]['calcd_timestamp']
  last_timestamp = flights[-1]['calcd_timestamp']
  included_seconds = last_timestamp - first_timestamp

  if included_seconds > SECONDS_IN_HOUR * hours:
    relevant_flights = [
        f for f in flights[:-1]
        if last_timestamp - f['calcd_timestamp'] < SECONDS_IN_HOUR * hours]
    value_min = min([DictGetReplaceNone(
        f, key, default_value=float('inf')) for f in relevant_flights])
    value_max = max([DictGetReplaceNone(
        f, key, default_value=float('-inf')) for f in relevant_flights])
    values_other = len(
        [1 for f in relevant_flights if isinstance(f.get(key), numbers.Number)])

    this_value = this_flight.get(key)

    if this_value and values_other:

      superlative = True
      if this_value > value_max and insight_max:
        absolute_string = absolute_list[1]
        other_value = value_max
      elif this_value < value_min and insight_min:
        absolute_string = absolute_list[0]
        other_value = value_min
      else:
        superlative = False

      if superlative:
        messages.append('%s has the %s %s (%d%s vs. %d%s) in last %d hours' % (
            this_flight_number, absolute_string, label,
            this_value, units, other_value, units, hours))

  return messages


def PercentileScore(scores, value):
  """Returns the percentile that a particular value is in a list of numbers.

  Roughly inverts numpy.percentile. That is, numpy.percentile(scores_list, percentile)
  to get the value of the list that is at that percentile;
  PercentileScore(scores_list, value) will yield back approximately that percentile.

  If the value matches identical elements in the list, this function takes the average
  position of those identical values to compute a percentile. Thus, for some lists
  (i.e.: where there are lots of flights that have a 0 second delay, or a 100% delay
  frequency), you may not get a percentile of 0 or 100 even with values equal to the
  min or max element in the list.

  Args:
    scores: the list of numbers.
    value: the value for which we want to determine the percentile.

  Returns:
    Returns an integer percentile in the range [0, 100] inclusive.
  """
  scores = sorted(list(scores))
  count_values_below_score = len([1 for s in scores if s < value])
  count_values_at_score = len([1 for s in scores if s == value])
  percentile = (count_values_below_score + count_values_at_score / 2) / len(scores)
  return round(percentile*100)


def FlightInsightGroupPercentile(
    flights,
    group_function,
    value_function,
    value_string_function,
    group_label,
    value_label,
    min_days=1,
    lookback_days=30,
    min_this_group_size=0,
    min_comparison_group_size=0,
    min_group_qty=0,
    percentile_low=10,
    percentile_high=90):
  """Generates a string about extreme values of groups of flights.

  Generates text of the following form for the "focus" flight in the data.
  - flight SIA31 (n=7) has a delay frequency in the 95th %tile, with 100% of flights
    delayed an average of 6m over the last 4d1h
  - flight UAL300 (n=5) has a delay time in the 1st %tile, with an average delay of 0m
  over the last 4d5h

  Args:
    flights: the list of the raw data from which the insights will be generated,
        where the flights are listed in order of observation - i.e.: flights[0] was the
        earliest seen, and flights[-1] is the most recent flight for which we are
        attempting to generate an insight.
    group_function: function that, when called with a flight, returns the grouping key.
        That is, for example, group_function(flight) = 'B739'
    value_function: function that, when called with a list of flights, returns the
        value to be used for the comparison to identify min / max. Typically, the count,
        but could also be a sum, standard deviation, etc. - for perhaps the greatest
        range in flight altitude. If the group does not have a valid value and so
        should be excluded from comparison - i.e.: average delay of a group of flights
        which did not have a calculable_delay on any flight, this function should
        return None.
    value_string_function: function that, when called with the two parameters flights
        and value, returns a string (inclusive of units and label) that should be
        displayed to describe the quantity. For instance, if value_function returns
        seconds, value_string_function could convert that to a string '3h5m'. Or if
        value_function returns an altitude range, value_string_function could return
        a string 'altitude range of 900ft (1100ft - 2000ft)'.
    group_label: string to identify the group type - i.e.: 'aircraft' or 'flight' in
        the examples above.
    value_label: string to identify the value - i.e.: 'flights' in the examples above,
        but might also be i.e.: longest *delay*, or other quantity descriptor.
    min_days: the minimum amount of history required to start generating insights
        about delays.
    lookback_days: the maximum amount of history which will be considered in generating
        insights about delays.
    min_this_group_size: even if this group has, say, the maximum average delay, if its
        a group of size 1, that is not necessarily very interesting. This sets the
        minimum group size for the focus flight.
    min_comparison_group_size: similarly, comparing the focus group to groups of size
        one does not necessarily produce a meaningful comparison; this sets to minimum
        size for the other groups.
    min_group_qty: when generating a percentile, if there are only 3 or 4 groups among
        which to generate a percentile (i.e.: only a handful of destinations have been
        seen so far, etc.) then it is not necessarily very interesting to generate a
        message; this sets the minimum quantity of groups necessary (including the
        focus group) to generate a message.
    percentile_low: number [0, 100] inclusive that indicates the percentile that the focus
        flight group must equal or be less than for the focus group to trigger an insight;
        if None, no high percentile insight will be generated.
    percentile_high: number [0, 100] inclusive that indicates the percentile that the
        focus flight group must equal or be greater than for the focus group to trigger an
        insight; if None, no high percentile insight will be generated.

  Returns:
    List of printable string messages; if no messages or insights to generate, then
    the list will be empty.
  """
  messages = []
  first_timestamp = flights[0]['calcd_timestamp']
  last_timestamp = flights[-1]['calcd_timestamp']
  included_seconds = last_timestamp - first_timestamp

  if included_seconds > SECONDS_IN_DAY * min_days:

    relevant_flights = [
        f for f in flights if
        last_timestamp - f['calcd_timestamp'] < SECONDS_IN_DAY * lookback_days]

    grouped_flights = {}
    for flight in relevant_flights:
      group = group_function(flight)
      grouping = grouped_flights.get(group, [])
      grouping.append(flight)
      grouped_flights[group] = grouping

    grouped_values = {g: value_function(grouped_flights[g]) for g in grouped_flights}
    this_group = group_function(relevant_flights[-1])
    this_value = grouped_values[this_group]
    # Remove those for which no value could be calculated
    grouped_values = {
        g: grouped_values[g] for g in grouped_values if grouped_values[g] is not None}

    min_value_percentile = float('-inf')
    if percentile_low is not None:
      min_value_percentile = numpy.percentile(
          list(grouped_values.values()),
          percentile_low)

    max_value_percentile = float('inf')
    if percentile_high is not None:
      max_value_percentile = numpy.percentile(
          list(grouped_values.values()),
          percentile_high)

    if this_value and len(grouped_values) > min_group_qty:

      this_group_size = len(grouped_flights[this_group])

      time_horizon_string = SecondsToDdHh(
          last_timestamp - relevant_flights[0]['calcd_timestamp'])
      min_comparison_group_size_string = ''
      if min_comparison_group_size > 1:
        min_comparison_group_size_string = ' amongst %s with at least %d flights' % (
            group_label, min_comparison_group_size)

      ordinal = lambda n: '%d%s' % (
          n, 'tsnrhtdd'[(math.floor(n/10)%10 != 1)*(n%10 < 4)*n%10::4])

      # FLIGHT X (n=7) is has the Xth percentile of DELAYS, with an average delay of
      # 80 MINUTES
      if this_group_size > min_this_group_size and (
          this_value <= min_value_percentile or this_value >= max_value_percentile):
        messages = [
            '%s %s (n=%d) has a %s in the %s %%tile, with %s over the last %s%s' % (
                group_label,
                this_group,
                this_group_size,
                value_label,
                ordinal(PercentileScore(grouped_values.values(), this_value)),
                value_string_function(grouped_flights[this_group], this_value),
                time_horizon_string,
                min_comparison_group_size_string)]

  return messages


def FlightInsightSuperlativeGroup(
    flights,
    group_function,
    value_function,
    value_string_function,
    group_label,
    value_label,
    absolute_list,
    min_days=1,
    lookback_days=30,
    min_this_group_size=0,
    min_comparison_group_size=0,
    insight_min=True,
    insight_max=True):
  """Generates a string about extreme values of groups of flights.

  Generates text of the following form for the "focus" flight in the data.
  - aircraft B739 (n=7) is tied with B738 and A303 for the most flights at 7 flights
    over the last 3d7h amongst aircraft with a least 5 flights
  - aircraft B739 (n=7) is tied with 17 others for the most flights at 7 flights over
    the last 3d7h amongst aircraft with a least 5 flights
  - flight UAL1075 (n=12) has the most flights with 12 flights; the next most flights
    is 11 flights over the last 7d5h

  Args:
    flights: the list of the raw data from which the insights will be generated,
        where the flights are listed in order of observation - i.e.: flights[0] was the
        earliest seen, and flights[-1] is the most recent flight for which we are
        attempting to generate an insight.
    group_function: function that, when called with a flight, returns the grouping key.
        That is, for example, group_function(flight) = 'B739'
    value_function: function that, when called with a list of flights, returns the
        value to be used for the comparison to identify min / max. Typically, the count,
        but could also be a sum, standard deviation, etc. - for perhaps the greatest
        range in flight altitude. If the group does not have a valid value and so
        should be excluded from comparison - i.e.: average delay of a group of flights
        which did not have a calculable_delay on any flight, this function should
        return None.
    value_string_function: function that, when called with the two parameters flights
        and value, returns a string (inclusive of units and label) that should be
        displayed to describe the quantity. For instance, if value_function returns
        seconds, value_string_function could convert that to a string '3h5m'. Or if
        value_function returns an altitude range, value_string_function could return
        a string 'altitude range of 900ft (1100ft - 2000ft)'.
    group_label: string to identify the group type - i.e.: 'aircraft' or 'flight' in
        the examples above.
    value_label: string to identify the value - i.e.: 'flights' in the examples above,
        but might also be i.e.: longest *delay*, or other quantity descriptor.
    absolute_list: a 2-tuple of strings that is used to label the min and the max - i.e.:
        ('most', 'least'), or ('lowest average', 'highest average').
    min_days: the minimum amount of history required to start generating insights
        about delays.
    lookback_days: the maximum amount of history which will be considered in generating
        insights about delays.
    min_this_group_size: even if this group has, say, the maximum average delay, if its
        a group of size 1, that is not necessarily very interesting. This sets the
        minimum group size for the focus flight.
    min_comparison_group_size: similarly, comparing the focus group to groups of size
        one does not necessarily produce a meaningful comparison; this sets to minimum
        size for the other groups.
    insight_min: boolean indicating whether to possibly generate insight based on the
        occurrence of the min value.
    insight_max: boolean indicating whether to possibly generate insight based on the
        occurrence of the max value.

  Returns:
    List of printable string messages; if no messages or insights to generate, then
    the list will be empty.
  """
  messages = []
  first_timestamp = flights[0]['calcd_timestamp']
  last_timestamp = flights[-1]['calcd_timestamp']
  included_seconds = last_timestamp - first_timestamp

  if included_seconds > SECONDS_IN_DAY * min_days:

    relevant_flights = [
        f for f in flights if
        last_timestamp - f['calcd_timestamp'] < SECONDS_IN_DAY * lookback_days]

    grouped_flights = {}
    for flight in relevant_flights:
      group = group_function(flight)
      grouping = grouped_flights.get(group, [])
      grouping.append(flight)
      grouped_flights[group] = grouping

    grouped_values = {g: value_function(grouped_flights[g]) for g in grouped_flights}

    grouped_values = {g: value_function(grouped_flights[g]) for g in grouped_flights}
    this_group = group_function(relevant_flights[-1])
    this_value = grouped_values[this_group]
    # Remove those for which no value could be calculated
    grouped_values = {
        g: grouped_values[g] for g in grouped_values if grouped_values[g] is not None}

    other_values = list(grouped_values.values())
    other_values = [v for v in other_values if v > min_comparison_group_size]
    if this_value in other_values:
      other_values.remove(this_value)

    if other_values:
      min_value = min(other_values)
      max_value = max(other_values)

      if this_value:
        if this_value > max_value and insight_max:
          superlative = True
          equality = False
          superlative_string = absolute_list[1]
          next_value = max_value
        elif this_value == max_value and insight_max:
          superlative = False
          equality = True
          superlative_string = absolute_list[1]
        elif this_value < min_value and insight_min:
          superlative = True
          equality = False
          superlative_string = absolute_list[0]
          next_value = min_value
        elif this_value == min_value and insight_min:
          superlative = False
          equality = True
          superlative_string = absolute_list[0]
        else:
          superlative = False
          equality = False

        this_group_size = len(grouped_flights[this_group])
        time_horizon_string = SecondsToDdHh(
            last_timestamp - relevant_flights[0]['calcd_timestamp'])
        min_comparison_group_size_string = ''
        if min_comparison_group_size > 1:
          min_comparison_group_size_string = (
              ' amongst %s with at least %d flights' %
              (group_label, min_comparison_group_size))

        # flight x (n=7) is tied with a, b, and c for the (longest average, shortest
        # average) delay at 80 minutes
        # flight x is tied with a, b, and c for the (most frequent, least frequent)
        # delay at 30%
        if equality and this_group_size > min_this_group_size:

          identical_groups = sorted([
              str(g) for g in grouped_values
              if grouped_values[g] == this_value and g != this_group])
          if len(identical_groups) > 4:
            identical_string = '%d others' % len(identical_groups)
          elif len(identical_groups) > 1:
            identical_string = (
                '%s and %s' % (', '.join(identical_groups[:-1]), identical_groups[-1]))
          else:
            identical_string = str(identical_groups[0])

          messages = [
              '%s %s (n=%d) is tied with %s for the %s %s at %s over the last %s%s' % (
                  group_label,
                  this_group,
                  this_group_size,
                  identical_string,
                  superlative_string,
                  value_label,
                  value_string_function(flights, this_value),
                  time_horizon_string,
                  min_comparison_group_size_string)]

        elif superlative and this_group_size > min_this_group_size:
          messages = [
              '%s %s (n=%d) has the %s %s with %s; the next '
              '%s %s is %s over the last %s%s' % (
                  group_label,
                  this_group,
                  this_group_size,
                  superlative_string,
                  value_label,
                  value_string_function(flights, this_value),
                  superlative_string,
                  value_label,
                  value_string_function(flights, next_value),
                  time_horizon_string,
                  min_comparison_group_size_string)]

  return messages


def AverageDelay(flights):
  """Returns the average delay time for a list of flights.

  Args:
    flights: the list of the raw flight data.

  Returns:
    Average seconds of flight delay, calculated as the total seconds delayed amongst
    all the flights that have a positive delay, divided by the total number of flights
    that have a calculable delay. If no flights have a calculable delay, returns None.
  """
  delay_seconds = [
      f['calcd_delay_seconds']
      if f['calcd_delay_seconds'] > 0 else 0
      for f in flights if f['calcd_calculable_delay']]
  average_delay = None
  if delay_seconds:
    average_delay = sum(delay_seconds) / len(delay_seconds)
  return average_delay


def PercentDelay(flights):
  """Returns the percentage of flights that have a positive delay for a list of flights.

  Args:
    flights: the list of the raw flight data.

  Returns:
    Percentage of flights with a delay, calculated as the count of flights with a
    positive delay divided by the total number of flights that have a calculable delay.
    If no flights have a calculable delay, returns None.
  """
  calculable_delay_seconds = [
      f['calcd_delay_seconds'] for f in flights if f['calcd_calculable_delay']]
  delay_count = sum([1 for s in calculable_delay_seconds if s > 0])
  percent_delay = None
  if calculable_delay_seconds:
    percent_delay = delay_count / len(calculable_delay_seconds)
  return percent_delay


def FlightInsightFirstInstance(
    flights,
    key,
    label,
    days=7,
    additional_descriptor_fcn=''):
  """Generates string indicating the flight has the first instance of a particular key.

  Generates text of the following form for the "focus" flight in the data.
  - N311CG is the first time aircraft GLF6 (Gulfstream Aerospace Gulfstream G650
    (twin-jet)) has been seen since at least 7d5h ago
  - PCM8679 is the first time airline Westair Industries has been seen since 9d0h ago

  Args:
    flights: the list of the raw data from which the insights will be generated,
        where the flights are listed in order of observation - i.e.: flights[0] was the
        earliest seen, and flights[-1] is the most recent flight for which we are
        attempting to generate an insight.
    key: the key of the attribute of interest - i.e.: 'destination_iata'.
    label: the human-readable string that should be displayed in the message - i.e.:
        'destination'.
    days: the minimum time of interest for an insight - i.e.: we probably see LAX every
        hour, but we are only interested in particular attributes that have not been
        seen for at least some number of days. Note, however, that the code will go back
        even further to find the last time that attribute was observed, or if never
        observed, indicating "at least".
    additional_descriptor_fcn: a function that, when passed a flight, returns an
        additional parenthetical notation to include about the attribute or flight
        observed - such as expanding the IATA airport code to its full name, etc.

  Returns:
    List of printable string messages; if no messages or insights to generate, then
    the list will be empty.
  """
  messages = []
  this_flight = flights[-1]
  this_flight_number = this_flight['dump_flight_number']
  first_timestamp = flights[0]['calcd_timestamp']
  last_timestamp = flights[-1]['calcd_timestamp']
  included_seconds = last_timestamp - first_timestamp

  if included_seconds > SECONDS_IN_DAY * days:
    this_instance = DictGetReplaceNone(this_flight, key)
    matching = [f for f in flights[:-1] if DictGetReplaceNone(f, key) == this_instance]

    last_potential_observation_sec = included_seconds
    if matching:
      last_potential_observation_sec = last_timestamp - matching[-1]['calcd_timestamp']

    if this_instance and last_potential_observation_sec > SECONDS_IN_DAY * days:
      additional_descriptor = ''
      if additional_descriptor_fcn:
        additional_descriptor = ' (%s)' % additional_descriptor_fcn(this_flight)
      last_potential_observation_string = SecondsToDdHh(last_potential_observation_sec)
      if matching:
        messages.append(
            '%s is the first time %s %s%s has been seen since %s ago' %
            (this_flight_number, label, this_instance, additional_descriptor,
             last_potential_observation_string))
      else:
        messages.append(
            '%s is the first time %s %s%s has been seen since at least %s ago' %
            (this_flight_number, label, this_instance, additional_descriptor,
             last_potential_observation_string))

  return messages


def FlightInsightSuperlativeVertrate(flights, hours=24):
  """Generates string about the climb rate of the flight being an extreme value.

  Generates text of the following form for the "focus" flight in the data.
  - UAL631   has the fastest ascent rate (5248fpm, 64fpm faster than next fastest) in
    last 24 hours
  - CKS1820 has the fastest descent rate (-1152fpm, -1088fpm faster than next fastest)
    in last 24 hours

  While this is conceptually similar to the more generic FlightInsightSuperlativeVertrate
  function, vert_rate - because it can be either positive or negative, with different
  signs requiring different labeling and comparisons - it needs its own special handling.

  Args:
    flights: the list of the raw data from which the insights will be generated,
        where the flights are listed in order of observation - i.e.: flights[0] was the
        earliest seen, and flights[-1] is the most recent flight for which we are
        attempting to generate an insight.
    hours: the time horizon over which to look for superlative flights.

  Returns:
    List of printable string messages; if no messages or insights to generate, then
    the list will be empty.
  """
  messages = []
  this_flight = flights[-1]
  this_flight_number = this_flight['dump_flight_number']
  first_timestamp = flights[0]['calcd_timestamp']
  last_timestamp = flights[-1]['calcd_timestamp']
  sufficient_data = (last_timestamp - first_timestamp) > SECONDS_IN_HOUR * hours
  pinf = float('inf')
  ninf = float('-inf')

  if sufficient_data:
    relevant_flights = [
        f for f in flights[:-1]
        if last_timestamp - f['calcd_timestamp'] < SECONDS_IN_HOUR * hours]

    def AscentRate(f, default):
      vert_rate = DictGetReplaceNone(f, 'vert_rate', default_value=default)
      if vert_rate < 0:
        vert_rate = default
      return vert_rate

    other_ascents = len([
        1 for f in relevant_flights
        if isinstance(f.get('vert_rate'), numbers.Number) and AscentRate(f, ninf) > 0])
    if other_ascents:
      ascent_min = min(
          [AscentRate(f, pinf) for f in relevant_flights if AscentRate(f, ninf) > 0])
      ascent_max = max(
          [AscentRate(f, ninf) for f in relevant_flights if AscentRate(f, ninf) > 0])

    def DescentRate(f, default):
      vert_rate = DictGetReplaceNone(f, 'vert_rate', default_value=default)
      if vert_rate > 0:
        vert_rate = default
      return vert_rate

    other_descents = len([
        1 for f in relevant_flights
        if isinstance(f.get('vert_rate'), numbers.Number) and DescentRate(f, pinf) < 0])
    if other_descents:
      descent_min = min(
          [DescentRate(f, pinf) for f in relevant_flights if DescentRate(f, pinf) < 0])
      descent_max = max(
          [DescentRate(f, ninf) for f in relevant_flights if DescentRate(f, pinf) < 0])

    this_vert_rate = this_flight.get('vert_rate')

    if this_vert_rate is not None and this_vert_rate >= 0:
      this_ascent = this_vert_rate
      this_descent = None
    else:
      this_descent = this_vert_rate
      this_ascent = None

    if this_ascent and other_ascents and this_ascent > ascent_max:
      messages.append('%s has the fastest ascent rate (%d%s, %d%s faster '
                      'than next fastest) in last %d hours' % (
                          this_flight_number, this_ascent, CLIMB_RATE_UNITS,
                          this_ascent - ascent_max, CLIMB_RATE_UNITS, hours))
    elif this_ascent and other_ascents and this_ascent < ascent_min:
      messages.append('%s has the slowest ascent rate (%d%s, %d%s slower '
                      'than next slowest) in last %d hours' % (
                          this_flight_number, this_ascent, CLIMB_RATE_UNITS,
                          ascent_min - this_ascent, CLIMB_RATE_UNITS, hours))
    elif this_descent and other_descents and this_descent < descent_min:
      messages.append('%s has the fastest descent rate (%d%s, %d%s faster '
                      'than next fastest) in last %d hours' % (
                          this_flight_number, this_descent, CLIMB_RATE_UNITS,
                          this_descent - descent_min, CLIMB_RATE_UNITS, hours))
    elif this_descent and other_descents and this_descent > descent_max:
      messages.append('%s has the slowest descent rate (%d%s, %d%s slower '
                      'than next slowest) in last %d hours' % (
                          this_flight_number, this_descent, CLIMB_RATE_UNITS,
                          descent_max - this_descent, CLIMB_RATE_UNITS, hours))

  return messages


def FlightInsightDelays(
    flights,
    min_days=1,
    lookback_days=30,
    min_late_percentage=0.75,
    min_this_delay_minutes=0,
    min_average_delay_minutes=0):
  """Generates string about the delays this flight has seen in the past.

  Only if this flight has a caclculable delay itself, this will generate text of the
  following form for the "focus" flight in the data.
  - This 8m delay is the longest UAL1175 has seen in the last 9 days (avg delay is 4m);
    overall stats: 1 early; 9 late; 10 total
  - With todays delay of 7m, UAL1175 is delayed 88% of the time in the last 8 days for
    avg delay of 4m; overall stats: 1 early; 8 late; 9 total

  Args:
    flights: the list of the raw data from which the insights will be generated,
        where the flights are listed in order of observation - i.e.: flights[0] was the
        earliest seen, and flights[-1] is the most recent flight for which we are
        attempting to generate an insight.
    min_days: the minimum amount of history required to start generating insights
        about delays.
    lookback_days: the maximum amount of history which will be considered in generating
        insights about delays.
    min_late_percentage: flights that are not very frequently delayed are not
        necessarily very interesting to generate insights about; this specifies the
        minimum percentage the flight must be late to generate a message that focuses
        on the on-time percentage.
    min_this_delay_minutes: a delay of 1 minute is not necessarily interesting; this
        specifies the minimum delay time this instance of the flight must be late to
        generate a message that focuses on this flight's delay.
    min_average_delay_minutes: an average delay of only 1 minute, even if it happens
        every day, is not necessarily very interesting; this specifies the minimum
        average delay time to generate either type of delay message.

  Returns:
    List of printable string messages; if no messages or insights to generate, then
    the list will be empty.
  """
  messages = []
  this_flight = flights[-1]
  this_flight_number = this_flight.get('dump_flight_number', '')
  first_timestamp = flights[0]['calcd_timestamp']
  last_timestamp = flights[-1]['calcd_timestamp']
  included_seconds = last_timestamp - first_timestamp

  if (included_seconds > SECONDS_IN_DAY * min_days
      and this_flight['calcd_calculable_delay']):
    this_delay_seconds = this_flight['calcd_delay_seconds']
    relevant_flights = [
        f for f in flights if
        last_timestamp - f['calcd_timestamp'] < SECONDS_IN_DAY * lookback_days and
        this_flight_number == f.get('dump_flight_number', '')]

    if (
        len(relevant_flights) > 1 and
        this_delay_seconds >= min_this_delay_minutes*SECONDS_IN_MINUTE):
      delay_seconds_list = [
          f['calcd_delay_seconds'] for f in relevant_flights
          if f.get('calcd_calculable_delay')]
      delay_unknown_count = len(relevant_flights) - len(delay_seconds_list)
      delay_ontime_count = len([d for d in delay_seconds_list if not d])
      delay_early_count = len([d for d in delay_seconds_list if d < 0])
      delay_late_count = len([d for d in delay_seconds_list if d > 0])

      delay_late_avg_sec = 0
      delay_late_max_sec = 0

      superlative = False
      if delay_late_count > 1:
        delay_late_avg_sec = sum(
            [d for d in delay_seconds_list if d > 0]) / delay_late_count

        # max / min excluding this flight
        delay_late_max_sec = max([d for d in delay_seconds_list[:-1] if d > 0])
        delay_late_min_sec = min([d for d in delay_seconds_list[:-1] if d > 0])

        if delay_late_max_sec > 0:
          if this_delay_seconds > delay_late_max_sec:
            delay_keyword = 'longest'
            superlative = True
          if this_delay_seconds < delay_late_min_sec:
            delay_keyword = 'shortest'
            superlative = True

      overall_stats_elements = []
      if delay_early_count:
        overall_stats_elements.append('%d early' % delay_early_count)
      if delay_ontime_count:
        overall_stats_elements.append('%d ontime' % delay_ontime_count)
      if delay_late_count:
        overall_stats_elements.append('%d late' % delay_late_count)
      if delay_unknown_count:
        overall_stats_elements.append('%d unknown' % delay_unknown_count)
      overall_stats_elements.append('%d total' % len(relevant_flights))
      overall_stats_text = '; '.join(overall_stats_elements)

      days_history = (int(
          round(last_timestamp - relevant_flights[0]['calcd_timestamp']) / SECONDS_IN_DAY)
                      + 1)

      late_percentage = delay_late_count / len(relevant_flights)

      if (superlative and
          delay_late_avg_sec >= min_average_delay_minutes * SECONDS_IN_MINUTE):
        messages = [
            'This %s delay is the %s %s has seen in the last %d days (avg delay is %s);'
            ' overall stats: %s' % (
                SecondsToHhMm(this_delay_seconds),
                delay_keyword,
                this_flight_number.strip(),
                days_history,
                SecondsToHhMm(delay_late_avg_sec),
                overall_stats_text)]
      elif (late_percentage > min_late_percentage and
            delay_late_avg_sec >= min_average_delay_minutes * SECONDS_IN_MINUTE):
        # it's just been delayed frequently!
        messages = [
            'With today''s delay of %s, %s is delayed %d%% of the time in the last %d '
            'days for avg delay of %s; overall stats: %s' % (
                SecondsToHhMm(this_delay_seconds),
                this_flight_number.strip(),
                int(100 * late_percentage),
                days_history,
                SecondsToHhMm(delay_late_avg_sec),
                overall_stats_text)]
  return messages


def FlightInsights(flights):
  """Identifies an interesting attribute or pattern about the most recently seen flight.

  Generates a possibly-empty list of messages about the flight.

  Args:
    flights: List of all flights where the last flight in the list is the focus flight
        for which we are trying to identify something interesting.

  Returns:
    List of printable strings (with embedded new line characters) for something
    interesting about the flight; if there isn't anything interesting, returns an empty
    list.
  """
  messages = []

  # This flight number was last seen x days ago
  messages.extend(FlightInsightLastSeen(flights, days_ago=2))

  # Yesterday this same flight flew a materially different type of aircraft
  messages.extend(FlightInsightDifferentAircraft(flights, percent_size_difference=0.1))

  # This is the 3rd flight to the same destination in the last hour
  messages.extend(FlightInsightNthFlight(flights, hours=1, min_multiple_flights=2))

  # This is the [lowest / highest] [speed / altitude / climbrate] in the last 24 hours
  messages.extend(FlightInsightSuperlativeAttribute(
      flights, 'speed', 'groundspeed', SPEED_UNITS, ['slowest', 'fastest'], hours=24))
  messages.extend(FlightInsightSuperlativeAttribute(
      flights, 'altitude', 'altitude', DISTANCE_UNITS, ['lowest', 'highest'], hours=24))
  messages.extend(FlightInsightSuperlativeVertrate(flights))

  # First instances: destination, first aircraft, etc.
  messages.extend(FlightInsightFirstInstance(
      flights, 'destination_iata', 'destination', days=7,
      additional_descriptor_fcn=lambda f: f['destination_friendly']))
  messages.extend(FlightInsightFirstInstance(
      flights, 'origin_iata', 'origin', days=7,
      additional_descriptor_fcn=lambda f: f['origin_friendly']))
  messages.extend(FlightInsightFirstInstance(
      flights, 'airline_short_name', 'airline', days=7))
  messages.extend(FlightInsightFirstInstance(
      flights, 'aircraft_type_code', 'aircraft', days=7,
      additional_descriptor_fcn=lambda f: f['aircraft_type_friendly']))

  # This is the longest / shortest delay this flight has seen in the last 30 days at
  # 2h5m; including today, this flight has been delayed x of the last y times.
  messages.extend(FlightInsightDelays(
      flights, min_late_percentage=0.75,
      min_this_delay_minutes=0,
      min_average_delay_minutes=0))

  # flight UAL1 (n=5) has a delay frequency in the 72nd %tile, with 100% of flights
  # delayed an average of 44m over the last 4d13h
  messages.extend(FlightInsightGroupPercentile(
      flights,
      group_function=lambda flight: DictGetReplaceNone(
          flight, 'dump_flight_number', KEY_NOT_PRESENT_STRING).strip(),
      value_function=PercentDelay,
      value_string_function=
      lambda flights, value: '%d%% of flights delayed an average of %s'
      % (round(value*100), SecondsToHhMm(AverageDelay(flights))),
      group_label='flight',
      value_label='delay frequency',
      min_days=1,
      min_this_group_size=4,
      min_comparison_group_size=0,
      min_group_qty=0,
      lookback_days=30,
      percentile_low=None,
      percentile_high=70))

  # flight UAL1 (n=5) has a delay time in the 90th %tile, with average delay of
  # 2h17m over the last 4d13h
  messages.extend(FlightInsightGroupPercentile(
      flights,
      group_function=lambda flight: DictGetReplaceNone(
          flight, 'dump_flight_number', KEY_NOT_PRESENT_STRING).strip(),
      value_function=AverageDelay,
      value_string_function=
      lambda flights, value: 'average delay of %s' % SecondsToHhMm(value),
      group_label='flight',
      value_label='delay time',
      min_days=1,
      min_this_group_size=4,
      min_comparison_group_size=0,
      min_group_qty=0,
      lookback_days=30,
      percentile_low=10,
      percentile_high=90))

  messages = [
      Screenify(textwrap.wrap(t.upper(), width=SPLITFLAP_CHARS_PER_LINE))
      for t in messages]

  return messages


def MessageboardHistogramsTestHarness():
  """Test harness to generate messageboard histograms."""
  flights = UnpickleFlights(PICKLEFILE_30D)
  messages = MessageboardHistograms(flights, 'aircraft', '30d', 'all', False)
  for message in messages:
    print(message)


def HistogramSettingsHours(how_much_history):
  """Extracts the desired history (in hours) from the histogram configuration string.

  Args:
    how_much_history: string from the histogram config file.

  Returns:
    Number of hours of history to include in the histogram.
  """
  if how_much_history == 'today':
    hours = HoursSinceMidnight()
  elif how_much_history == '24h':
    hours = 24
  elif how_much_history == '7d':
    hours = 7 * HOURS_IN_DAY
  elif how_much_history == '30d':
    hours = 30 * HOURS_IN_DAY
  else:




                            <----SKIPPED LINES---->




  Args:
    max_screens: string from the histogram config file.

  Returns:
    Number of maximum number of screens to display for a splitflap histogram.
  """
  if max_screens == '_1':
    screen_limit = 1
  elif max_screens == '_2':
    screen_limit = 2
  elif max_screens == '_5':
    screen_limit = 5
  elif max_screens == 'all':
    screen_limit = 0  # no limit on screens
  else:
    LogMessage('Histogram form has invalid value for max_screens: %s' % max_screens)
    screen_limit = 1
  return screen_limit


def HistogramSettingsKeySortTitle(which, max_altitude=45000, max_distance=3300):
  """Provides the arguments necessary to generate a histogram from the config string.

  The same parameters are used to generate either a splitflap text or web-rendered
  histogram in terms of the histogram title, the keyfunction, and how to sort the keys.
  For a given histogram name (based on the names defined in the histogram config file),
  this provides those parameters.

  Args:
    which: string from the histogram config file indicating the histogram to provide
        settings for.
    max_altitude: indicates the maximum altitude that should be included on the
        altitude labels.
    max_distance: indicates the maximum distance that should be included on the
        distance labels.

  Returns:
    A 3-tuple of the parameters used by either CreateSingleHistogramChart or
    MessageboardHistogram, of the keyfunction, sort, and title.
  """





  if which == 'destination':
    key = lambda k: k.get('destination_iata', KEY_NOT_PRESENT_STRING)
    sort = 'value'
    title = 'Destination'
  elif which == 'origin':
    key = lambda k: k.get('origin_iata', KEY_NOT_PRESENT_STRING)
    sort = 'value'
    title = 'Origin'
  elif which == 'hour':
    key = lambda k: datetime.datetime.fromtimestamp(
        k.get('calcd_timestamp', KEY_NOT_PRESENT_STRING), TZ).strftime('%H')
    sort = 'key'
    title = 'Hour'
  elif which == 'airline':
    key = lambda k: k.get('airline_short_name', KEY_NOT_PRESENT_STRING)
    sort = 'value'
    title = 'Airline'
  elif which == 'aircraft':
    key = lambda k: k.get('aircraft_type_code', KEY_NOT_PRESENT_STRING)
    sort = 'value'
    title = 'Aircraft'
  elif which == 'altitude':
    key = lambda k: '%2d'%int(k['altitude']/1000)
    sort = ['%2d'%x for x in range(0, int((max_altitude+1)/1000))]
    title = 'Altitude (1000s of ft)'
  elif which == 'bearing':
    key = lambda k: ConvertBearingToCompassDirection(k['track'], pad=True, length=3)

    sort = [d.rjust(3) for d in DIRECTIONS_16]
    title = 'Bearing'
  elif which == 'distance':
    key = lambda k: '%2d'%int(k['calcd_min_feet']/100)
    sort = ['%2d'%x for x in range(0, int((max_distance+1)/100))]
    title = 'Min Dist (100s of ft)'
  elif which == 'day_of_week':
    key = lambda k: datetime.datetime.fromtimestamp(
        k.get('calcd_timestamp', KEY_NOT_PRESENT_STRING), TZ).strftime('%a')
    sort = DAYS_OF_WEEK
    title = 'Day of Week'
  elif which == 'day_of_month':
    key = lambda k: datetime.datetime.fromtimestamp(
        k.get('calcd_timestamp', KEY_NOT_PRESENT_STRING), TZ).strftime('%d')
    today_day = datetime.datetime.now(TZ).day
    days = list(range(today_day, 0, -1))  # today down to the first of the month
    days.extend(range(31, today_day, -1))  # 31st of the month down to day after today
    days = [str(d).rjust(2) for d in days]
    sort = days
    title = 'Day of Month'
  else:
    LogMessage(
        'Histogram form has invalid value for which_histograms: %s' % which)
    return HistogramSettingsKeySortTitle(
        'destination', max_altitude=max_altitude, max_distance=max_distance)

  return (key, sort, title)


def ImageHistograms(
    flights,
    which_histograms,
    how_much_history,
    filename_prefix=HISTOGRAM_IMAGE_PREFIX,
    filename_suffix=HISTOGRAM_IMAGE_SUFFIX):
  """Generates multiple split histogram images.

  Args:
    flights: the iterable of the raw data from which the histogram will be generated;
        each element of the iterable is a dictionary, that contains at least the key
        'calcd_display_time', and depending on other parameters, also potentially
        'calcd_min_feet' amongst others.
    which_histograms: string paramater indicating which histogram(s) to generate, which
        can be either the special string 'all', or a string linked to a specific
        histogram.
    how_much_history: string parameter taking a value among ['today', '24h', '7d', '30d].
    filename_prefix: this string indicates the file path and name prefix for the images
        that are created. File names are created in the form [prefix]name.[suffix], i.e.:
        if the prefix is histogram_ and the suffix is png, then the file name might be
        histogram_aircraft.png.
    filename_suffix: see above; also interpreted by savefig to generate the correct
        format.

  Returns:
    List of the names of the histograms generated.
  """
  hours = HistogramSettingsHours(how_much_history)

  histograms_to_generate = []
  if which_histograms in ['destination', 'all']:
    histograms_to_generate.append({'generate': 'destination'})
  if which_histograms in ['origin', 'all']:




                            <----SKIPPED LINES---->




        exhaustive=histogram.get('exhaustive', False))
    filename = filename_prefix + histogram['generate'] + '.' + filename_suffix
    matplotlib.pyplot.savefig(filename)
    matplotlib.pyplot.close()

  histograms_generated = [h['generate'] for h in histograms_to_generate]
  return histograms_generated


def MessageboardHistograms(
    flights,
    which_histograms,
    how_much_history,
    max_screens,
    data_summary):
  """Generates multiple split flap screen histograms.

  Args:
    flights: the iterable of the raw data from which the histogram will be generated;
        each element of the iterable is a dictionary, that contains at least the key
        'calcd_display_time', and depending on other parameters, also potentially
        'calcd_min_feet' amongst others.
    which_histograms: string paramater indicating which histogram(s) to generate, which
        can be either the special string 'all', or a string linked to a specific
        histogram.
    how_much_history: string parameter taking a value among ['today', '24h', '7d', '30d].
    max_screens: string parameter taking a value among ['_1', '_2', '_5', or 'all'].
    data_summary: parameter that evaluates to a boolean indicating whether the data
        summary screen in the histogram should be displayed.

  Returns:
    Returns a list of printable strings (with embedded new line characters) representing
    the histogram, for each screen in the histogram.
  """
  messages = []

  hours = HistogramSettingsHours(how_much_history)
  screen_limit = HistogramSettingsScreens(max_screens)

  histograms_to_generate = []
  if which_histograms in ['destination', 'all']:
    histograms_to_generate.append({
        'generate': 'destination',

        'columns': 3})
  if which_histograms in ['origin', 'all']:
    histograms_to_generate.append({
        'generate': 'origin',

        'columns': 3})
  if which_histograms in ['hour', 'all']:
    histograms_to_generate.append({
        'generate': 'hour',
        'columns': 4,
        'suppress_percent_sign': True,
        'column_divider': '|'})
  if which_histograms in ['airline', 'all']:
    histograms_to_generate.append({
        'generate': 'airline'})
  if which_histograms in ['aircraft', 'all']:
    histograms_to_generate.append({
        'generate': 'aircraft'})
  if which_histograms in ['altitude', 'all']:
    histograms_to_generate.append({
        'generate': 'altitude',
        'columns': 3})
  if which_histograms in ['bearing', 'all']:
    histograms_to_generate.append({
        'generate': 'bearing',

        'columns': 3})
  if which_histograms in ['distance', 'all']:
    histograms_to_generate.append({
        'generate': 'distance',
        'columns': 3})
  if ((which_histograms == 'all' and how_much_history == '7d')
      or which_histograms == 'day_of_week'):
    histograms_to_generate.append({
        'generate': 'day_of_week',
        'columns': 3})

  if ((which_histograms == 'all' and how_much_history == '30d')
      or which_histograms == 'day_of_month'):
    histograms_to_generate.append({
        'generate': 'day_of_month',
        'columns': 4,
        'suppress_percent_sign': True,
        'column_divider': '|'})


  for histogram in histograms_to_generate:
    this_histogram = which_histograms
    if this_histogram == 'all':
      this_histogram = histogram['generate']
    (key, sort, title) = HistogramSettingsKeySortTitle(this_histogram)

    histogram = MessageboardHistogram(
        flights,
        key,
        sort,
        title,
        screen_limit=screen_limit,
        columns=histogram.get('columns', 2),
        suppress_percent_sign=histogram.get('suppress_percent_sign', False),
        column_divider=histogram.get('column_divider', ' '),
        data_summary=data_summary,
        hours=hours)

    messages.extend(histogram)

  return messages


def MessageboardHistogram(
    data,
    keyfunction,
    sort_type,
    title,
    screen_limit=1,
    columns=2,
    column_divider=' ',
    data_summary=False,
    hours=0,
    suppress_percent_sign=False):

  """Generates a text representation of one histogram that can be rendered on the display.

  Args:
    data: the iterable of the raw data from which the histogram will be generated;
        each element of the iterable is a dictionary, that contains at least the key
        'calcd_display_time', and depending on other parameters, also potentially
        'calcd_min_feet' amongst others.
    keyfunction: the function that determines how the key or label of the histogram
        should be generated; it is called for each element of the data iterable. For
        instance, to simply generate a histogram on the attribute 'heading',
        keyfunction would be lambda a: a['heading'].
    sort_type: determines how the keys (and the corresponding values) are sorted; see
        GenerateHistogramData docstring for details
    title: string title, potentially truncated to fit, to be displayed for the histogram
    screen_limit: maximum number of screens to be displayed for the histogram; a value
        of zero is interpreted to mean no limit on screens.
    columns: number of columns of data to be displayed for the histogram; note that the
        keys of the histogram may need to be truncated in length to fit the display
        as more columns are squeezed into the space
    column_divider: string for the character(s) to be used to divide the columns
    data_summary: boolean indicating whether to augment the title with a second header
        line about the data presented in the histogram
    hours: integer indicating the oldest data to be included in the histogram
    suppress_percent_sign: boolean indicating whether to suppress the percent sign
        in the data (but to add it to the title) to reduce the amount of string
        truncation potentially necessary for display of the keys



  Returns:
    Returns a list of printable strings (with embedded new line characters) representing
    the histogram.
  """
  title_lines = 1
  if data_summary:
    title_lines += 1
  available_entries_per_screen = (SPLITFLAP_LINE_COUNT - title_lines) *  columns
  available_entries_total = available_entries_per_screen * screen_limit
  (values, keys, unused_filtered_data) = GenerateHistogramData(
      data, keyfunction, sort_type, truncate=available_entries_total, hours=hours)

  screen_count = math.ceil(len(keys) / available_entries_per_screen)

  column_width = int(
      (SPLITFLAP_CHARS_PER_LINE - len(column_divider)*(columns - 1)) / columns)





  # i.e.: ' 10%' or ' 10', depending on suppress_percent_sign
  value_size = 3
  printed_percent_sign = ''







  augment_title_units = ' %'
  if not suppress_percent_sign:
    value_size += 1
    printed_percent_sign = '%'
    augment_title_units = ''
  column_key_width = column_width - value_size

  total = sum(values)

  if data_summary:
    if hours:
      hours_of_data = min(hours, DataHistoryHours(data))
    else:
      hours_of_data = DataHistoryHours(data)
    time_horizon_text = 'Last %s' % SecondsToDdHh(hours_of_data * SECONDS_IN_HOUR)

    summary_text = '%s (n=%d)' % (time_horizon_text, sum(values))
    summary_text = summary_text.center(SPLITFLAP_CHARS_PER_LINE)

  split_flap_boards = []
  for screen in range(screen_count):
    if screen_count == 1:
      counter = ''
    else:
      counter = ' (%d/%d)' % (screen+1, screen_count)
    screen_title = '%s%s%s' % (
        title[:SPLITFLAP_CHARS_PER_LINE - len(counter) - len(augment_title_units)],
        augment_title_units, counter)

    screen_title = screen_title.center(SPLITFLAP_CHARS_PER_LINE)
    start_index = screen*available_entries_per_screen
    end_index = min((screen+1)*available_entries_per_screen-1, len(keys)-1)
    number_of_entries = end_index - start_index + 1
    number_of_lines = math.ceil(number_of_entries / columns)

    lines = []

    lines.append(screen_title.upper())
    if data_summary:
      lines.append(summary_text.upper())
    for line_index in range(number_of_lines):
      key_value = []
      for column_index in range(columns):
        index = start_index + column_index*number_of_lines + line_index
        if index <= end_index:
          # If the % is >=1%, display right-justified 2 digit percent, i.e. ' 5%'
          # Otherwise, if it rounds to at least 0.1%, display i.e. '.5%'
          if round(values[index]/total*100) >= 1:
            value_string = '%2d' % round(values[index]/total*100)
          elif round(values[index]/total*1000)/10 >= 0.1:
            value_string = ('%.1f' % (round(values[index]/total*1000)/10))[1:]
          else:







            value_string = ' 0'
          key_value.append('%s %s%s' % (
              str(keys[index])[:column_key_width].ljust(column_key_width),
              value_string,
              printed_percent_sign))

      line = (column_divider.join(key_value)).upper()
      lines.append(line)

    split_flap_boards.append(Screenify(lines))

  return split_flap_boards


def GetNowInTimeZoneOfArbtraryFlight(flights):
  """Returns datetime now in the timezone of an arbitrarily selected flight.

  Args:
    flights: iterable with dictionaries of the flight details.

  Returns:
    Timezone-aware datetime.
  """
  arbitrary_time_string = flights[0]['calcd_display_time']
  arbitrary_time = datetime.datetime.strptime(
      arbitrary_time_string, '%Y-%m-%d %H:%M:%S.%f%z')
  tz = arbitrary_time.tzinfo
  now = datetime.datetime.now(tz)
  return now


def ConvertHourStringTimeString(flight):
  """Convert calcd_display_time on flight to a string like '12a' or ' 1p'."""
  time_string = flight.get('calcd_display_time', '')
  if time_string:
    hour_string = time_string[11:13]
    hour_0_23 = int(hour_string)
    is_pm = int(hour_0_23/12) == 1
    hour_number = hour_0_23 % 12
    if hour_number == 0:
      hour_number = 12
    out_string = str(hour_number).rjust(2)
    if is_pm:
      out_string += 'p'
    else:
      out_string += 'a'
  else:
    out_string = KEY_NOT_PRESENT_STRING
  return out_string


def TruncatePickledFlights(file=PICKLEFILE_30D, days=30):
  """Truncate the pickled flights file to have at most some number of days of history.

  Delete old flights from the given pickle repository.

  Args:
    file: name (potentially including path) of the pickled file
    days: maximum number of days (measured from now) to include in the file
  """
  flights = UnpickleFlights(file)
  now = GetNowInTimeZoneOfArbtraryFlight(flights)

  # We want to be cautious, not overwriting the original file until we're sure this has
  # completed; this is why we use the tmp file
  tmp_file = file+'.tmp'
  for flight in flights:
    if HoursSinceFlight(now, flight['calcd_display_time']) <= days*HOURS_IN_DAY:
      PickleFlight(flight, tmp_file)

  shutil.move(tmp_file, file)


def UnpickleFlights(file):
  """Load a repository of pickled flight data into memory.

  Args:
    file: name (potentially including path) of the pickled file

  Returns:
    Return a list of all the flights, in the same sequence as written to the file.
  """
  flights = []
  f = open(file, 'rb')
  try:
    while True:
      data = pickle.load(f)
      flights.append(data)
  except EOFError:
    pass

  f.close()

  return flights


def PickleFlight(flight, file):
  """Append one pickled flight to the end of binary file.

  Args:
    flight: data to pickle
    file: name (potentially including path) of the pickled file
  """
  try:
    with open(file, 'ab') as f:
      f.write(pickle.dumps(flight))

  except IOError:
    LogMessage('Unable to append pickle ' + file)


def UpdateAircraftList(persistent_nearby_aircraft, current_nearby_aircraft, now):
  """Identifies newly seen aircraft and removes aircraft that haven't been seen recently.

  Updates persistent_nearby_aircraft as follows: flights that have been last seen more
  than PERSISTENCE_SECONDS seconds ago are removed; new flights in current_nearby_aircraft
  are added. Also identifies newly-seen aircraft and updates the last-seen timestamp of
  flights that have been seen again.

  Args:
    persistent_nearby_aircraft: dictionary where keys are flight numbers, and the values
      are the time the flight was last seen.
    current_nearby_aircraft: dictionary where keys are flight numbers, and the values are
      themselves dictionaries with key-value pairs about that flight, with at least one of
      the kv-pairs being the time the flight was seen.
    now: the timestamp of the flights in the current_nearby_aircraft.

  Returns:
    A list of newly-nearby flight numbers.
  """
  newly_nearby_flight_numbers = []
  for flight_number in current_nearby_aircraft:
    if flight_number not in persistent_nearby_aircraft:
      newly_nearby_flight_numbers.append(flight_number)
    persistent_nearby_aircraft[flight_number] = now
  flights_to_delete = []
  for flight_number in persistent_nearby_aircraft:
    if (flight_number not in current_nearby_aircraft
        and (now - persistent_nearby_aircraft[flight_number]) > PERSISTENCE_SECONDS):
      flights_to_delete.append(flight_number)
  for flight_number in flights_to_delete:
    del persistent_nearby_aircraft[flight_number]
  return newly_nearby_flight_numbers


def ScanForNewFlights(persistent_nearby_aircraft):
  """Determines if there are any new aircraft in the radio message.

  The radio is continuously dumping new json messages to the Raspberry pi with all the
  flights currently observed. This function picks up the latest radio json, and for
  any new nearby flights - there should generally be at most one new flight on each
  pass through - gets additional flight data from FlightAware and augments the flight
  definition with the relevant fields to keep.

  Args:
    persistent_nearby_aircraft: dictionary where keys are flight numbers, and the values
      are the time the flight was last seen.

  Returns:
    A 3-tuple:
    - updated persistent_nearby_aircraft
    - current_nearby_aircraft
    - (possibly empty) dictionary of flight attributes of the new flight upon its
      first observation.
  """
  flight_details = {}

  dump_json = ReadFile(DUMP_JSON_FILE, log_exception=True)

  if dump_json:
    (current_nearby_aircraft, now) = ParseDumpJson(dump_json)

    newly_nearby_flight_numbers = UpdateAircraftList(
        persistent_nearby_aircraft, current_nearby_aircraft, now)

    if newly_nearby_flight_numbers:

      if len(newly_nearby_flight_numbers) > 1:
        newly_nearby_flight_numbers_str = ', '.join(newly_nearby_flight_numbers)
        newly_nearby_flight_details_str = '\n'.join(
            [str(current_nearby_aircraft[f]) for f in newly_nearby_flight_numbers])
        LogMessage('Multiple newly-nearby flights: %s\n%s' % (
            newly_nearby_flight_numbers_str, newly_nearby_flight_details_str))

      flight_number = newly_nearby_flight_numbers[0]

      if flight_number:
        flight_aware_json = GetFlightAwareJson(flight_number)
      else:
        flight_aware_json = {}

      if flight_aware_json:
        flight_details = ParseFlightAwareJson(flight_aware_json)
      else:
        LogMessage('No json returned from Flightaware for flight: %s' % flight_number)
        flight_details = {}

      # Augment FlightAware details with radio / radio-derived details
      flight_details.update(current_nearby_aircraft[flight_number])

      # Augment FlightAware details with useful displayable attributes
      AugmentWithDisplayableAirline(flight_details)
      AugmentWithDisplayableAircraft(flight_details)
      AugmentWithDisplayableOriginDestination(flight_details)
      AugmentWithDisplayableDelay(flight_details)
      AugmentWithDisplayableTimeRemaining(flight_details)

  return (persistent_nearby_aircraft, current_nearby_aircraft, flight_details)


def ReadSettings(filename):
  """Parse delimited string of settings in file to a dict of key value pairs.

  Parses a string like 'distance=1426;altitude=32559;on=23;off=24;delay=15;insights=all;'
  into key value pairs.

  Args:
    filename: string of the filename to open, potentially also including the full path.

  Returns:
    Dict of key value pairs contained in the setting file; empty dict if file not
    available or if delimiters missing.
  """

  settings_dict = {}
  settings = ReadFile(filename)
  for setting in settings.split(';'):
    if '=' in setting:
      kv_list = setting.split('=')
      k = kv_list[0]
      v = kv_list[1]
      if v.isdigit():
        v = int(v)
      settings_dict[k] = v
  return settings_dict


def FlightMeetsDisplayCriteria(flight, configuration):
  """Returns boolean indicating whether the screen is currently accepting new flight data.

  Based on the configuration file, determines whether the flight data should be displayed.
  Specifically, the configuration:
  - may include 'enabled' indicating whether screen should be driven at all
  - should include 'on' & 'off' parameters indicating minute (from midnight) of operation
  - should include altitude & elevation parameters indicating max values of interest

  Args:
    flight: dictionary of flight attributes.
    configuration: dictionary of configuration attributes.

  Returns:
    Boolean as described.
  """
  flight_meets_criteria = 'enabled' in configuration

  flight_altitude = DictGetReplaceNone(flight, 'altitude', float('inf'))
  config_max_altitude = configuration['altitude']

  if flight_altitude > config_max_altitude:
    flight_meets_criteria = False
  else:
    flight_distance = DictGetReplaceNone(flight, 'calcd_min_feet', float('inf'))
    config_max_distance = configuration['distance']
    if flight_distance > config_max_distance:
      flight_meets_criteria = False
    else:
      flight_timestamp = flight['calcd_timestamp']
      dt = datetime.datetime.fromtimestamp(flight_timestamp, TZ)
      minute_of_day = dt.hour * MINUTES_IN_HOUR + dt.minute
      if minute_of_day < configuration['on'] or minute_of_day > configuration['off']:
        flight_meets_criteria = False

  return flight_meets_criteria


def MaintainRollingWebLog(message, max_count, filename=ROLLING_MESSAGE_FILE):
  """Maintains a rolling text file of at most max_count printed messages.

  Newest data at top and oldest data at the end, of at most max_count messages,
  where the delimiter between each message is identified by a special fixed string.

  Args:
    message: text message to prepend to the file.
    max_count: maximum number of messages to keep in the file; the max_count+1st message
        is deleted.
    filename: the file to update.
  """
  rolling_log_header = '='*len(SPLITFLAP_HEADER)
  existing_file = ReadFile(filename)
  log_message_count = existing_file.count(rolling_log_header)
  if log_message_count >= max_count:
    message_start_list = [i for i in range(0, len(existing_file))
                          if existing_file[i:].startswith(rolling_log_header)]
    existing_file_to_keep = existing_file[:message_start_list[max_count - 1]]
  else:
    existing_file_to_keep = existing_file

  t = datetime.datetime.now(TZ).strftime('%m/%d/%Y, %H:%M:%S')
  new_message = (
      '\n'.join([rolling_log_header, t, '', message])
      + '\n' + existing_file_to_keep)
  try:
    with open(filename, 'w') as f:
      f.write(new_message)
  except IOError:
    LogMessage('Unable to maintain rolling log at ' + filename)





def CurrentFlightPosition(flights, last_location_detail):
  focus_flight = flights[-1]
  focus_flight_number = focus_flight['dump_flight_number']
  lines = []

  new_flight = (
      not last_location_detail or
      last_location_detail.get('dump_flight_number') != focus_flight_number)

  def TrackMissingElements(key, missing):
    value = focus_flight.get(key)
    if value is None:
      missing.append(key)
    return value, missing

  missing_elements = []
  if new_flight:
    calculations_count = 0
    (track, missing_elements) = TrackMissingElements('track', missing_elements)
    (altitude, missing_elements) = TrackMissingElements('altitude', missing_elements)
    (speed, missing_elements) = TrackMissingElements('speed', missing_elements)
    (lat, missing_elements) = TrackMissingElements('lat', missing_elements)
    (lon, missing_elements) = TrackMissingElements('lon', missing_elements)
    (vert_rate, missing_elements) = TrackMissingElements('vert_rate', missing_elements)
    (last_timestamp, missing_elements) = TrackMissingElements(
        'calcd_timestamp', missing_elements)
  else:
    # this is a flight we've already started tracking loc of, so we know
    # all elements are present in last_location_detail
    calculations_count = last_location_detail['calculations_count'] + 1
    track = last_location_detail['track']
    altitude = last_location_detail['altitude']
    speed = last_location_detail['speed']
    lat = last_location_detail['lat']
    lon = last_location_detail['lon']
    vert_rate = last_location_detail['vert_rate']
    last_timestamp = last_location_detail['calcd_timestamp']

  if not missing_elements:
    now = time.time()
    meters_per_second = speed*METERS_PER_SECOND_IN_KNOTS
    elapsed_sec = now - last_timestamp
    meters_traveled = meters_per_second * elapsed_sec
    plane_pos = TrajectoryLatLon((lat, lon), meters_traveled, track)
    (az_degrees, alt_degrees, distance, crow_distance) = Angles(
        HOME, HOME_ALT, plane_pos, altitude / FEET_IN_METER)


    if new_flight:
      lines.append('='*80)
      lines.append('New flight: %s' % focus_flight_number)
      lines.append('='*80)
    else:
      lines.append('Existing flight: %s' % focus_flight_number)

    lines.append(
        'track: %.2fdg; speed: %.2f m/s; t since last pos: %.4fsec; covered %.2f meters' %
        (track, meters_per_second, elapsed_sec, meters_traveled))

    if new_flight:
      lines.append('Orig pos: (%.4f, %.4f); Alt: %d; hdist: %d' % (
          focus_flight.get('lat', 0),
          focus_flight.get('lon', 0),
          focus_flight.get('altitude', 0),
          focus_flight.get('distance_meters', 0)))  #WHY = 0 on first data pull?
    else:
      lines.append(
          'Last calc: (%.4f, %.4f); Alt: %d; hdist: %d; '
          'dist: %d; Az: %.4fdg; El: %.4fdg' % (
              last_location_detail.get('lat', 0),
              last_location_detail.get('lon', 0),
              last_location_detail.get('altitude', 0),
              last_location_detail.get('distance_meters', 0),
              last_location_detail.get('crow_distance', 0),
              last_location_detail.get('azimuth', 0),
              last_location_detail.get('elevation', 0)))


    # save the calc'd location at the calc'd time
    last_location_detail = {
        'track': track,
        'altitude': altitude,
        'speed': speed,
        'lat': plane_pos[0],
        'lon': plane_pos[1],
        'vert_rate': vert_rate,
        'calcd_timestamp': now,
        'azimuth': az_degrees,
        'elevation': alt_degrees,
        'distance_meters': distance,
        'crow_distance': crow_distance,
        'dump_flight_number': focus_flight_number,
        'calculations_count': calculations_count}

    lines.append(
        'Curr calc: (%.4f, %.4f); Alt: %d; hdist: %d; '
        'dist: %d; Az: %.4fdg; El: %.4fdg' % (
            last_location_detail.get('lat', 0),
            last_location_detail.get('lon', 0),
            last_location_detail.get('altitude', 0),
            last_location_detail.get('distance_meters', 0),
            last_location_detail.get('crow_distance', 0),
            last_location_detail.get('azimuth', 0),
            last_location_detail.get('elevation', 0)))


    LogMessage('\n'.join(lines))
  elif focus_flight == last_location_detail['dump_flight_number']:
    LogMessage('Flight %s missing %s in calculating the current position' % (
        focus_flight, str(missing_elements)))

  return last_location_detail










def TriggerHistograms(flights, histogram_settings):
  """Triggers the text-based or web-based histograms.

  Based on the histogram settings, determines whether to generate text or image histograms
  (or both). For image histograms, also generates empty images for the histograms not
  created so that broken image links are not displayed in the webpage.

  Args:
    flights: List of flight attribute dictionaries.
    histogram_settings: Dictionary of histogram parameters.

  Returns:
    List of histogram messages, if text-based histograms are selected; empty list
    otherwise.
  """
  histogram_messages = []

  if histogram_settings['type'] in ('messageboard', 'both'):




                            <----SKIPPED LINES---->




        histogram_settings['histogram_history'],
        histogram_settings['histogram_max_screens'],
        histogram_settings.get('histogram_data_summary', False))
  if histogram_settings['type'] in ('images', 'both'):
    histograms_generated = ImageHistograms(
        flights,
        histogram_settings['histogram'],
        histogram_settings['histogram_history'])
    all_available_histograms = [
        'destination', 'origin', 'hour', 'airline', 'aircraft', 'altitude',
        'bearing', 'distance', 'day_of_week', 'day_of_month']
    for histogram in all_available_histograms:
      if histogram not in histograms_generated:
        missing_filename = (
            HISTOGRAM_IMAGE_PREFIX + histogram + '.' + HISTOGRAM_IMAGE_SUFFIX)
        shutil.copyfile(HISTOGRAM_EMPTY_IMAGE_FILE, missing_filename)

  return histogram_messages























































































































































































































































































































































































































































def main():
  """Traffic cop between incoming radio flight messages, configuration, and messageboard.

  This is the main logic, checking for new flights, augmenting the radio signal with
  additional web-scraped data, and generating messages in a form presentable to the
  messageboard.
  """





  already_running_id = CheckIfProcessRunning('messageboard.py')
  if already_running_id:
    os.kill(already_running_id, signal.SIGKILL)

  TruncatePickledFlights(file=PICKLEFILE_30D)
  flights = UnpickleFlights(PICKLEFILE_30D)
  configuration = ReadSettings(CONFIG_FILE)
  last_distance = configuration.get('distance')
  last_altitude = configuration.get('altitude')




































  persistent_nearby_aircraft = {} # key = flight number; value = last seen



  # Next up to print is element 0; this is a list of tuples:
  # Element#1: flag indicating the type of message that this is
  # Element#2: the message itself
  message_queue = []
  next_message_time = time.time()
  last_location_detail = {}

  while True:










    new_configuration = ReadSettings(CONFIG_FILE)
    if (new_configuration.get('distance') != configuration.get('distance') or
        new_configuration.get('altitude') != configuration.get('altitude')):


      last_distance = configuration.get('distance')
      last_altitude = configuration.get('altitude')
      SaveTimeOfDayHistogramPng(
          flights,
          new_configuration['distance'],
          new_configuration['altitude'],
          7,
          last_max_distance_feet=last_distance,
          last_max_altitude_feet=last_altitude)
    configuration = new_configuration
























































































    histogram = ReadSettings(HOURLY_HISTOGRAM_FILE)
    if histogram:






      histogram_messages = TriggerHistograms(flights, histogram)
      histogram_messages = [(FLAG_MSG_HISTOGRAM, m) for m in histogram_messages]
      message_queue.extend(histogram_messages)
      os.remove(HOURLY_HISTOGRAM_FILE)

    (persistent_nearby_aircraft, current_nearby_aircraft, flight_details) = (
        ScanForNewFlights(persistent_nearby_aircraft))

    if flight_details:
      flights.append(flight_details)
      PickleFlight(flight_details, PICKLEFILE_30D)
      PickleFlight(flight_details, PICKLEFILE_ARCHIVE)
      flight_messages = []

      if FlightMeetsDisplayCriteria(flight_details, configuration):
        flight_messages.append((
            FLAG_MSG_FLIGHT,
            CreateMessageAboutFlight(flight_details)))

        next_message_time = time.time()  # display the next message about this flight now!
        # and delete any queued "interesting" messages about other flights that have
        # not yet displayed, since a newer flight has taken precedence
        message_queue = [m for m in message_queue if m[0] != FLAG_MSG_INTERESTING]

        flight_insights_enabled_string = configuration.get('insights', 'hide')
        if flight_insights_enabled_string in ('one', 'all'):
          insight_messages = FlightInsights(flights)
          if flight_insights_enabled_string == 'one' and insight_messages:
            insight_messages = [random.choice(insight_messages)]
          insight_messages = [(FLAG_MSG_INTERESTING, m) for m in insight_messages]
          flight_messages.extend(insight_messages)

        # i.e.: we need to make sure the final step is to insert the flight message
        # as the first message
        flight_messages.reverse()

        for flight_message in flight_messages:
          message_queue.insert(0, flight_message)


    new_plane = flights[-1]['dump_flight_number'] != last_location_detail.get(
        'dump_flight_number')
    if flights and (new_plane or last_location_detail.get('calculations_count', 0) <= 10):
      last_location_detail = CurrentFlightPosition(
          flights, last_location_detail)

    # check time & if appropriate, display next message from queue
    if time.time() >= next_message_time:





      if message_queue:
        next_message_time += configuration['delay']
        next_message = message_queue.pop(0)


        LogMessage(next_message[1], file=ALL_MESSAGE_FILE)
        MaintainRollingWebLog(next_message[1], 25)

      else:
        next_message_time += 1

    time.sleep(1)



if __name__ == "__main__":



  main()