<----SKIPPED LINES---->




import bs4
import dateutil.relativedelta
import filelock
import numpy
import matplotlib
import matplotlib.pyplot
import psutil
import pycurl
import pytz
import requests
import tzlocal
import unidecode

# This is the directory that stores all the ancillary messageboard configuration files
# that do not need to be exposed via the webserver
MESSAGEBOARD_PATH = '/home/pi/splitflap/'
# This is the directory of the webserver; files placed here are available at
# http://adsbx-custom.local/; files placed in this directly are visible via a browser
WEBSERVER_PATH = '/var/www/html/'

import arduino  # module expects paths to be set before import  # pylint: disable=C0413

RASPBERRY_PI = psutil.sys.platform.title() == 'Linux'
if RASPBERRY_PI:
  import gpiozero  # pylint: disable=E0401
  import RPi.GPIO  # pylint: disable=E0401

SHUTDOWN_SIGNAL = False


SIMULATION = False
SIMULATION_COUNTER = 0
SIMULATION_PREFIX = 'SIM_'
PICKLE_DUMP_JSON_FILE = 'pickle/dump_json.pk'
PICKLE_FA_JSON_FILE = 'pickle/fa_json.pk'
DUMP_JSONS = None  # loaded only if in simulation mode
FA_JSONS = None  # loaded only if in simulation mode

HOME_LAT = 37.64406
HOME_LON = -122.43463
HOME = (HOME_LAT, HOME_LON) # lat / lon tuple of antenna
HOME_ALT = 29  #altitude in meters
RADIUS = 6371.0e3  # radius of earth in meters

FEET_IN_METER = 3.28084
FEET_IN_MILE = 5280
METERS_PER_SECOND_IN_KNOTS = 0.514444

MIN_METERS = 5000/FEET_IN_METER # only planes within this distance will be detailed




                            <----SKIPPED LINES---->




FLAG_INSIGHT_GROUNDSPEED = 3
FLAG_INSIGHT_ALTITUDE = 4
FLAG_INSIGHT_VERTRATE = 5
FLAG_INSIGHT_FIRST_DEST = 6
FLAG_INSIGHT_FIRST_ORIGIN = 7
FLAG_INSIGHT_FIRST_AIRLINE = 8
FLAG_INSIGHT_FIRST_AIRCRAFT = 9
FLAG_INSIGHT_LONGEST_DELAY = 10
FLAG_INSIGHT_FLIGHT_DELAY_FREQUENCY = 11
FLAG_INSIGHT_FLIGHT_DELAY_TIME = 12
FLAG_INSIGHT_AIRLINE_DELAY_FREQUENCY = 13
FLAG_INSIGHT_AIRLINE_DELAY_TIME = 14
FLAG_INSIGHT_DESTINATION_DELAY_FREQUENCY = 15
FLAG_INSIGHT_DESTINATION_DELAY_TIME = 16
FLAG_INSIGHT_HOUR_DELAY_FREQUENCY = 17
FLAG_INSIGHT_HOUR_DELAY_TIME = 18
FLAG_INSIGHT_DATE_DELAY_FREQUENCY = 19
FLAG_INSIGHT_DATE_DELAY_TIME = 20
INSIGHT_TYPES = 21

# GPIO connections - value is the RPi pin number for the error light
GPIO_ERROR_VESTABOARD_CONNECTION = 22
GPIO_ERROR_FLIGHT_AWARE_CONNECTION = 23
GPIO_ERROR_ARDUINO_SERVO_CONNECTION = 24
GPIO_ERROR_ARDUINO_REMOTE_CONNECTION = 25
GPIO_ERROR_BATTERY_CHARGE = 26
GPIO_FAN = 27
GPIO_UNUSED_1 = 5  # for future expansion
GPIO_UNUSED_2 = 6  # for future expansion

TEMP_FAN_TURN_ON_CELSIUS = 65
TEMP_FAN_TURN_OFF_CELSIUS = 55































#if running on raspberry, then need to prepend path to file names
if RASPBERRY_PI:
  PICKLE_FLIGHTS = MESSAGEBOARD_PATH + PICKLE_FLIGHTS
  LOGFILE = MESSAGEBOARD_PATH + LOGFILE
  PICKLE_DUMP_JSON_FILE = MESSAGEBOARD_PATH + PICKLE_DUMP_JSON_FILE
  PICKLE_FA_JSON_FILE = MESSAGEBOARD_PATH + PICKLE_FA_JSON_FILE

  HISTOGRAM_CONFIG_FILE = WEBSERVER_PATH + HISTOGRAM_CONFIG_FILE
  CONFIG_FILE = WEBSERVER_PATH + CONFIG_FILE
  ROLLING_MESSAGE_FILE = WEBSERVER_PATH + ROLLING_MESSAGE_FILE
  ALL_MESSAGE_FILE = WEBSERVER_PATH + ALL_MESSAGE_FILE
  ROLLING_LOGFILE = WEBSERVER_PATH + ROLLING_LOGFILE
  STDERR_FILE = WEBSERVER_PATH + STDERR_FILE
  BACKUP_FILE = WEBSERVER_PATH + BACKUP_FILE
  SERVICE_VERIFICATION_FILE = WEBSERVER_PATH + SERVICE_VERIFICATION_FILE

  HISTOGRAM_IMAGE_PREFIX = WEBSERVER_PATH + WEBSERVER_IMAGE_FOLDER + HISTOGRAM_IMAGE_PREFIX
  HISTOGRAM_EMPTY_IMAGE_FILE = (
      WEBSERVER_PATH + WEBSERVER_IMAGE_FOLDER + HISTOGRAM_EMPTY_IMAGE_FILE)




                            <----SKIPPED LINES---->




SECONDS_IN_HOUR = SECONDS_IN_MINUTE * MINUTES_IN_HOUR
MINUTES_IN_DAY = MINUTES_IN_HOUR * HOURS_IN_DAY
SECONDS_IN_DAY = SECONDS_IN_HOUR * HOURS_IN_DAY

# Units confirmed here:
# www.adsbexchange.com/forum/threads/units-in-the-dump1090-json-file.630617/#post-639541
CLIMB_RATE_UNITS = 'fpm'
#speed units from tracker are knots, based on dump-1090/track.c
#https://github.com/SDRplay/dump1090/blob/master/track.c
SPEED_UNITS = 'kn'
DISTANCE_UNITS = 'ft'  # altitude

# For displaying histograms
# If a key is not present, how should it be displayed in histograms?
KEY_NOT_PRESENT_STRING = 'Unknown'
OTHER_STRING = 'Other' # What key strings should be listed last in sequence?
# What key strings should be listed last in sequence?
SORT_AT_END_STRINGS = [OTHER_STRING, KEY_NOT_PRESENT_STRING]
# What is the sorted sequence of keys for days of week?
DAYS_OF_WEEK = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat']



aircraft_length = {} # in meters
aircraft_length['Airbus A220-100 (twin-jet)'] = 35
aircraft_length['Airbus A300F4-600 (twin-jet)'] = 54.08
aircraft_length['Airbus A319 (twin-jet)'] = 33.84
aircraft_length['Airbus A320 (twin-jet)'] = 37.57
aircraft_length['Airbus A320neo (twin-jet)'] = 37.57
aircraft_length['Airbus A321 (twin-jet)'] = 44.51
aircraft_length['Airbus A321neo (twin-jet)'] = 44.51
aircraft_length['Airbus A330-200 (twin-jet)'] = 58.82
aircraft_length['Airbus A330-300 (twin-jet)'] = 63.67
aircraft_length['Airbus A340-300 (quad-jet)'] = 63.69
aircraft_length['Airbus A350-1000 (twin-jet)'] = 73.79
aircraft_length['Airbus A350-900 (twin-jet)'] = 66.8
aircraft_length['Airbus A380-800 (quad-jet)'] = 72.72
aircraft_length['Boeing 737-400 (twin-jet)'] = 36.4
aircraft_length['Boeing 737-700 (twin-jet)'] = 33.63
aircraft_length['Boeing 737-800 (twin-jet)'] = 39.47
aircraft_length['Boeing 737-900 (twin-jet)'] = 42.11
aircraft_length['Boeing 747-400 (quad-jet)'] = 36.4
aircraft_length['Boeing 747-8 (quad-jet)'] = 76.25
aircraft_length['Boeing 757-200 (twin-jet)'] = 47.3
aircraft_length['Boeing 757-300 (twin-jet)'] = 54.4
aircraft_length['Boeing 767-200 (twin-jet)'] = 48.51
aircraft_length['Boeing 767-300 (twin-jet)'] = 54.94
aircraft_length['Boeing 777 (twin-jet)'] = (63.73 + 73.86) / 2
aircraft_length['Boeing 777-200 (twin-jet)'] = 63.73
aircraft_length['Boeing 777-200LR/F (twin-jet)'] = 63.73
aircraft_length['Boeing 777-300ER (twin-jet)'] = 73.86
aircraft_length['Boeing 787-10 (twin-jet)'] = 68.28
aircraft_length['Boeing 787-8 (twin-jet)'] = 56.72
aircraft_length['Boeing 787-9 (twin-jet)'] = 62.81
aircraft_length['Canadair Regional Jet CRJ-200 (twin-jet)'] = 26.77
aircraft_length['Canadair Regional Jet CRJ-700 (twin-jet)'] = 32.3
aircraft_length['Canadair Regional Jet CRJ-900 (twin-jet)'] = 36.2
aircraft_length['Canadair Challenger 350 (twin-jet)'] = 20.9
aircraft_length['Bombardier Challenger 300 (twin-jet)'] = 20.92
aircraft_length['Embraer 170/175 (twin-jet)'] = (29.90 + 31.68) / 2

aircraft_length['EMBRAER 175 (long wing) (twin-jet)'] = 31.68
aircraft_length['Embraer ERJ-135 (twin-jet)'] = 26.33
aircraft_length['Cessna Caravan (single-turboprop)'] = 11.46

aircraft_length['Cessna Citation II (twin-jet)'] = 14.54
aircraft_length['Cessna Citation V (twin-jet)'] = 14.91
aircraft_length['Cessna Citation X (twin-jet)'] = 22.04
aircraft_length['Cessna Skyhawk (piston-single)'] = 8.28
aircraft_length['Cessna Skylane (piston-single)'] = 8.84
aircraft_length['Cessna Citation Sovereign (twin-jet)'] = 19.35
aircraft_length['Cessna T206 Turbo Stationair (piston-single)'] = 8.61
aircraft_length['Beechcraft Bonanza (33) (piston-single)'] = 7.65
aircraft_length['Beechcraft Super King Air 200 (twin-turboprop)'] = 13.31
aircraft_length['Beechcraft Super King Air 350 (twin-turboprop)'] = 14.22
aircraft_length['Beechcraft King Air 90 (twin-turboprop)'] = 10.82

aircraft_length['Pilatus PC-12 (single-turboprop)'] = 14.4


def Log(message, file=None):
  """Write a message to a logfile along with a timestamp.

  Args:
    message: string message to write
    file: string representing file name and, if needed, path to the file to write to

  """
  # can't define as a default parameter because LOGFILE name is potentially
  # modified based on SIMULATION flag
  if not file:
    file = LOGFILE





  if file == LOGFILE:
    lock = filelock.FileLock(LOGFILE_LOCK)
    lock.acquire()

  try:
    with open(file, 'a') as f:
      # by excluding the timestamp, file diffs become easy between runs
      if not SIMULATION or file == LOGFILE:
        f.write('='*80+'\n')
        f.write(str(datetime.datetime.now(TZ))+'\n')
        f.write('\n')
      f.write(str(message)+'\n')
  except IOError:
    Log('Unable to append to ' + file)

  if file == LOGFILE:
    lock.release()

  existing_log_lines = ReadFile(LOGFILE).splitlines()
  with open(ROLLING_LOGFILE, 'w') as f:
    f.write('\n'.join(existing_log_lines[-1000:]))





def MaintainRollingWebLog(message, max_count, filename=None):
  """Maintains a rolling text file of at most max_count printed messages.

  Newest data at top and oldest data at the end, of at most max_count messages,
  where the delimiter between each message is identified by a special fixed string.

  Args:
    message: text message to prepend to the file.
    max_count: maximum number of messages to keep in the file; the max_count+1st message
      is deleted.
    filename: the file to update.
  """
  # can't define as a default parameter because ROLLING_MESSAGE_FILE name is potentially
  # modified based on SIMULATION flag
  if not filename:
    filename = ROLLING_MESSAGE_FILE
  rolling_log_header = '='*(SPLITFLAP_CHARS_PER_LINE + 2)
  existing_file = ReadFile(filename)




                            <----SKIPPED LINES---->




    available or if delimiters missing.
  """
  settings_dict = {}
  for setting in settings.split(';'):
    if '=' in setting:
      kv_list = setting.split('=')
      k = kv_list[0]
      v = kv_list[1]
      if v.isdigit():
        v = int(v)
      else:
        try:
          v = float(v)
        except ValueError:
          pass
      settings_dict[k] = v

  return settings_dict










def WriteFile(filename, text, log_exception=False):
  """Writes the text to the file, returning boolean indicating success.

  Args:
    filename: string of the filename to open, potentially also including the full path.
    text: the text to write
    log_exception: boolean indicating whether to log an exception if file not found.

  Returns:
    Boolean indicating whether the write was successful.
  """
  try:
    with open(filename, 'w') as content_file:
      content_file.write(text)
  except IOError:
    if log_exception:
      Log('Unable to write to '+filename)
    return False
  return True





                            <----SKIPPED LINES---->




    newly_nearby_flight_numbers = UpdateAircraftList(
        persistent_nearby_aircraft, current_nearby_aircraft, now)

    if newly_nearby_flight_numbers:

      if len(newly_nearby_flight_numbers) > 1:
        newly_nearby_flight_numbers_str = ', '.join(newly_nearby_flight_numbers)
        newly_nearby_flight_details_str = '\n'.join(
            [str(current_nearby_aircraft[f]) for f in newly_nearby_flight_numbers])
        Log('Multiple newly-nearby flights: %s\n%s' % (
            newly_nearby_flight_numbers_str, newly_nearby_flight_details_str))
      flight_number = newly_nearby_flight_numbers[0]

      flight_aware_json = {}
      if SIMULATION:
        json_times = [j[1] for j in FA_JSONS]
        if json_time in json_times:
          flight_aware_json = FA_JSONS[json_times.index(json_time)][0]
      elif flight_number:
        flight_aware_json = GetFlightAwareJson(flight_number)

        UpdateStatusLight(GPIO_ERROR_FLIGHT_AWARE_CONNECTION, False)
        if not flight_aware_json:
          Log('No json returned from Flightaware for flight: %s' % flight_number)
          UpdateStatusLight(GPIO_ERROR_FLIGHT_AWARE_CONNECTION, True)

      flight_details = {}
      if flight_aware_json:
        flight_details = ParseFlightAwareJson(flight_aware_json)

      if not SIMULATION and log_jsons:
        PickleObjectToFile((flight_aware_json, now), PICKLE_FA_JSON_FILE, True)

      # Augment FlightAware details with radio / radio-derived details
      flight_details.update(current_nearby_aircraft[flight_number])

      # Augment with the past location data
      flight_details['persistent_path'] = persistent_path[flight_number][1]

  return (
      persistent_nearby_aircraft,
      flight_details,
      now,




                            <----SKIPPED LINES---->





        if aircraft.get('squawk') is not None:
          simplified_aircraft['squawk'] = aircraft.get('squawk')

        track = aircraft.get('track')
        if isinstance(track, numbers.Number):
          min_meters = MinMetersToHome((lat, lon), track)
          simplified_aircraft['track'] = track
          simplified_aircraft['min_feet'] = min_meters * FEET_IN_METER

        if HaversineDistanceMeters(HOME, (lat, lon)) < MIN_METERS:
          nearby_aircraft[flight_number] = simplified_aircraft

        # keep all that track info - once we start reporting on a nearby flight, it will
        # become part of the flight's persistent record. Also, note that as we are
        # building a list of tracks for each flight, and we are later assigning the
        # flight dictionary to point to the list, we just simply need to continue
        # updating this list to keep the dictionary up to date (i.e.: we don't need
        # to directly touch the flights dictionary in main).
        (last_seen, current_path) = persistent_path.get(flight_number, (None, []))
        if simplified_aircraft not in current_path:



          current_path.append(simplified_aircraft)
        persistent_path[flight_number] = (now, current_path)

  # if the flight was last seen too far in the past, remove the track info
  for flight_number in list(persistent_path.keys()):
    (last_seen, current_path) = persistent_path[flight_number]
    if last_seen < now - PERSISTENCE_SECONDS:
      persistent_path.pop(flight_number)

  return (nearby_aircraft, now, json_desc_dict, persistent_path)


def GetFlightAwareJson(flight_number):
  """Scrapes the text json message from FlightAware for a given flight number.

  Given a flight number, loads the corresponding FlightAware webpage for that flight and
  extracts the relevant script that contains all the flight details from that page.

  Args:
    flight_number: text flight number (i.e.: SWA1234)

  Returns:
    Text representation of the json message from FlightAware.
  """
  url = 'https://flightaware.com/live/flight/' + flight_number
  try:
    response = requests.get(url)
  except requests.exceptions.RequestException as e:
    Log('Unable to query FA for URL due to %s: %s' % (e, url))
    return ''
  soup = bs4.BeautifulSoup(response.text, "html.parser")
  l = soup.find_all('script')
  flight_script = None
  for script in l:
    if "trackpollBootstrap" in str(script):
      flight_script = str(script)
      break
  if not flight_script:
    Log('Unable to find trackpollBootstrap script in page: ' + response.text)
    return ''
  first_open_curly_brace = flight_script.find('{')
  last_close_curly_brace = flight_script.rfind('}')
  flight_json = flight_script[first_open_curly_brace:last_close_curly_brace+1]
  return flight_json


def Unidecode(s):
  """Convert a special unicode characters to closest ASCII representation."""
  if s is not None:
    s = unidecode.unidecode(s)
  return s




                            <----SKIPPED LINES---->




    message_queue: List of flight messages that have not yet been printed.
    flights: List of flights dictionaries.
  """
  if flights:
    histogram = {
        'type': 'both',
        'histogram':'all',
        'histogram_history':'30d',
        'histogram_max_screens': '_2',
        'histogram_data_summary': 'on'}
    message_queue.extend(TriggerHistograms(flights, histogram))

    while message_queue:
      ManageMessageQueue(message_queue, 0, {'setting_delay': 0})
    SaveFlightsByAltitudeDistanceCSV(flights)
    SaveFlightsToCSV(flights)

  # repickle to a new .pk with full track info
  file_parts = PICKLE_FLIGHTS.split('.')
  new_pickle_file = '.'.join([file_parts[0] + '_full_path', file_parts[1]])
  if os.path.exists(new_pickle_file):
    os.remove(new_pickle_file)
  for flight in flights:
    PickleObjectToFile(flight, new_pickle_file, False)

  print('Simulation complete after %s dump json messages processed' % len(DUMP_JSONS))


def SimulationSlowdownNearFlight(flights, persistent_nearby_aircraft):
  """Slows down simulations when a reported-upon flight is nearby."""
  if flights and flights[-1].get('flight_number') in persistent_nearby_aircraft:
    time.sleep(arduino.WRITE_DELAY_TIME)


def DumpJsonChanges():
  """Identifies if sequential dump json files changes, for simulation optimization.

  If we are logging the radio output faster than it is updating, then there will be
  sequential log files in the json list that are identical; we only need to process the
  first of these, and can ignore subsequent ones, without any change of output in the
  simulation results. This function identifies whether the current active json changed
  from the prior one.

  Returns:
    Boolean - True if different (and processing needed), False if identical
  """
  if SIMULATION_COUNTER == 0:
    return True
  (this_json, unused_now) = DUMP_JSONS[SIMULATION_COUNTER]
  (last_json, unused_now) = DUMP_JSONS[SIMULATION_COUNTER - 1]
  return this_json != last_json


def CheckRebootNeeded(startup_time, message_queue, json_desc_dict):
  """Reboot if running for over 24 hours and all is quiet, or if its just been too long."""









  reboot = False


  running_hours = (time.time() - startup_time) / SECONDS_IN_HOUR
  if (
      running_hours >= HOURS_IN_DAY and
      not message_queue and
      not json_desc_dict.get('radio_range_flights')):
    reboot = True

  if (
      running_hours > HOURS_IN_DAY * 1.5 and
      not message_queue and
      int(EpochDisplayTime(time.time(), '%-H')) >= 3):
    reboot = True

  if reboot:
    Log('Reboot needed after running for %.2f hours' % running_hours)











    global SHUTDOWN_SIGNAL
    SHUTDOWN_SIGNAL = True

  return reboot


def RequestGracefulShutdown(signalNumber, unused_frame):
  """Sets flag so that the main loop will terminate when it completes the iteration.
















  The function signature is defined by the python language - i.e.: these two variables
  are passed automatically for registered signals.

  """
  global SHUTDOWN_SIGNAL
  SHUTDOWN_SIGNAL = True
  Log('%d received termination signal %d (%s)' % (
      os.getpid(), signalNumber,
      signal.Signals(signalNumber).name))  # pylint: disable=E1101


def PerformGracefulShutdown(queues, shutdown, reboot=False):
  """Complete the graceful shutdown process by cleaning up.

  Args:
    queues: iterable of queues shared with child processes to be closed
    shutdown: tuple of shared flags with child processes to initiate shutdown in children
    reboot: boolean indicating whether we should trigger a reboot
  """
  reboot_msg = ''
  if reboot:
    reboot_msg = ' and rebooting'
  Log('Shutting down self (%d)%s' % (os.getpid(), reboot_msg))

  for q in queues:
    q.close()
  for v in shutdown:  # send the shutdown signal to child processes
    v.value = 1

  RPi.GPIO.cleanup()
  if reboot:


    os.system('sudo reboot')
  sys.exit()


def FindRunningParents():
  """Returns list of proc ids of processes with identically-named python file running.

  In case there are multiple children processes spawned with the same name, such as via
  multiprocessing, this will only return the parent id (since a killed child process
  will likely just be respawned).
  """
  this_process_id = os.getpid()
  this_process_name = os.path.basename(sys.argv[0])
  pids = []
  pid_pairs = []
  for proc in psutil.process_iter():
    try:
      # Check if process name contains this_process_name.
      commands = proc.as_dict(attrs=['cmdline', 'pid', 'ppid'])
      if commands['cmdline']:
        command_running = any(
            [this_process_name in s for s in commands['cmdline']])
        if command_running:
          pids.append(commands['pid'])
          pid_pairs.append((commands['pid'], commands['ppid']))
    except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
      pass

  # Exclude those pids that have a parent that is also a pid
  final_pids = []
  for pid_pair in pid_pairs:
    if pid_pair[1] not in pids:
      final_pids.append(pid_pair[0])
  # Exclude this pid
  final_pids.pop(final_pids.index(this_process_id))

  return sorted(final_pids)


def WaitUntilKillComplete(already_running_ids, max_seconds=10):
  """Prevents main loop from starting until other instance, if any, completes shutdown.

  A termination command send to any other identically-named process may take a few
  seconds to complete because that other process is allowed to finish the current iteration
  in the main loop. Typically, that iteration in the other process will complete before
  this process finishes the initialization and starts. But in limited scenarios, that might
  not happen, such as if the other process is in the middle of generating a lot of
  histogram images, or if this process does not have much data to load.

  This function ensures that this process does not start the main loop until the other
  process terminates. If it detects that the other process is still running, it waits
  for up to max_seconds. If the other process does not terminate before that time limit,
  then this process terminates.
  """
  still_running_ids = FindRunningParents()
  if not still_running_ids:
    return
  # still_running_ids should at least be a subset of already_running_ids
  new_processes = sorted(list(set(still_running_ids).difference(set(already_running_ids))))
  if new_processes:
    # uh-oh! at least one new started up in the interim? exit!
    Log('Kill signal sent to %s from this process %s, but it seems like there is '
        'at least one new process running, %s!' % (
            str(already_running_ids), str(os.getpid()), str(new_processes)))
    sys.exit()
  # phew - they're a subset; so they probably got the signal; just wait a few secs
  elif still_running_ids:
    n = 0
    running_parents = FindRunningParents()
    while running_parents:
      if n == max_seconds:
        Log('Kill signal sent from this process %d to %s, but %s still '
            'running after waiting cume %d seconds; exiting anyways' % (
                os.getpid(), str(already_running_ids), str(running_parents), n+1))
        sys.exit()




      n += 1
      time.sleep(1)
      Log('Kill signal sent from this process %d to %s, but %s still '
          'running after waiting cume %d seconds' % (
              os.getpid(), str(already_running_ids), str(running_parents), n+1))
      running_parents = FindRunningParents()


def InitArduinos(configuration):
  """Initializes and starts the two arduino threads with new shared-memory queues."""
  to_remote_q = multiprocessing.Queue()
  to_servo_q = multiprocessing.Queue()
  to_main_q = multiprocessing.Queue()
  shutdown_remote = multiprocessing.Value('i')  # shared flag to initiate shutdown
  shutdown_servo = multiprocessing.Value('i')  # shared flag to initiate shutdown
  shutdown = (shutdown_remote, shutdown_servo)

  remote, servo = ValidateArduinosRunning(
      None, None, to_remote_q, to_servo_q, to_main_q, shutdown, configuration)
  return (remote, servo, to_remote_q, to_servo_q, to_main_q, shutdown)


def RefreshArduinos(
    remote, servo,
    to_remote_q, to_servo_q, to_main_q, shutdown,




                            <----SKIPPED LINES---->





def ValidateArduinosRunning(
    remote, servo, to_remote_q, to_servo_q, to_main_q, shutdown, configuration):
  """Ensures that each of the enabled arduinos are running, restarting if needed."""
  remote = ValidateSingleRunning(
      'enable_remote' in configuration,
      arduino.RemoteMain, p=remote,
      args=(to_remote_q, to_main_q, shutdown[0]))
  servo = ValidateSingleRunning(
      'enable_servos' in configuration,
      arduino.ServoMain, p=servo,
      args=(to_servo_q, to_main_q, shutdown[1]))
  return remote, servo


def ValidateSingleRunning(enabled, start_function, p=None, args=()):
  """Restarts a new instance of multiprocessing process if not running"""
  if not SHUTDOWN_SIGNAL:

    if not enabled:
      if p is not None:  # must have just shut this down
        args[2].value = 1  # signal a shutdown on the shared state arg





      else:
        return None

    elif p is None or not p.is_alive():
      Log('Process (%s) for %s is not alive; restarting' % (str(p), str(start_function)))
      args[2].value = 0  # (re)set shutdown flag to allow function to run
      p = multiprocessing.Process(target=start_function, args=args)
      p.daemon = True
      p.start()

  return p


def EnqueueArduinos(flights, json_desc_dict, configuration, to_servo_q, to_remote_q):
  """Send latest data to arduinos via their shared-memory queues"""
  last_flight = {}
  if flights:
    last_flight = dict(flights[-1])

  if SIMULATION:
    now = json_desc_dict['now']
  else:
    now = time.time()






                            <----SKIPPED LINES---->




      messageboard_flight_index = IdentifyFlightDisplayed(
          flights, configuration, display_all_hours=True)
      if messageboard_flight_index is not None:
        message_queue = [m for m in message_queue if m[0] != FLAG_MSG_INTERESTING]
        flight_message = CreateMessageAboutFlight(flights[messageboard_flight_index])
        message_queue = [(FLAG_MSG_FLIGHT, flight_message)]
        next_message_time = time.time()

    elif command == 'histogram':
      if not flights:
        Log('Histogram requested by remote %s but no flights in memory' % str(args))
      else:
        histogram_type, histogram_history = args
        message_queue.extend(MessageboardHistograms(
            flights,
            histogram_type,
            histogram_history,
            '_1',
            False))

    elif command == 'update_settings':
      WriteFile(CONFIG_FILE, *args)

    else:
      Log('Improper command from arduinos: %s / %s' % (command, args))

  return message_queue, next_message_time


def PublishMessage(
    s,
    subscription_id='12fd73cd-75ef-4cae-bbbf-29b2678692c1',
    key='c5f62d44-e30d-4c43-a43e-d4f65f4eb399',
    secret='b00aeb24-72f3-467c-aad2-82ba5e5266ca',
    timeout=3):
  """Publishes a text string to a Vestaboard.

  The message is pushed to the vestaboard splitflap display by way of its web services;
  see https://docs.vestaboard.com/introduction for more details.

  Args:




                            <----SKIPPED LINES---->




      'https://platform.vestaboard.com/subscriptions/%s/message' % subscription_id)
  curl.setopt(pycurl.HTTPHEADER, [
      'X-Vestaboard-Api-Key:%s' % key, 'X-Vestaboard-Api-Secret:%s' % secret])
  curl.setopt(pycurl.TIMEOUT_MS, timeout*1000)
  curl.setopt(pycurl.POST, 1)

  curl.setopt(pycurl.WRITEFUNCTION, lambda x: None) # to keep stdout clean

  # preparing body the way pycurl.READDATA wants it
  body_as_dict = {'text': s}
  body_as_json_string = json.dumps(body_as_dict) # dict to json
  body_as_file_object = io.StringIO(body_as_json_string)

  # prepare and send. See also: pycurl.READFUNCTION to pass function instead
  curl.setopt(pycurl.READDATA, body_as_file_object)
  curl.setopt(pycurl.POSTFIELDSIZE, len(body_as_json_string))
  try:
    curl.perform()
  except pycurl.error as e:
    Log('curl.perform() failed with message %s' % e)
    UpdateStatusLight(GPIO_ERROR_VESTABOARD_CONNECTION, True)
    error_code = True
  else:
    # you may want to check HTTP response code, e.g.
    status_code = curl.getinfo(pycurl.RESPONSE_CODE)
    if status_code != 200:
      Log('Server returned HTTP status code %d for message %s' % (status_code, s))
    UpdateStatusLight(GPIO_ERROR_VESTABOARD_CONNECTION, True)
    error_code = True

  curl.close()
  if not error_code:
    UpdateStatusLight(GPIO_ERROR_VESTABOARD_CONNECTION, False)


def ManageMessageQueue(message_queue, next_message_time, configuration):
  """Check time & if appropriate, display next message from queue.

  Args:
    message_queue: FIFO list of message tuples of (message type, message string).
    next_message_time: epoch at which next message should be displayed
    configuration: dictionary of configuration attributes.

  Returns:
    Next_message_time, potentially updated if a message has been displayed, or unchanged
    if no message was displayed.
  """
  if message_queue and (time.time() >= next_message_time or SIMULATION):

    if SIMULATION:  # drain the queue because the messages come so fast
      messages_to_display = list(message_queue)
      # passed by reference, so clear it out since we drained it to the display
      del message_queue[:]




                            <----SKIPPED LINES---->




  be considered correct.

  They are "correct" in the sense that that new insight was not available at the time
  that older flight was seen, but it is not correct in the sense that, because this new
  insight is starting out with an incidence in the historical data of zero, this
  new insight may be reported more frequently than desired until it "catches up".

  So this method replays the flight history with the latest insight code, regenerating
  the insight distribution for each flight.
  """
  directory, file = os.path.split(full_path)
  all_files = os.listdir(directory)
  files = sorted([os.path.join(directory, f) for f in all_files if file in f])
  for f in files:

    print('Bootstrapping %s' % f)
    configuration = ReadAndParseSettings(CONFIG_FILE)
    flights = []
    tmp_f = f + 'tmp'

    if os.path.exists(tmp_f):
      os.remove(tmp_f)

    if os.path.exists(f):
      mtime = os.path.getmtime(f)
      flights = UnpickleObjectFromFile(f, False)
      for (n, flight) in enumerate(flights):
        if n/25 == int(n/25):
          print(' - %d' % n)
        CreateFlightInsights(flights[:n+1], configuration.get('insights', 'hide'), {})
        PickleObjectToFile(flight, tmp_f, False)

      if mtime == os.path.getmtime(f):
        shutil.move(tmp_f, f)
      else:
        print('Aborted: failed to bootstrap %s: file changed while in process' % full_path)
        return


def ResetLogs(config):
  """Clears the non-scrolling logs if reset_logs in config."""
  if 'reset_logs' in config:
    Log('Reset logs')
    for f in (STDERR_FILE, BACKUP_FILE, SERVICE_VERIFICATION_FILE):
      if os.path.exists(f):
        os.remove(f)
        open(f, 'a').close()
    config.pop('reset_logs')
    config = BuildSettings(config)
    WriteFile(CONFIG_FILE, config)
  return config


def CheckTemperature(fan_power):
  """Turn on fan if temperature exceeds threshold."""
  if RASPBERRY_PI:
    temperature = gpiozero.CPUTemperature().temperature
    if temperature > TEMP_FAN_TURN_ON_CELSIUS and not fan_power:
      fan_power = True
      RPi.GPIO.output(GPIO_FAN, RPi.GPIO.HIGH)
      Log('Fan turned on at temperature %.1f' % temperature)
    elif temperature < TEMP_FAN_TURN_OFF_CELSIUS and fan_power:
      fan_power = False
      RPi.GPIO.output(GPIO_FAN, RPi.GPIO.LOW)
      Log('Fan turned off at temperature %.1f' % temperature)
  return fan_power


pin_values = {}
def SetPinsAsOutput():
  """Initialize output GPIO pins for output on Raspberry Pi."""
  global pin_values




  for pin in (
      GPIO_ERROR_VESTABOARD_CONNECTION, GPIO_ERROR_FLIGHT_AWARE_CONNECTION,
      GPIO_ERROR_ARDUINO_SERVO_CONNECTION, GPIO_ERROR_ARDUINO_REMOTE_CONNECTION,
      GPIO_ERROR_BATTERY_CHARGE, GPIO_FAN):









    if RASPBERRY_PI:
      RPi.GPIO.setup(pin, RPi.GPIO.OUT)
      RPi.GPIO.output(pin, RPi.GPIO.LOW)
    pin_values[pin] = False








def UpdateStatusLight(pin, value):
  """Sets the Raspberry Pi GPIO pin high (True) or low (False) based on value."""
  global pin_values





  if RASPBERRY_PI:

    if value:
      RPi.GPIO.output(pin, RPi.GPIO.HIGH)

    else:
      RPi.GPIO.output(pin, RPi.GPIO.LOW)




  if pin_values[pin] != value:
    Log('Error light on pin %d set to %s' % (pin, value))
    pin_values[pin] = value










def main():
  """Traffic cop between incoming radio flight messages, configuration, and messageboard.

  This is the main logic, checking for new flights, augmenting the radio signal with
  additional web-scraped data, and generating messages in a form presentable to the
  messageboard.
  """


  # Since this clears log files, it should occur first before we start logging
  if '-s' in sys.argv:
    global SIMULATION_COUNTER
    SimulationSetup()

  # This flag slows down simulation time around a flight, great for debugging the arduinos
  simulation_slowdown = bool('-f' in sys.argv)

  # Redirect any errors to a log file instead of the screen, and add a datestamp
  if not SIMULATION:
    sys.stderr = open(STDERR_FILE, 'a')
    Log('', STDERR_FILE)

  Log('Starting up process %d' % os.getpid())
  already_running_ids = FindRunningParents()
  if already_running_ids:
    for pid in already_running_ids:
      Log('Sending termination signal to %d' % pid)
      os.kill(pid, signal.SIGTERM)

  if RASPBERRY_PI:
    RPi.GPIO.setmode(RPi.GPIO.BCM)
  SetPinsAsOutput()
  fan_power = False

  configuration = ReadAndParseSettings(CONFIG_FILE)
  startup_time = time.time()
  json_desc_dict = {}

  flights = UnpickleObjectFromFile(PICKLE_FLIGHTS, True, max_days=MAX_INSIGHT_HORIZON_DAYS)
  # Clear the loaded flight of any cached data, identified by keys with a specific
  # suffix, since code fixes may change the values for some of those cached elements
  for flight in flights:
    for key in list(flight.keys()):
      if key.endswith(CACHED_ELEMENT_PREFIX):
        flight.pop(key)

  # If we're displaying just a single insight message, we want it to be something
  # unique, to the extent possible; this dict holds a count of the diff types of messages
  # displayed so far
  insight_message_distribution = {}

  # bootstrap the flight insights distribution from a list of insights on each




                            <----SKIPPED LINES---->




           configuration.get('log_jsons', False))

      if flight:
        flights.append(flight)
        remote, servo = RefreshArduinos(
            remote, servo,
            to_remote_q, to_servo_q, to_main_q, shutdown,
            flights, json_desc_dict, configuration)

        flight_meets_display_criteria = FlightMeetsDisplayCriteria(
            flight, configuration, log=True)
        if flight_meets_display_criteria:
          flight_message = (FLAG_MSG_FLIGHT, CreateMessageAboutFlight(flight))

          # display the next message about this flight now!
          next_message_time = time.time()
          message_queue.insert(0, flight_message)
          # and delete any queued insight messages about other flights that have
          # not yet displayed, since a newer flight has taken precedence
          messages_to_delete = [m for m in message_queue if m[0] == FLAG_MSG_INTERESTING]
          if messages_to_delete:
            Log(
                'Deleting messages from queue due to new-found plane: %s' %
                messages_to_delete)
          message_queue = [m for m in message_queue if m[0] != FLAG_MSG_INTERESTING]

          # Though we also manage the message queue outside this conditional as well,
          # because it can take a half second to generate the flight insights, this allows
          # this message to start displaying on the board immediately, so it's up there
          # when it's most relevant
          next_message_time = ManageMessageQueue(
              message_queue, next_message_time, configuration)

          insight_messages = CreateFlightInsights(
              flights, configuration.get('insights'), insight_message_distribution)
          if configuration.get('next_flight', 'off') == 'on':
            next_flight_text = FlightInsightNextFlight(flights, configuration)
            if next_flight_text:
              insight_messages.insert(0, next_flight_text)

          insight_messages = [(FLAG_MSG_INTERESTING, m) for m in insight_messages]




                            <----SKIPPED LINES---->





        PickleObjectToFile(flight, PICKLE_FLIGHTS, not SIMULATION)

      else:
        remote, servo = RefreshArduinos(
            remote, servo,
            to_remote_q, to_servo_q, to_main_q, shutdown,
            flights, json_desc_dict, configuration)

    message_queue, next_message_time = ProcessArduinoCommmands(
        to_main_q, flights, configuration, message_queue, next_message_time)

    if SIMULATION:
      if now:
        simulated_hour = EpochDisplayTime(now, '%Y-%m-%d %H:00%z')
      if simulated_hour != prev_simulated_hour:
        print(simulated_hour)
        prev_simulated_hour = simulated_hour

    histogram = ReadAndParseSettings(HISTOGRAM_CONFIG_FILE)
    if os.path.exists(HISTOGRAM_CONFIG_FILE):
      os.remove(HISTOGRAM_CONFIG_FILE)

    # We also need to make sure there are flights on which to generate a histogram! Why
    # might there not be any flights? Primarily during a simulation, if there's a
    # lingering histogram file at the time of history restart.
    if histogram and not flights:
      Log('Histogram requested (%s) but no flights in memory' % histogram)
    if histogram and flights:
      message_queue.extend(TriggerHistograms(flights, histogram))

    # check time & if appropriate, display next message from queue
    next_message_time = ManageMessageQueue(message_queue, next_message_time, configuration)

    reboot = CheckRebootNeeded(startup_time, message_queue, json_desc_dict)

    fan_power = CheckTemperature(fan_power)

    if not SIMULATION:
      time.sleep(max(0, next_loop_time - time.time()))
      next_loop_time = time.time() + LOOP_DELAY_SECONDS
    else:
      SIMULATION_COUNTER += 1
      if simulation_slowdown:
        SimulationSlowdownNearFlight(flights, persistent_nearby_aircraft)
    if SHUTDOWN_SIGNAL:  # do a graceful exit
      PerformGracefulShutdown((to_remote_q, to_servo_q, to_main_q), shutdown)

  if SIMULATION:
    SimulationEnd(message_queue, flights)
  PerformGracefulShutdown((to_remote_q, to_servo_q, to_main_q), shutdown, reboot)


if __name__ == "__main__":
  #interrupt, as in ctrl-c
  signal.signal(signal.SIGINT, RequestGracefulShutdown)

  #terminate, when another instance found or via kill
  signal.signal(signal.SIGTERM, RequestGracefulShutdown)

  if '-i' in sys.argv:
    BootstrapInsightList()
  else:
    main()

                            <----SKIPPED LINES---->




import bs4
import dateutil.relativedelta
import filelock
import numpy
import matplotlib
import matplotlib.pyplot
import psutil
import pycurl
import pytz
import requests
import tzlocal
import unidecode

# This is the directory that stores all the ancillary messageboard configuration files
# that do not need to be exposed via the webserver
MESSAGEBOARD_PATH = '/home/pi/splitflap/'
# This is the directory of the webserver; files placed here are available at
# http://adsbx-custom.local/; files placed in this directly are visible via a browser
WEBSERVER_PATH = '/var/www/html/'

VERBOSE = False  # additional messages logged

RASPBERRY_PI = psutil.sys.platform.title() == 'Linux'
if RASPBERRY_PI:
  import gpiozero  # pylint: disable=E0401
  import RPi.GPIO  # pylint: disable=E0401

SHUTDOWN_SIGNAL = False
REBOOT_SIGNAL = False

SIMULATION = False
SIMULATION_COUNTER = 0
SIMULATION_PREFIX = 'SIM_'
PICKLE_DUMP_JSON_FILE = 'pickle/dump_json.pk'
PICKLE_FA_JSON_FILE = 'pickle/fa_json.pk'
DUMP_JSONS = None  # loaded only if in simulation mode
FA_JSONS = None  # loaded only if in simulation mode

HOME_LAT = 37.64406
HOME_LON = -122.43463
HOME = (HOME_LAT, HOME_LON) # lat / lon tuple of antenna
HOME_ALT = 29  #altitude in meters
RADIUS = 6371.0e3  # radius of earth in meters

FEET_IN_METER = 3.28084
FEET_IN_MILE = 5280
METERS_PER_SECOND_IN_KNOTS = 0.514444

MIN_METERS = 5000/FEET_IN_METER # only planes within this distance will be detailed




                            <----SKIPPED LINES---->




FLAG_INSIGHT_GROUNDSPEED = 3
FLAG_INSIGHT_ALTITUDE = 4
FLAG_INSIGHT_VERTRATE = 5
FLAG_INSIGHT_FIRST_DEST = 6
FLAG_INSIGHT_FIRST_ORIGIN = 7
FLAG_INSIGHT_FIRST_AIRLINE = 8
FLAG_INSIGHT_FIRST_AIRCRAFT = 9
FLAG_INSIGHT_LONGEST_DELAY = 10
FLAG_INSIGHT_FLIGHT_DELAY_FREQUENCY = 11
FLAG_INSIGHT_FLIGHT_DELAY_TIME = 12
FLAG_INSIGHT_AIRLINE_DELAY_FREQUENCY = 13
FLAG_INSIGHT_AIRLINE_DELAY_TIME = 14
FLAG_INSIGHT_DESTINATION_DELAY_FREQUENCY = 15
FLAG_INSIGHT_DESTINATION_DELAY_TIME = 16
FLAG_INSIGHT_HOUR_DELAY_FREQUENCY = 17
FLAG_INSIGHT_HOUR_DELAY_TIME = 18
FLAG_INSIGHT_DATE_DELAY_FREQUENCY = 19
FLAG_INSIGHT_DATE_DELAY_TIME = 20
INSIGHT_TYPES = 21











TEMP_FAN_TURN_ON_CELSIUS = 65
TEMP_FAN_TURN_OFF_CELSIUS = 55

# GPIO relay connections - (GPIO pin, true message, false message, relay number)
GPIO_ERROR_VESTABOARD_CONNECTION = (
    22, 'Vestaboard unavailable', 'Vestaboard available', 1)
GPIO_ERROR_FLIGHT_AWARE_CONNECTION = (
    23, 'FlightAware not available', 'FlightAware available', 2)
GPIO_ERROR_ARDUINO_SERVO_CONNECTION = (
    24,
    'Servos not running or lost connection',
    'Handshake with servo Arduino received',
    3)
GPIO_ERROR_ARDUINO_REMOTE_CONNECTION = (
    25,
    'Remote not running or lost connection',
    'Handshake with remote Arduino received',
    4)
GPIO_ERROR_BATTERY_CHARGE = (26, 'Remote battery is low', 'Remote battery recharged', 5)
GPIO_FAN = (
    5,
    'RPi above %dC degrees; fan switched on' % TEMP_FAN_TURN_ON_CELSIUS,
    'RPi below %dC degrees; fan switched off' % TEMP_FAN_TURN_OFF_CELSIUS, 7)

# for future expansion
GPIO_UNUSED_1 = (
    27, 'Undefined condition set to true', 'Undefined condition set to false', 6)
GPIO_UNUSED_2 = (
    6, 'Undefined condition set to true', 'Undefined condition set to false', 8)

# GPIO pushbutton connections - (GPIO pin switch in; GPIO pin LED out)
GPIO_SOFT_RESET = (20, 21)

#if running on raspberry, then need to prepend path to file names
if RASPBERRY_PI:
  PICKLE_FLIGHTS = MESSAGEBOARD_PATH + PICKLE_FLIGHTS
  LOGFILE = MESSAGEBOARD_PATH + LOGFILE
  PICKLE_DUMP_JSON_FILE = MESSAGEBOARD_PATH + PICKLE_DUMP_JSON_FILE
  PICKLE_FA_JSON_FILE = MESSAGEBOARD_PATH + PICKLE_FA_JSON_FILE

  HISTOGRAM_CONFIG_FILE = WEBSERVER_PATH + HISTOGRAM_CONFIG_FILE
  CONFIG_FILE = WEBSERVER_PATH + CONFIG_FILE
  ROLLING_MESSAGE_FILE = WEBSERVER_PATH + ROLLING_MESSAGE_FILE
  ALL_MESSAGE_FILE = WEBSERVER_PATH + ALL_MESSAGE_FILE
  ROLLING_LOGFILE = WEBSERVER_PATH + ROLLING_LOGFILE
  STDERR_FILE = WEBSERVER_PATH + STDERR_FILE
  BACKUP_FILE = WEBSERVER_PATH + BACKUP_FILE
  SERVICE_VERIFICATION_FILE = WEBSERVER_PATH + SERVICE_VERIFICATION_FILE

  HISTOGRAM_IMAGE_PREFIX = WEBSERVER_PATH + WEBSERVER_IMAGE_FOLDER + HISTOGRAM_IMAGE_PREFIX
  HISTOGRAM_EMPTY_IMAGE_FILE = (
      WEBSERVER_PATH + WEBSERVER_IMAGE_FOLDER + HISTOGRAM_EMPTY_IMAGE_FILE)




                            <----SKIPPED LINES---->




SECONDS_IN_HOUR = SECONDS_IN_MINUTE * MINUTES_IN_HOUR
MINUTES_IN_DAY = MINUTES_IN_HOUR * HOURS_IN_DAY
SECONDS_IN_DAY = SECONDS_IN_HOUR * HOURS_IN_DAY

# Units confirmed here:
# www.adsbexchange.com/forum/threads/units-in-the-dump1090-json-file.630617/#post-639541
CLIMB_RATE_UNITS = 'fpm'
#speed units from tracker are knots, based on dump-1090/track.c
#https://github.com/SDRplay/dump1090/blob/master/track.c
SPEED_UNITS = 'kn'
DISTANCE_UNITS = 'ft'  # altitude

# For displaying histograms
# If a key is not present, how should it be displayed in histograms?
KEY_NOT_PRESENT_STRING = 'Unknown'
OTHER_STRING = 'Other' # What key strings should be listed last in sequence?
# What key strings should be listed last in sequence?
SORT_AT_END_STRINGS = [OTHER_STRING, KEY_NOT_PRESENT_STRING]
# What is the sorted sequence of keys for days of week?
DAYS_OF_WEEK = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat']

import arduino  # module expects paths to be set before import  # pylint: disable=C0413

aircraft_length = {} # in meters
aircraft_length['Airbus A220-100 (twin-jet)'] = 35
aircraft_length['Airbus A300F4-600 (twin-jet)'] = 54.08
aircraft_length['Airbus A319 (twin-jet)'] = 33.84
aircraft_length['Airbus A320 (twin-jet)'] = 37.57
aircraft_length['Airbus A320neo (twin-jet)'] = 37.57
aircraft_length['Airbus A321 (twin-jet)'] = 44.51
aircraft_length['Airbus A321neo (twin-jet)'] = 44.51
aircraft_length['Airbus A330-200 (twin-jet)'] = 58.82
aircraft_length['Airbus A330-300 (twin-jet)'] = 63.67
aircraft_length['Airbus A340-300 (quad-jet)'] = 63.69
aircraft_length['Airbus A350-1000 (twin-jet)'] = 73.79
aircraft_length['Airbus A350-900 (twin-jet)'] = 66.8
aircraft_length['Airbus A380-800 (quad-jet)'] = 72.72
aircraft_length['Boeing 737-400 (twin-jet)'] = 36.4
aircraft_length['Boeing 737-700 (twin-jet)'] = 33.63
aircraft_length['Boeing 737-800 (twin-jet)'] = 39.47
aircraft_length['Boeing 737-900 (twin-jet)'] = 42.11
aircraft_length['Boeing 747-400 (quad-jet)'] = 36.4
aircraft_length['Boeing 747-8 (quad-jet)'] = 76.25
aircraft_length['Boeing 757-200 (twin-jet)'] = 47.3
aircraft_length['Boeing 757-300 (twin-jet)'] = 54.4
aircraft_length['Boeing 767-200 (twin-jet)'] = 48.51
aircraft_length['Boeing 767-300 (twin-jet)'] = 54.94
aircraft_length['Boeing 777 (twin-jet)'] = (63.73 + 73.86) / 2
aircraft_length['Boeing 777-200 (twin-jet)'] = 63.73
aircraft_length['Boeing 777-200LR/F (twin-jet)'] = 63.73
aircraft_length['Boeing 777-300ER (twin-jet)'] = 73.86
aircraft_length['Boeing 787-10 (twin-jet)'] = 68.28
aircraft_length['Boeing 787-8 (twin-jet)'] = 56.72
aircraft_length['Boeing 787-9 (twin-jet)'] = 62.81
aircraft_length['Canadair Regional Jet CRJ-200 (twin-jet)'] = 26.77
aircraft_length['Canadair Regional Jet CRJ-700 (twin-jet)'] = 32.3
aircraft_length['Canadair Regional Jet CRJ-900 (twin-jet)'] = 36.2
aircraft_length['Canadair Challenger 350 (twin-jet)'] = 20.9
aircraft_length['Bombardier Challenger 300 (twin-jet)'] = 20.92
aircraft_length['Embraer 170/175 (twin-jet)'] = (29.90 + 31.68) / 2
aircraft_length['Embraer Phenom 300 (twin-jet)'] = 15.9
aircraft_length['EMBRAER 175 (long wing) (twin-jet)'] = 31.68
aircraft_length['Embraer ERJ-135 (twin-jet)'] = 26.33
aircraft_length['Cessna Caravan (single-turboprop)'] = 11.46
aircraft_length['Cessna Citation CJ2+ (twin-jet)'] = 14.53
aircraft_length['Cessna Citation II (twin-jet)'] = 14.54
aircraft_length['Cessna Citation V (twin-jet)'] = 14.91
aircraft_length['Cessna Citation X (twin-jet)'] = 22.04
aircraft_length['Cessna Skyhawk (piston-single)'] = 8.28
aircraft_length['Cessna Skylane (piston-single)'] = 8.84
aircraft_length['Cessna Citation Sovereign (twin-jet)'] = 19.35
aircraft_length['Cessna T206 Turbo Stationair (piston-single)'] = 8.61
aircraft_length['Beechcraft Bonanza (33) (piston-single)'] = 7.65
aircraft_length['Beechcraft Super King Air 200 (twin-turboprop)'] = 13.31
aircraft_length['Beechcraft Super King Air 350 (twin-turboprop)'] = 14.22
aircraft_length['Beechcraft King Air 90 (twin-turboprop)'] = 10.82
aircraft_length['Learjet 45 (twin-jet)'] = 17.68
aircraft_length['Pilatus PC-12 (single-turboprop)'] = 14.4


def Log(message, file=None, rolling=None):
  """Write a message to a logfile along with a timestamp.

  Args:
    message: string message to write
    file: string representing file name and, if needed, path to the file to write to
    rolling: name of file that will keep only the last n files of file
  """
  # can't define as a default parameter because LOGFILE name is potentially
  # modified based on SIMULATION flag
  if not file:
    file = LOGFILE

  # special case: for the main logfile, we always keep a rolling log
  if not rolling and file == LOGFILE:
    rolling = ROLLING_LOGFILE

  #if file == LOGFILE:
  #  lock = filelock.FileLock(LOGFILE_LOCK)
  #  lock.acquire()

  try:
    with open(file, 'a') as f:
      # by excluding the timestamp, file diffs become easier between runs
      if not SIMULATION or file == LOGFILE:
        f.write('='*80+'\n')
        f.write(str(datetime.datetime.now(TZ))+'\n')
        f.write('\n')
      f.write(str(message)+'\n')
  except IOError:
    Log('Unable to append to ' + file)

  if rolling:


    existing_log_lines = ReadFile(file).splitlines()
    with open(rolling, 'w') as f:
      f.write('\n'.join(existing_log_lines[-1000:]))

  #if file == LOGFILE:
  #  lock.release()


def MaintainRollingWebLog(message, max_count, filename=None):
  """Maintains a rolling text file of at most max_count printed messages.

  Newest data at top and oldest data at the end, of at most max_count messages,
  where the delimiter between each message is identified by a special fixed string.

  Args:
    message: text message to prepend to the file.
    max_count: maximum number of messages to keep in the file; the max_count+1st message
      is deleted.
    filename: the file to update.
  """
  # can't define as a default parameter because ROLLING_MESSAGE_FILE name is potentially
  # modified based on SIMULATION flag
  if not filename:
    filename = ROLLING_MESSAGE_FILE
  rolling_log_header = '='*(SPLITFLAP_CHARS_PER_LINE + 2)
  existing_file = ReadFile(filename)




                            <----SKIPPED LINES---->




    available or if delimiters missing.
  """
  settings_dict = {}
  for setting in settings.split(';'):
    if '=' in setting:
      kv_list = setting.split('=')
      k = kv_list[0]
      v = kv_list[1]
      if v.isdigit():
        v = int(v)
      else:
        try:
          v = float(v)
        except ValueError:
          pass
      settings_dict[k] = v

  return settings_dict


def RemoveSetting(configuration, setting):
  """Removes the named setting from the configuration file."""
  configuration.pop(setting)
  configuration = BuildSettings(configuration)
  WriteFile(CONFIG_FILE, configuration)
  return configuration


def WriteFile(filename, text, log_exception=False):
  """Writes the text to the file, returning boolean indicating success.

  Args:
    filename: string of the filename to open, potentially also including the full path.
    text: the text to write
    log_exception: boolean indicating whether to log an exception if file not found.

  Returns:
    Boolean indicating whether the write was successful.
  """
  try:
    with open(filename, 'w') as content_file:
      content_file.write(text)
  except IOError:
    if log_exception:
      Log('Unable to write to '+filename)
    return False
  return True





                            <----SKIPPED LINES---->




    newly_nearby_flight_numbers = UpdateAircraftList(
        persistent_nearby_aircraft, current_nearby_aircraft, now)

    if newly_nearby_flight_numbers:

      if len(newly_nearby_flight_numbers) > 1:
        newly_nearby_flight_numbers_str = ', '.join(newly_nearby_flight_numbers)
        newly_nearby_flight_details_str = '\n'.join(
            [str(current_nearby_aircraft[f]) for f in newly_nearby_flight_numbers])
        Log('Multiple newly-nearby flights: %s\n%s' % (
            newly_nearby_flight_numbers_str, newly_nearby_flight_details_str))
      flight_number = newly_nearby_flight_numbers[0]

      flight_aware_json = {}
      if SIMULATION:
        json_times = [j[1] for j in FA_JSONS]
        if json_time in json_times:
          flight_aware_json = FA_JSONS[json_times.index(json_time)][0]
      elif flight_number:
        flight_aware_json = GetFlightAwareJson(flight_number)
        if flight_aware_json:
          UpdateStatusLight(GPIO_ERROR_FLIGHT_AWARE_CONNECTION, False)
        else:
          Log('No json returned from Flightaware for flight: %s' % flight_number)
          UpdateStatusLight(GPIO_ERROR_FLIGHT_AWARE_CONNECTION, True)

      flight_details = {}
      if flight_aware_json:
        flight_details = ParseFlightAwareJson(flight_aware_json)

      if not SIMULATION and log_jsons:
        PickleObjectToFile((flight_aware_json, now), PICKLE_FA_JSON_FILE, True)

      # Augment FlightAware details with radio / radio-derived details
      flight_details.update(current_nearby_aircraft[flight_number])

      # Augment with the past location data
      flight_details['persistent_path'] = persistent_path[flight_number][1]

  return (
      persistent_nearby_aircraft,
      flight_details,
      now,




                            <----SKIPPED LINES---->





        if aircraft.get('squawk') is not None:
          simplified_aircraft['squawk'] = aircraft.get('squawk')

        track = aircraft.get('track')
        if isinstance(track, numbers.Number):
          min_meters = MinMetersToHome((lat, lon), track)
          simplified_aircraft['track'] = track
          simplified_aircraft['min_feet'] = min_meters * FEET_IN_METER

        if HaversineDistanceMeters(HOME, (lat, lon)) < MIN_METERS:
          nearby_aircraft[flight_number] = simplified_aircraft

        # keep all that track info - once we start reporting on a nearby flight, it will
        # become part of the flight's persistent record. Also, note that as we are
        # building a list of tracks for each flight, and we are later assigning the
        # flight dictionary to point to the list, we just simply need to continue
        # updating this list to keep the dictionary up to date (i.e.: we don't need
        # to directly touch the flights dictionary in main).
        (last_seen, current_path) = persistent_path.get(flight_number, (None, []))
        if (  # flight position has been updated with this radio signal
            not current_path or
            simplified_aircraft.get('lat') != current_path[-1].get('lat') or
            simplified_aircraft.get('lon') != current_path[-1].get('lon')):
          current_path.append(simplified_aircraft)
        persistent_path[flight_number] = (now, current_path)

  # if the flight was last seen too far in the past, remove the track info
  for flight_number in list(persistent_path.keys()):
    (last_seen, current_path) = persistent_path[flight_number]
    if last_seen < now - PERSISTENCE_SECONDS:
      persistent_path.pop(flight_number)

  return (nearby_aircraft, now, json_desc_dict, persistent_path)


def GetFlightAwareJson(flight_number):
  """Scrapes the text json message from FlightAware for a given flight number.

  Given a flight number, loads the corresponding FlightAware webpage for that flight and
  extracts the relevant script that contains all the flight details from that page.

  Args:
    flight_number: text flight number (i.e.: SWA1234)

  Returns:
    Text representation of the json message from FlightAware.
  """
  url = 'https://flightaware.com/live/flight/' + flight_number
  try:
    response = requests.get(url)
  except requests.exceptions.RequestException as e:
    Log('Unable to query FA for URL due to %s: %s' % (e, url))
    return ''
  soup = bs4.BeautifulSoup(response.text, 'html.parser')
  l = soup.find_all('script')
  flight_script = None
  for script in l:
    if "trackpollBootstrap" in str(script):
      flight_script = str(script)
      break
  if not flight_script:
    Log('Unable to find trackpollBootstrap script in page: ' + response.text)
    return ''
  first_open_curly_brace = flight_script.find('{')
  last_close_curly_brace = flight_script.rfind('}')
  flight_json = flight_script[first_open_curly_brace:last_close_curly_brace+1]
  return flight_json


def Unidecode(s):
  """Convert a special unicode characters to closest ASCII representation."""
  if s is not None:
    s = unidecode.unidecode(s)
  return s




                            <----SKIPPED LINES---->




    message_queue: List of flight messages that have not yet been printed.
    flights: List of flights dictionaries.
  """
  if flights:
    histogram = {
        'type': 'both',
        'histogram':'all',
        'histogram_history':'30d',
        'histogram_max_screens': '_2',
        'histogram_data_summary': 'on'}
    message_queue.extend(TriggerHistograms(flights, histogram))

    while message_queue:
      ManageMessageQueue(message_queue, 0, {'setting_delay': 0})
    SaveFlightsByAltitudeDistanceCSV(flights)
    SaveFlightsToCSV(flights)

  # repickle to a new .pk with full track info
  file_parts = PICKLE_FLIGHTS.split('.')
  new_pickle_file = '.'.join([file_parts[0] + '_full_path', file_parts[1]])

  RemoveFile(new_pickle_file)
  for flight in flights:
    PickleObjectToFile(flight, new_pickle_file, False)

  print('Simulation complete after %s dump json messages processed' % len(DUMP_JSONS))


def SimulationSlowdownNearFlight(flights, persistent_nearby_aircraft):
  """Slows down simulations when a reported-upon flight is nearby."""
  if flights and flights[-1].get('flight_number') in persistent_nearby_aircraft:
    time.sleep(arduino.WRITE_DELAY_TIME)


def DumpJsonChanges():
  """Identifies if sequential dump json files changes, for simulation optimization.

  If we are logging the radio output faster than it is updating, then there will be
  sequential log files in the json list that are identical; we only need to process the
  first of these, and can ignore subsequent ones, without any change of output in the
  simulation results. This function identifies whether the current active json changed
  from the prior one.

  Returns:
    Boolean - True if different (and processing needed), False if identical
  """
  if SIMULATION_COUNTER == 0:
    return True
  (this_json, unused_now) = DUMP_JSONS[SIMULATION_COUNTER]
  (last_json, unused_now) = DUMP_JSONS[SIMULATION_COUNTER - 1]
  return this_json != last_json


def CheckRebootNeeded(startup_time, message_queue, json_desc_dict, configuration):
  """Reboot based on duration instance has been running.

  Reboot needed in one of the following situations:
  - All quiet: if running for over 24 hours and all is quiet (message queue empty and
    no planes in radio).
  - Mostly quiet: if running for over 36 hours and message queue is empty and it's 3a.
  - Reboot requested via html form.

  Also checks if reset requested via html form.
  """
  reboot = False
  end_process = False

  running_hours = (time.time() - startup_time) / SECONDS_IN_HOUR
  if (
      running_hours >= HOURS_IN_DAY and
      not message_queue and
      not json_desc_dict.get('radio_range_flights')):
    reboot = True
    Log('All quiet reboot needed after running for %.2f hours' % running_hours)
  if (
      running_hours > HOURS_IN_DAY * 1.5 and
      not message_queue and
      int(EpochDisplayTime(time.time(), '%-H')) >= 3):
    reboot = True


    Log('Early morning reboot needed after running for %.2f hours' % running_hours)
  if 'soft_reboot' in configuration:
    reboot = True
    Log('Soft reboot requested via web form')
    RemoveSetting(configuration, 'soft_reboot')

  if 'end_process' in configuration:
    Log('Process end requested via web form')
    RemoveSetting(configuration, 'end_process')
    end_process = True

  if reboot or end_process:
    global SHUTDOWN_SIGNAL
    SHUTDOWN_SIGNAL = True

  return reboot


def InterruptRebootFromButton():
  """Sets flag so that the main loop will terminate when it completes the iteration.

  This function is only triggered by an physical button press.
  """
  global SHUTDOWN_SIGNAL
  SHUTDOWN_SIGNAL = True

  global REBOOT_SIGNAL
  REBOOT_SIGNAL = True

  RPi.GPIO.output(GPIO_SOFT_RESET[1], False)  # signal that reset received
  Log('Soft reboot requested by button push')


def InterruptShutdownFromSignal(signalNumber, unused_frame):
  """Sets flag so that the main loop will terminate when it completes the iteration.

  The function signature is defined by the python language - i.e.: these two variables
  are passed automatically for registered signals.  This function is only triggered by an
  interrupt signal.
  """
  global SHUTDOWN_SIGNAL
  SHUTDOWN_SIGNAL = True
  Log('%d received termination signal %d (%s)' % (
      os.getpid(), signalNumber,
      signal.Signals(signalNumber).name))  # pylint: disable=E1101


def PerformGracefulShutdown(queues, shutdown, reboot):
  """Complete the graceful shutdown process by cleaning up.

  Args:
    queues: iterable of queues shared with child processes to be closed
    shutdown: tuple of shared flags with child processes to initiate shutdown in children
    reboot: boolean indicating whether we should trigger a reboot
  """
  reboot_msg = ''
  if reboot:
    reboot_msg = ' and rebooting'
  Log('Shutting down self (%d)%s' % (os.getpid(), reboot_msg))

  for q in queues:
    q.close()
  for v in shutdown:  # send the shutdown signal to child processes
    v.value = 1
  if RASPBERRY_PI:
    RPi.GPIO.cleanup()

  if reboot or REBOOT_SIGNAL:
    time.sleep(10)  # wait 10 seconds for children to shut down as well
    os.system('sudo reboot')
  sys.exit()


def FindRunningParents():
  """Returns list of proc ids of processes with identically-named python file running.

  In case there are multiple children processes spawned with the same name, such as via
  multiprocessing, this will only return the parent id (since a killed child process
  will likely just be respawned).
  """
  this_process_id = os.getpid()
  this_process_name = os.path.basename(sys.argv[0])
  pids = []
  pid_pairs = []
  for proc in psutil.process_iter():
    try:
      # Check if process name contains this_process_name.
      commands = proc.as_dict(attrs=['cmdline', 'pid', 'ppid'])
      if commands['cmdline']:
        command_running = any(
            [this_process_name in s for s in commands['cmdline']])
        if command_running:
          pids.append(commands['pid'])
          pid_pairs.append((commands['pid'], commands['ppid']))
    except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
      pass

  # Exclude those pids that have a parent that is also a pid
  final_pids = []
  for pid_pair in pid_pairs:
    if pid_pair[1] not in pids:
      final_pids.append(pid_pair[0])
  # Exclude this pid
  final_pids.pop(final_pids.index(this_process_id))

  return sorted(final_pids)


def WaitUntilKillComplete(already_running_ids, max_seconds=30):
  """Prevents main loop from starting until other instance, if any, completes shutdown.

  A termination command send to any other identically-named process may take a few
  seconds to complete because that other process is allowed to finish the current iteration
  in the main loop. Typically, that iteration in the other process will complete before
  this process finishes the initialization and starts. But in limited scenarios, that might
  not happen, such as if the other process is in the middle of generating a lot of
  histogram images, or if this process does not have much data to load.

  This function ensures that this process does not start the main loop until the other
  process terminates. If it detects that the other process is still running, it waits
  for up to max_seconds. If the other process does not terminate before that time limit,
  then this restarts the RPi.
  """
  still_running_ids = FindRunningParents()
  if not still_running_ids:
    return
  # still_running_ids should at least be a subset of already_running_ids
  new_processes = sorted(list(set(still_running_ids).difference(set(already_running_ids))))
  if new_processes:
    # uh-oh! at least one new started up in the interim? exit!
    Log('Kill signal sent to %s from this process %s, but it seems like there is '
        'at least one new process running, %s!' % (
            str(already_running_ids), str(os.getpid()), str(new_processes)))
    sys.exit()
  # phew - they're a subset; so they probably got the signal; just wait a few secs
  elif still_running_ids:
    n = 0
    running_parents = FindRunningParents()
    while running_parents:
      if n == max_seconds:
        Log('Kill signal sent from this process %d to %s, but %s still '
            'running after waiting cume %d seconds; rebooting' % (
                os.getpid(), str(already_running_ids), str(running_parents), n+1))
        PerformGracefulShutdown((), (), True)
      if not n % 3:
        Log('Kill signal sent from this process %d to %s, but %s still '
            'running after waiting cume %d seconds' % (
                os.getpid(), str(already_running_ids), str(running_parents), n))
      n += 1
      time.sleep(1)



      running_parents = FindRunningParents()


def InitArduinos(configuration):
  """Initializes and starts the two arduino threads with new shared-memory queues."""
  to_remote_q = multiprocessing.Queue()
  to_servo_q = multiprocessing.Queue()
  to_main_q = multiprocessing.Queue()
  shutdown_remote = multiprocessing.Value('i')  # shared flag to initiate shutdown
  shutdown_servo = multiprocessing.Value('i')  # shared flag to initiate shutdown
  shutdown = (shutdown_remote, shutdown_servo)

  remote, servo = ValidateArduinosRunning(
      None, None, to_remote_q, to_servo_q, to_main_q, shutdown, configuration)
  return (remote, servo, to_remote_q, to_servo_q, to_main_q, shutdown)


def RefreshArduinos(
    remote, servo,
    to_remote_q, to_servo_q, to_main_q, shutdown,




                            <----SKIPPED LINES---->





def ValidateArduinosRunning(
    remote, servo, to_remote_q, to_servo_q, to_main_q, shutdown, configuration):
  """Ensures that each of the enabled arduinos are running, restarting if needed."""
  remote = ValidateSingleRunning(
      'enable_remote' in configuration,
      arduino.RemoteMain, p=remote,
      args=(to_remote_q, to_main_q, shutdown[0]))
  servo = ValidateSingleRunning(
      'enable_servos' in configuration,
      arduino.ServoMain, p=servo,
      args=(to_servo_q, to_main_q, shutdown[1]))
  return remote, servo


def ValidateSingleRunning(enabled, start_function, p=None, args=()):
  """Restarts a new instance of multiprocessing process if not running"""
  if not SHUTDOWN_SIGNAL:

    if not enabled:
      if p is not None:  # must have just requested a disabling of single instance
        args[2].value = 1  # trigger a shutdown on the single instance
      return None

    if p is None or not p.is_alive():
      if p is None:
        Log('Process for %s starting for first time' % str(start_function))
      else:



        Log('Process (%s) for %s died; restarting' % (str(p), str(start_function)))
      args[2].value = 0  # (re)set shutdown flag to allow function to run
      p = multiprocessing.Process(target=start_function, args=args)
      p.daemon = True
      p.start()

  return p


def EnqueueArduinos(flights, json_desc_dict, configuration, to_servo_q, to_remote_q):
  """Send latest data to arduinos via their shared-memory queues"""
  last_flight = {}
  if flights:
    last_flight = dict(flights[-1])

  if SIMULATION:
    now = json_desc_dict['now']
  else:
    now = time.time()






                            <----SKIPPED LINES---->




      messageboard_flight_index = IdentifyFlightDisplayed(
          flights, configuration, display_all_hours=True)
      if messageboard_flight_index is not None:
        message_queue = [m for m in message_queue if m[0] != FLAG_MSG_INTERESTING]
        flight_message = CreateMessageAboutFlight(flights[messageboard_flight_index])
        message_queue = [(FLAG_MSG_FLIGHT, flight_message)]
        next_message_time = time.time()

    elif command == 'histogram':
      if not flights:
        Log('Histogram requested by remote %s but no flights in memory' % str(args))
      else:
        histogram_type, histogram_history = args
        message_queue.extend(MessageboardHistograms(
            flights,
            histogram_type,
            histogram_history,
            '_1',
            False))

    elif command == 'update_configuration':
      WriteFile(CONFIG_FILE, *args)

    else:
      Log('Improper command from arduinos: %s / %s' % (command, args))

  return message_queue, next_message_time


def PublishMessage(
    s,
    subscription_id='12fd73cd-75ef-4cae-bbbf-29b2678692c1',
    key='c5f62d44-e30d-4c43-a43e-d4f65f4eb399',
    secret='b00aeb24-72f3-467c-aad2-82ba5e5266ca',
    timeout=3):
  """Publishes a text string to a Vestaboard.

  The message is pushed to the vestaboard splitflap display by way of its web services;
  see https://docs.vestaboard.com/introduction for more details.

  Args:




                            <----SKIPPED LINES---->




      'https://platform.vestaboard.com/subscriptions/%s/message' % subscription_id)
  curl.setopt(pycurl.HTTPHEADER, [
      'X-Vestaboard-Api-Key:%s' % key, 'X-Vestaboard-Api-Secret:%s' % secret])
  curl.setopt(pycurl.TIMEOUT_MS, timeout*1000)
  curl.setopt(pycurl.POST, 1)

  curl.setopt(pycurl.WRITEFUNCTION, lambda x: None) # to keep stdout clean

  # preparing body the way pycurl.READDATA wants it
  body_as_dict = {'text': s}
  body_as_json_string = json.dumps(body_as_dict) # dict to json
  body_as_file_object = io.StringIO(body_as_json_string)

  # prepare and send. See also: pycurl.READFUNCTION to pass function instead
  curl.setopt(pycurl.READDATA, body_as_file_object)
  curl.setopt(pycurl.POSTFIELDSIZE, len(body_as_json_string))
  try:
    curl.perform()
  except pycurl.error as e:
    Log('curl.perform() failed with message %s' % e)

    error_code = True
  else:
    # you may want to check HTTP response code, e.g.
    status_code = curl.getinfo(pycurl.RESPONSE_CODE)
    if status_code != 200:
      Log('Server returned HTTP status code %d for message %s' % (status_code, s))

      error_code = True

  curl.close()

  UpdateStatusLight(GPIO_ERROR_VESTABOARD_CONNECTION, error_code)


def ManageMessageQueue(message_queue, next_message_time, configuration):
  """Check time & if appropriate, display next message from queue.

  Args:
    message_queue: FIFO list of message tuples of (message type, message string).
    next_message_time: epoch at which next message should be displayed
    configuration: dictionary of configuration attributes.

  Returns:
    Next_message_time, potentially updated if a message has been displayed, or unchanged
    if no message was displayed.
  """
  if message_queue and (time.time() >= next_message_time or SIMULATION):

    if SIMULATION:  # drain the queue because the messages come so fast
      messages_to_display = list(message_queue)
      # passed by reference, so clear it out since we drained it to the display
      del message_queue[:]




                            <----SKIPPED LINES---->




  be considered correct.

  They are "correct" in the sense that that new insight was not available at the time
  that older flight was seen, but it is not correct in the sense that, because this new
  insight is starting out with an incidence in the historical data of zero, this
  new insight may be reported more frequently than desired until it "catches up".

  So this method replays the flight history with the latest insight code, regenerating
  the insight distribution for each flight.
  """
  directory, file = os.path.split(full_path)
  all_files = os.listdir(directory)
  files = sorted([os.path.join(directory, f) for f in all_files if file in f])
  for f in files:

    print('Bootstrapping %s' % f)
    configuration = ReadAndParseSettings(CONFIG_FILE)
    flights = []
    tmp_f = f + 'tmp'

    RemoveFile(tmp_f)


    if os.path.exists(f):
      mtime = os.path.getmtime(f)
      flights = UnpickleObjectFromFile(f, False)
      for (n, flight) in enumerate(flights):
        if n/25 == int(n/25):
          print(' - %d' % n)
        CreateFlightInsights(flights[:n+1], configuration.get('insights', 'hide'), {})
        PickleObjectToFile(flight, tmp_f, False)

      if mtime == os.path.getmtime(f):
        shutil.move(tmp_f, f)
      else:
        print('Aborted: failed to bootstrap %s: file changed while in process' % full_path)
        return


def ResetLogs(config):
  """Clears the non-scrolling logs if reset_logs in config."""
  if 'reset_logs' in config:
    Log('Reset logs')
    for f in (STDERR_FILE, BACKUP_FILE, SERVICE_VERIFICATION_FILE):
      if RemoveFile(f):

        open(f, 'a').close()
    config.pop('reset_logs')
    config = BuildSettings(config)
    WriteFile(CONFIG_FILE, config)
  return config


def CheckTemperature(fan_power):
  """Turn on fan if temperature exceeds threshold."""
  if RASPBERRY_PI:
    temperature = gpiozero.CPUTemperature().temperature
    if temperature > TEMP_FAN_TURN_ON_CELSIUS and not fan_power:
      fan_power = True
      RPi.GPIO.output(GPIO_FAN, True)
      Log('Fan turned on at temperature %.1f' % temperature)
    elif temperature < TEMP_FAN_TURN_OFF_CELSIUS and fan_power:
      fan_power = False
      RPi.GPIO.output(GPIO_FAN, False)
      Log('Fan turned off at temperature %.1f' % temperature)
  return fan_power


pin_values = {}  # caches last set value
def SetPinMode():
  """Initialize output GPIO pins for output on Raspberry Pi."""
  global pin_values

  if RASPBERRY_PI:
    RPi.GPIO.setmode(RPi.GPIO.BCM)

  for pin in (
      GPIO_ERROR_VESTABOARD_CONNECTION, GPIO_ERROR_FLIGHT_AWARE_CONNECTION,
      GPIO_ERROR_ARDUINO_SERVO_CONNECTION, GPIO_ERROR_ARDUINO_REMOTE_CONNECTION,
      GPIO_ERROR_BATTERY_CHARGE, GPIO_FAN, GPIO_UNUSED_1, GPIO_UNUSED_2):

    # Initialize some pins to start in error condition
    if pin in (
        GPIO_ERROR_ARDUINO_SERVO_CONNECTION, GPIO_ERROR_ARDUINO_REMOTE_CONNECTION,
        GPIO_UNUSED_1, GPIO_UNUSED_2):
      pin_values[pin[0]] = True
    else:
      pin_values[pin[0]] = False

    if RASPBERRY_PI:
      RPi.GPIO.setup(pin[0], RPi.GPIO.OUT)
      RPi.GPIO.output(pin[0], pin_values[pin[0]])

  if RASPBERRY_PI:  # configure soft reset button
    RPi.GPIO.setup(GPIO_SOFT_RESET[0], RPi.GPIO.IN, pull_up_down=RPi.GPIO.PUD_DOWN)
    RPi.GPIO.setup(GPIO_SOFT_RESET[1], RPi.GPIO.OUT)
    RPi.GPIO.output(GPIO_SOFT_RESET[1], True)
    RPi.GPIO.add_event_detect(GPIO_SOFT_RESET[0], RPi.GPIO.RISING)
    RPi.GPIO.add_event_callback(GPIO_SOFT_RESET[0], InterruptRebootFromButton)


def UpdateStatusLight(pin, value):
  """Sets the Raspberry Pi GPIO pin high (True) or low (False) based on value."""
  global pin_values

  if value:
    msg = pin[1]
  else:
    msg = pin[2]
  if RASPBERRY_PI:
    RPi.GPIO.output(pin[0], value)
    if value:
      pin_setting = 'HIGH'
      relay_light_value = 'OFF'
    else:
      pin_setting = 'LOW'
      relay_light_value = 'ON'
    msg += '; RPi GPIO pin %d set to %s; relay light #%d should now be %s' % (
        pin[0], pin_setting, pin[3], relay_light_value)

  if pin_values[pin[0]] != value:
    Log(msg)
    pin_values[pin[0]] = value


def RemoveFile(file):
  """Removes a file if it exists, returning a boolean indicating if it had existed."""
  if os.path.exists(file):
    os.remove(file)
    return True
  return False


def main():
  """Traffic cop between incoming radio flight messages, configuration, and messageboard.

  This is the main logic, checking for new flights, augmenting the radio signal with
  additional web-scraped data, and generating messages in a form presentable to the
  messageboard.
  """
  RemoveFile(LOGFILE_LOCK)

  # Since this clears log files, it should occur first before we start logging
  if '-s' in sys.argv:
    global SIMULATION_COUNTER
    SimulationSetup()

  # This flag slows down simulation time around a flight, great for debugging the arduinos
  simulation_slowdown = bool('-f' in sys.argv)

  # Redirect any errors to a log file instead of the screen, and add a datestamp
  if not SIMULATION:
    sys.stderr = open(STDERR_FILE, 'a')
    Log('', STDERR_FILE)

  Log('Starting up process %d' % os.getpid())
  already_running_ids = FindRunningParents()
  if already_running_ids:
    for pid in already_running_ids:
      Log('Sending termination signal to %d' % pid)
      os.kill(pid, signal.SIGTERM)

  SetPinMode()


  fan_power = False

  configuration = ReadAndParseSettings(CONFIG_FILE)
  startup_time = time.time()
  json_desc_dict = {}

  flights = UnpickleObjectFromFile(PICKLE_FLIGHTS, True, max_days=MAX_INSIGHT_HORIZON_DAYS)
  # Clear the loaded flight of any cached data, identified by keys with a specific
  # suffix, since code fixes may change the values for some of those cached elements
  for flight in flights:
    for key in list(flight.keys()):
      if key.endswith(CACHED_ELEMENT_PREFIX):
        flight.pop(key)

  # If we're displaying just a single insight message, we want it to be something
  # unique, to the extent possible; this dict holds a count of the diff types of messages
  # displayed so far
  insight_message_distribution = {}

  # bootstrap the flight insights distribution from a list of insights on each




                            <----SKIPPED LINES---->




           configuration.get('log_jsons', False))

      if flight:
        flights.append(flight)
        remote, servo = RefreshArduinos(
            remote, servo,
            to_remote_q, to_servo_q, to_main_q, shutdown,
            flights, json_desc_dict, configuration)

        flight_meets_display_criteria = FlightMeetsDisplayCriteria(
            flight, configuration, log=True)
        if flight_meets_display_criteria:
          flight_message = (FLAG_MSG_FLIGHT, CreateMessageAboutFlight(flight))

          # display the next message about this flight now!
          next_message_time = time.time()
          message_queue.insert(0, flight_message)
          # and delete any queued insight messages about other flights that have
          # not yet displayed, since a newer flight has taken precedence
          messages_to_delete = [m for m in message_queue if m[0] == FLAG_MSG_INTERESTING]
          if messages_to_delete and VERBOSE:
            Log(
                'Deleting messages from queue due to new-found plane: %s' %
                messages_to_delete)
          message_queue = [m for m in message_queue if m[0] != FLAG_MSG_INTERESTING]

          # Though we also manage the message queue outside this conditional as well,
          # because it can take a half second to generate the flight insights, this allows
          # this message to start displaying on the board immediately, so it's up there
          # when it's most relevant
          next_message_time = ManageMessageQueue(
              message_queue, next_message_time, configuration)

          insight_messages = CreateFlightInsights(
              flights, configuration.get('insights'), insight_message_distribution)
          if configuration.get('next_flight', 'off') == 'on':
            next_flight_text = FlightInsightNextFlight(flights, configuration)
            if next_flight_text:
              insight_messages.insert(0, next_flight_text)

          insight_messages = [(FLAG_MSG_INTERESTING, m) for m in insight_messages]




                            <----SKIPPED LINES---->





        PickleObjectToFile(flight, PICKLE_FLIGHTS, not SIMULATION)

      else:
        remote, servo = RefreshArduinos(
            remote, servo,
            to_remote_q, to_servo_q, to_main_q, shutdown,
            flights, json_desc_dict, configuration)

    message_queue, next_message_time = ProcessArduinoCommmands(
        to_main_q, flights, configuration, message_queue, next_message_time)

    if SIMULATION:
      if now:
        simulated_hour = EpochDisplayTime(now, '%Y-%m-%d %H:00%z')
      if simulated_hour != prev_simulated_hour:
        print(simulated_hour)
        prev_simulated_hour = simulated_hour

    histogram = ReadAndParseSettings(HISTOGRAM_CONFIG_FILE)

    RemoveFile(HISTOGRAM_CONFIG_FILE)

    # We also need to make sure there are flights on which to generate a histogram! Why
    # might there not be any flights? Primarily during a simulation, if there's a
    # lingering histogram file at the time of history restart.
    if histogram and not flights:
      Log('Histogram requested (%s) but no flights in memory' % histogram)
    if histogram and flights:
      message_queue.extend(TriggerHistograms(flights, histogram))

    # check time & if appropriate, display next message from queue
    next_message_time = ManageMessageQueue(message_queue, next_message_time, configuration)

    reboot = CheckRebootNeeded(startup_time, message_queue, json_desc_dict, configuration)

    fan_power = CheckTemperature(fan_power)

    if not SIMULATION:
      time.sleep(max(0, next_loop_time - time.time()))
      next_loop_time = time.time() + LOOP_DELAY_SECONDS
    else:
      SIMULATION_COUNTER += 1
      if simulation_slowdown:
        SimulationSlowdownNearFlight(flights, persistent_nearby_aircraft)
    if SHUTDOWN_SIGNAL:  # do a graceful exit
      PerformGracefulShutdown((to_remote_q, to_servo_q, to_main_q), shutdown, reboot)

  if SIMULATION:
    SimulationEnd(message_queue, flights)
  PerformGracefulShutdown((to_remote_q, to_servo_q, to_main_q), shutdown, reboot)


if __name__ == "__main__":
  #interrupt, as in ctrl-c
  signal.signal(signal.SIGINT, InterruptShutdownFromSignal)

  #terminate, when another instance found or via kill
  signal.signal(signal.SIGTERM, InterruptShutdownFromSignal)

  if '-i' in sys.argv:
    BootstrapInsightList()
  else:
    main()