#!/usr/bin/python3

import csv
import datetime
import gc
import io
import json
import math
import multiprocessing
import numbers
import os
import pickle
import queue
import re
import shutil
import signal
import statistics
import sys
import textwrap
import time
import tracemalloc
import types





                            <----SKIPPED LINES---->




# dropped from the nearby list
PERSISTENCE_SECONDS = 300
TRUNCATE = 50  # max number of keys to include in a histogram image file
# number of seconds to pause between each radio poll / command processing loop
LOOP_DELAY_SECONDS = 1

# number of seconds to wait between recording heartbeats to the status file
HEARTBEAT_SECONDS = 10

# version control directory
CODE_REPOSITORY = ''
VERSION_REPOSITORY = 'versions/'
VERSION_WEBSITE_PATH = VERSION_REPOSITORY
VERSION_MESSAGEBOARD = None
VERSION_ARDUINO = None
# histogram logic truncates to exactly 30 days of hours
MAX_INSIGHT_HORIZON_DAYS = 31

# This file is where the radio drops its json file
DUMP_JSON_FILE = '/run/readsb/aircraft.json'

# This file is where the query history to the flight aware webpage goes
FLIGHTAWARE_HISTORY_FILE = 'secure/flightaware.txt'

# At the time a flight is first identified as being of interest (in that
# it falls within MIN_METERS meters of HOME), it - and core attributes
# derived from FlightAware, if any - is appended to the end of this pickle
# file. However, since this file is cached in working memory, flights older
# than 30 days are flushed from this periodically.
PICKLE_FLIGHTS = 'pickle/flights.pk'

# This allows us to identify the full history (including what was last sent
# to the splitflap display in a programmatic fashion. While it may be
# interesting in its own right, its real use is to handle the "replay"
# button, so we know to enable it if what is displayed is the last flight.
PICKLE_SCREENS = 'pickle/screens.pk'

# Status data about messageboard systems - are they running, etc.  Specifically,
# has tuples of data (timestamp, system_id, status), where system_id is either
# the pin id of GPIO, or a 0 to indicate overall system, and status is boolean
PICKLE_SYSTEM_DASHBOARD = 'pickle/dashboard.pk'

# Flight-centric status data - what time was the flight first detected as




                            <----SKIPPED LINES---->




  PICKLE_FLIGHTS = MESSAGEBOARD_PATH + PICKLE_FLIGHTS
  PICKLE_SYSTEM_DASHBOARD = MESSAGEBOARD_PATH + PICKLE_SYSTEM_DASHBOARD
  PICKLE_FLIGHT_DASHBOARD = MESSAGEBOARD_PATH + PICKLE_FLIGHT_DASHBOARD
  LOGFILE = MESSAGEBOARD_PATH + LOGFILE
  PICKLE_DUMP_JSON_FILE = MESSAGEBOARD_PATH + PICKLE_DUMP_JSON_FILE
  PICKLE_FA_JSON_FILE = MESSAGEBOARD_PATH + PICKLE_FA_JSON_FILE
  PICKLE_SCREENS = MESSAGEBOARD_PATH + PICKLE_SCREENS
  CODE_REPOSITORY = MESSAGEBOARD_PATH

  HISTOGRAM_CONFIG_FILE = WEBSERVER_PATH + HISTOGRAM_CONFIG_FILE
  CONFIG_FILE = WEBSERVER_PATH + CONFIG_FILE
  ROLLING_MESSAGE_FILE = WEBSERVER_PATH + ROLLING_MESSAGE_FILE
  ALL_MESSAGE_FILE = WEBSERVER_PATH + ALL_MESSAGE_FILE
  ROLLING_LOGFILE = WEBSERVER_PATH + ROLLING_LOGFILE
  STDERR_FILE = WEBSERVER_PATH + STDERR_FILE
  BACKUP_FILE = WEBSERVER_PATH + BACKUP_FILE
  SERVICE_VERIFICATION_FILE = WEBSERVER_PATH + SERVICE_VERIFICATION_FILE
  UPTIMES_FILE = WEBSERVER_PATH + UPTIMES_FILE
  CODE_HISTORY_FILE = WEBSERVER_PATH + CODE_HISTORY_FILE
  NEW_AIRCRAFT_FILE = WEBSERVER_PATH + NEW_AIRCRAFT_FILE
  FLIGHTAWARE_HISTORY_FILE = WEBSERVER_PATH + FLIGHTAWARE_HISTORY_FILE

  HISTOGRAM_IMAGE_HTML = WEBSERVER_PATH + HISTOGRAM_IMAGE_HTML
  HOURLY_IMAGE_FILE = (
      WEBSERVER_PATH + WEBSERVER_IMAGE_RELATIVE_FOLDER + HOURLY_IMAGE_FILE)
  VERSION_REPOSITORY = WEBSERVER_PATH + VERSION_REPOSITORY

TIMEZONE = 'US/Pacific' # timezone of display
TZ = pytz.timezone(TIMEZONE)

# iata codes that we don't need to expand
KNOWN_AIRPORTS = ('SJC', 'SFO', 'OAK')

SPLITFLAP_CHARS_PER_LINE = 22
SPLITFLAP_LINE_COUNT = 6

DIRECTIONS_4 = ['N', 'E', 'S', 'W']
DIRECTIONS_8 = ['N', 'NE', 'E', 'SE', 'S', 'SW', 'W', 'NW']
DIRECTIONS_16 = ['N', 'NNE', 'NE', 'ENE', 'E', 'ESE', 'SE', 'SSE',
                 'S', 'SSW', 'SW', 'WSW', 'W', 'WNW', 'NW', 'NNW']





                            <----SKIPPED LINES---->




      newly_nearby_flight_identifiers.append(flight_identifier)
    persistent_nearby_aircraft[flight_identifier] = now

  flights_to_delete = []
  for flight_identifier in persistent_nearby_aircraft:
    if (flight_identifier not in current_nearby_aircraft
        and (now - persistent_nearby_aircraft[flight_identifier]) >
        PERSISTENCE_SECONDS):
      flights_to_delete.append(flight_identifier)
  for flight_identifier in flights_to_delete:
    del persistent_nearby_aircraft[flight_identifier]
  return newly_nearby_flight_identifiers


def ScanForNewFlights(
    persistent_nearby_aircraft, persistent_path, log_jsons, flights):
  """Determines if there are any new aircraft in the radio message.

  The radio is continuously dumping new json messages to the Raspberry pi with
  all the flights currently observed. This function picks up the latest radio
  json, and for  any new nearby flights - there should generally be at most one
  new flight on each pass through - gets additional flight data from
  FlightAware and augments the flight definition with the relevant fields to
  keep.

  Args:
    persistent_nearby_aircraft: dictionary where keys are flight numbers, and
      the values are the time the flight was last seen.
    persistent_path: dictionary where keys are flight numbers, and the values
      are a sequential list of the location-attributes in the json file; allows
      for tracking the flight path over time.
    log_jsons: boolean indicating whether we should pickle the JSONs.
    flights: list of flight dictionaries; if no json is returned, used to
      find a recent flight with same flight number to augment this flight with
      origin / destination / airline.

  Returns:
    A tuple:
    - updated persistent_nearby_aircraft
    - (possibly empty) dictionary of flight attributes of the new flight upon
      its first observation.
    - the time of the radio observation if present; None if no radio dump
    - a dictionary of attributes about the dump itself (i.e.: # of flights;
      furthest observed flight, etc.)
    - persistent_path, a data structure containing past details of a flight's
      location as described in ParseDumpJson
    - a text message indicating any errors in querying FlightAware or
      populating flight details


  """
  flight_details = {}
  error_message = ''
  now = time.time()
  if SIMULATION:
    (dump_json, json_time) = DUMP_JSONS[SIMULATION_COUNTER]
  else:
    dump_json = ReadFile(DUMP_JSON_FILE, log_exception=True)

  json_desc_dict = {}
  current_nearby_aircraft = {}
  if dump_json:
    (current_nearby_aircraft, now,
     json_desc_dict, persistent_path) = ParseDumpJson(
         dump_json, persistent_path)

    if not SIMULATION and log_jsons:
      PickleObjectToFile((dump_json, now), PICKLE_DUMP_JSON_FILE, True)

    newly_nearby_flight_identifiers = UpdateAircraftList(




                            <----SKIPPED LINES---->




    if newly_nearby_flight_identifiers:

      if len(newly_nearby_flight_identifiers) > 1:
        # newly_nearby_flight_identifiers is a list of 2-tuples, where each
        # tuple is (flight_number, squawk)
        newly_nearby_flight_identifiers_str = ', '.join(
            ['%s/%s ' % (*i,) for i in newly_nearby_flight_identifiers])
        newly_nearby_flight_details_str = '\n'.join([
            str(current_nearby_aircraft[f])
            for f in newly_nearby_flight_identifiers])
        Log('Multiple newly-nearby flights: %s\n%s' % (
            newly_nearby_flight_identifiers_str,
            newly_nearby_flight_details_str))
      flight_identifier = newly_nearby_flight_identifiers[0]

      flight_aware_json = {}
      if SIMULATION:
        json_times = [j[1] for j in FA_JSONS]
        if json_time in json_times:
          flight_aware_json = FA_JSONS[json_times.index(json_time)][0]

      elif flight_identifier[0]:
        flight_number = flight_identifier[0]
        flight_aware_json, error_message = GetFlightAwareJson(flight_number)

        if flight_aware_json:
          UpdateStatusLight(GPIO_ERROR_FLIGHT_AWARE_CONNECTION, False)
        else:
          failure_message = 'No json from Flightaware for flight %s: %s' % (
              flight_number, error_message[:500])
          Log(failure_message)
          UpdateStatusLight(
              GPIO_ERROR_FLIGHT_AWARE_CONNECTION, True, failure_message)

      flight_details = {}
      if flight_aware_json:
        flight_details = ParseFlightAwareJson(flight_aware_json)
      elif flight_identifier[0]:  # if there's a flight number but no json
        flight_details, derived_attr_msg = FindAttributesFromSimilarFlights(
            flight_identifier[0], flights)
        error_message += derived_attr_msg

      if not SIMULATION and log_jsons:
        PickleObjectToFile((flight_aware_json, now), PICKLE_FA_JSON_FILE, True)

      # Augment FlightAware details with radio / radio-derived details
      flight_details.update(current_nearby_aircraft[flight_identifier])

      # Augment with the past location data; the [1] is because recall that
      # persistent_path[key] is actually a 2-tuple, the first element being
      # the most recent time seen, and the second element being the actual
      # path. But we do not need to keep around the most recent time seen any
      # more.
      flight_details['persistent_path'] = persistent_path[flight_identifier][1]

  return (
      persistent_nearby_aircraft,
      flight_details,
      now,
      json_desc_dict,
      persistent_path,
      error_message)



def DescribeDumpJson(parsed):
  """Generates dict with descriptive attributes about the dump json file.

  Args:
    parsed: The parsed json file.

  Returns:
    Dictionary with attributes about radio range, number of flights seen, etc.
  """
  json_desc_dict = {}
  json_desc_dict['now'] = parsed['now']

  aircraft = [a for a in parsed['aircraft'] if a['seen'] < PERSISTENCE_SECONDS]
  json_desc_dict['radio_range_flights'] = len(aircraft)

  aircraft_with_pos = [a for a in aircraft if 'lat' in a and 'lon' in a]
  current_distances = [HaversineDistanceMeters(
      HOME, (a['lat'], a['lon'])) for a in aircraft_with_pos]




                            <----SKIPPED LINES---->




        # just simply need to continue updating this list to keep the
        # dictionary up to date (i.e.: we don't need to directly touch the
        # flights dictionary in main).
        (last_seen, current_path) = persistent_path.get(id_to_use, (None, []))
        if (  # flight position has been updated with this radio signal
            not current_path or
            simplified_aircraft.get('lat') != current_path[-1].get('lat') or
            simplified_aircraft.get('lon') != current_path[-1].get('lon')):
          current_path.append(simplified_aircraft)
        persistent_path[id_to_use] = (now, current_path)

  # if the flight was last seen too far in the past, remove the track info
  for f in list(persistent_path.keys()):
    (last_seen, current_path) = persistent_path[f]
    if last_seen < now - PERSISTENCE_SECONDS:
      persistent_path.pop(f)

  return (nearby_aircraft, now, json_desc_dict, persistent_path)


def LogFlightAwareQuery(flight_number, error_message=''):
  """Log whenever we are about to attempt a query on FlightAware.

  To help troubleshoot the apparent frequency of FlightAware queries, we
  will log when we *desire* to make the queries (and for what flights),
  irrespective of whether such queries are throttled before the attempt,
  successful, or fail for some other reason.

  Specifically, we will log in a dedicated text file three fields:
  - the timestamp (epoch)
  - the flight number
  - the failure message, if any
  """
  with open(FLIGHTAWARE_HISTORY_FILE, 'a', newline='') as csv_file:
    writer = csv.writer(
        csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
    writer.writerow([time.time(), flight_number, error_message])


last_query_time = 0
def GetFlightAwareJson(flight_number):
  """Scrapes the text json message from FlightAware for a given flight number.

  Given a flight number, loads the corresponding FlightAware webpage for that
  flight and extracts the relevant script that contains all the flight details
  from that page.  But only queries at most once per fixed period of time
  so as to avoid being blocked.

  Args:
    flight_number: text flight number (i.e.: SWA1234)

  Returns:
    Two tuple:
     - Text representation of the json message from FlightAware.
     - Text string of error message, if any

  """
  min_query_delay_seconds = 90
  url = 'https://flightaware.com/live/flight/' + flight_number

  global last_query_time
  seconds_since_last_query = time.time() - last_query_time
  if last_query_time and seconds_since_last_query < min_query_delay_seconds:
    error_msg = (
        'Unable to query FA for URL since last query to FA was only %d seconds '
        'ago; min of %d seconds needed: %s' % (


            seconds_since_last_query, min_query_delay_seconds, url))
    LogFlightAwareQuery(flight_number, error_msg)
    return '', error_msg

  last_query_time = time.time()

  try:
    response = requests.get(url, timeout=5)

  except requests.exceptions.RequestException as e:
    error_msg = 'Unable to query FA for URL due to %s: %s' % (e, url)
    LogFlightAwareQuery(flight_number, error_msg)
    return '', error_msg

  LogFlightAwareQuery(flight_number)
  soup = bs4.BeautifulSoup(response.text, 'html.parser')
  l = soup.find_all('script')
  flight_script = None
  for script in l:
    if 'trackpollBootstrap' in str(script):
      flight_script = str(script)
      break
  if not flight_script:
    error_msg = (
        'Unable to find trackpollBootstrap script in page: ' + response.text)
    Log(error_msg)
    return '', error_msg
  first_open_curly_brace = flight_script.find('{')
  last_close_curly_brace = flight_script.rfind('}')
  flight_json = flight_script[first_open_curly_brace:last_close_curly_brace+1]
  return flight_json, ''


def Unidecode(s):
  """Convert a special unicode characters to closest ASCII representation."""
  if s is not None:
    s = unidecode.unidecode(s)
  return s


def FindAttributesFromSimilarFlights(this_flight_number, flights):
  """Returns a dictionary with info about a flight based on other flights.

  We may not get a json from the internet about this flight for any number
  of reasons: internet down; website down; too frequent queries; etc.  However,
  there are still some basic attributes we can derive about this flight
  from past observations of the same flight number, or past observations
  about the flight number prefix.  Specifically, we can get the flight's
  airline, origin, and destination.

  Args:




                            <----SKIPPED LINES---->




        if n/25 == int(n/25):
          print(' - %d' % n)
        CreateFlightInsights(
            flights[:n+1], configuration.get('insights', 'hide'), {})
        PickleObjectToFile(flight, tmp_f, False)

      if mtime == os.path.getmtime(f):
        shutil.move(tmp_f, f)
      else:
        print('Aborted: failed to bootstrap %s: file changed while in process'
              % full_path)
        return


def ResetLogs(config):
  """Clears the non-scrolling logs if reset_logs in config."""
  if 'reset_logs' in config:
    removed_files = []
    for f in (
        STDERR_FILE, BACKUP_FILE, SERVICE_VERIFICATION_FILE,
        NEW_AIRCRAFT_FILE, FLIGHTAWARE_HISTORY_FILE):
      if RemoveFile(f):
        removed_files.append(f)
        open(f, 'a').close()
    Log('Reset logs: cleared files %s' % '; '.join(removed_files))
    config.pop('reset_logs')
    config = BuildSettings(config)
    WriteFile(CONFIG_FILE, config)
  return config


def CheckTemperature(last_logged=0):
  """Turn on fan if temperature exceeds threshold.

  Args:
    last_logged: epoch at which temperature was last logged.

  Returns:
    Epoch at which temperature was last logged.
  """
  if RASPBERRY_PI:




                            <----SKIPPED LINES---->





    ResetLogs(configuration)  # clear the logs if requested
    UpdateRollingLogSize(configuration)

    # if this is a SIMULATION, then process every diff dump. But if it
    # isn't a simulation, then only read & do related processing for the
    # next dump if the last-modified timestamp indicates the file has been
    # updated since it was last read.
    tmp_timestamp = 0
    if not SIMULATION:
      dump_json_exists = os.path.exists(DUMP_JSON_FILE)
      if dump_json_exists:
        tmp_timestamp = os.path.getmtime(DUMP_JSON_FILE)
    if (SIMULATION and DumpJsonChanges()) or (
        not SIMULATION and dump_json_exists and
        tmp_timestamp > last_dump_json_timestamp):

      last_dump_json_timestamp = tmp_timestamp

      (persistent_nearby_aircraft, flight, now, json_desc_dict,
       persistent_path, flight_aware_error_message) = ScanForNewFlights(

           persistent_nearby_aircraft,
           persistent_path,
           configuration.get('log_jsons', False),
           flights)

      # Logging: As part of the memory instrumentation, let's track
      # the length of these data structures
      if not iteration % 1000:
        lengths = [len(flights), len(screen_history)]
        Log('Iteration: %d: object lengths: %s' % (iteration, lengths))

      # because this might just be an updated instance of the previous
      # flight as more identifier information (squawk and or flight number)
      # comes in, we only want to process this if its a truly new flight
      new_flight_flag = ConfirmNewFlight(flight, flights)

      if new_flight_flag:
        time_new_flight_found = time.time()

        # Since we no longer need the memory-hogging prior persistent_path
        # stored on the flights, we can remove it
        if flights and 'persistent_path' in flights[-1]:
          del flights[-1]['persistent_path']





                            <----SKIPPED LINES---->





        PickleObjectToFile(
            flight, PICKLE_FLIGHTS, True, timestamp=flight['now'])

        # We record flight number, flight_meets_display_criteria,
        # reason_flight_fails_criteria, flight_aware_error_message, and
        # time stamps of when we confirmed new flight; when we generated
        # message; when we generated insight messages (if any), to a
        # to a pickle file so that we may construct a flight-centric
        # status report
        #
        # Specifically, we record a 3-tuple:
        # - flight number
        # - time stamp of the recording
        # - a dictionary of elements
        data = (
            DisplayFlightNumber(flight),
            time.time(),
            {
                'reason_flight_fails_criteria': reason_flight_fails_criteria,

                'flight_aware_error_message': flight_aware_error_message,
                'time_new_flight_found': time_new_flight_found,
                'time_flight_message_inserted': time_flight_message_inserted,
                'time_insight_message_inserted': time_insight_message_inserted})
        PickleObjectToFile(data, PICKLE_FLIGHT_DASHBOARD, True)

      else:
        remote, servo = RefreshArduinos(
            remote, servo,
            to_remote_q, to_servo_q, to_main_q, shutdown,
            flights, json_desc_dict, configuration, screen_history)

    message_queue, next_message_time = ProcessArduinoCommmands(
        to_main_q, flights, configuration, message_queue, next_message_time)

    personal_message = PersonalMessage(
        configuration, message_queue, personal_message)

    # MEMORY MANAGEMENT
    # it turns out we only need the last screen in the screen history, not




                            <----SKIPPED LINES---->

#!/usr/bin/python3


import datetime
import gc
import io
import json
import math
import multiprocessing
import numbers
import os
import pickle
import queue
import re
import shutil
import signal
import statistics
import sys
import textwrap
import time
import tracemalloc
import types





                            <----SKIPPED LINES---->




# dropped from the nearby list
PERSISTENCE_SECONDS = 300
TRUNCATE = 50  # max number of keys to include in a histogram image file
# number of seconds to pause between each radio poll / command processing loop
LOOP_DELAY_SECONDS = 1

# number of seconds to wait between recording heartbeats to the status file
HEARTBEAT_SECONDS = 10

# version control directory
CODE_REPOSITORY = ''
VERSION_REPOSITORY = 'versions/'
VERSION_WEBSITE_PATH = VERSION_REPOSITORY
VERSION_MESSAGEBOARD = None
VERSION_ARDUINO = None
# histogram logic truncates to exactly 30 days of hours
MAX_INSIGHT_HORIZON_DAYS = 31

# This file is where the radio drops its json file
DUMP_JSON_FILE = '/run/readsb/aircraft.json'




# At the time a flight is first identified as being of interest (in that
# it falls within MIN_METERS meters of HOME), it - and core attributes
# derived from FlightAware, if any - is appended to the end of this pickle
# file. However, since this file is cached in working memory, flights older
# than 30 days are flushed from this periodically.
PICKLE_FLIGHTS = 'pickle/flights.pk'

# This allows us to identify the full history (including what was last sent
# to the splitflap display in a programmatic fashion. While it may be
# interesting in its own right, its real use is to handle the "replay"
# button, so we know to enable it if what is displayed is the last flight.
PICKLE_SCREENS = 'pickle/screens.pk'

# Status data about messageboard systems - are they running, etc.  Specifically,
# has tuples of data (timestamp, system_id, status), where system_id is either
# the pin id of GPIO, or a 0 to indicate overall system, and status is boolean
PICKLE_SYSTEM_DASHBOARD = 'pickle/dashboard.pk'

# Flight-centric status data - what time was the flight first detected as




                            <----SKIPPED LINES---->




  PICKLE_FLIGHTS = MESSAGEBOARD_PATH + PICKLE_FLIGHTS
  PICKLE_SYSTEM_DASHBOARD = MESSAGEBOARD_PATH + PICKLE_SYSTEM_DASHBOARD
  PICKLE_FLIGHT_DASHBOARD = MESSAGEBOARD_PATH + PICKLE_FLIGHT_DASHBOARD
  LOGFILE = MESSAGEBOARD_PATH + LOGFILE
  PICKLE_DUMP_JSON_FILE = MESSAGEBOARD_PATH + PICKLE_DUMP_JSON_FILE
  PICKLE_FA_JSON_FILE = MESSAGEBOARD_PATH + PICKLE_FA_JSON_FILE
  PICKLE_SCREENS = MESSAGEBOARD_PATH + PICKLE_SCREENS
  CODE_REPOSITORY = MESSAGEBOARD_PATH

  HISTOGRAM_CONFIG_FILE = WEBSERVER_PATH + HISTOGRAM_CONFIG_FILE
  CONFIG_FILE = WEBSERVER_PATH + CONFIG_FILE
  ROLLING_MESSAGE_FILE = WEBSERVER_PATH + ROLLING_MESSAGE_FILE
  ALL_MESSAGE_FILE = WEBSERVER_PATH + ALL_MESSAGE_FILE
  ROLLING_LOGFILE = WEBSERVER_PATH + ROLLING_LOGFILE
  STDERR_FILE = WEBSERVER_PATH + STDERR_FILE
  BACKUP_FILE = WEBSERVER_PATH + BACKUP_FILE
  SERVICE_VERIFICATION_FILE = WEBSERVER_PATH + SERVICE_VERIFICATION_FILE
  UPTIMES_FILE = WEBSERVER_PATH + UPTIMES_FILE
  CODE_HISTORY_FILE = WEBSERVER_PATH + CODE_HISTORY_FILE
  NEW_AIRCRAFT_FILE = WEBSERVER_PATH + NEW_AIRCRAFT_FILE


  HISTOGRAM_IMAGE_HTML = WEBSERVER_PATH + HISTOGRAM_IMAGE_HTML
  HOURLY_IMAGE_FILE = (
      WEBSERVER_PATH + WEBSERVER_IMAGE_RELATIVE_FOLDER + HOURLY_IMAGE_FILE)
  VERSION_REPOSITORY = WEBSERVER_PATH + VERSION_REPOSITORY

TIMEZONE = 'US/Pacific' # timezone of display
TZ = pytz.timezone(TIMEZONE)

# iata codes that we don't need to expand
KNOWN_AIRPORTS = ('SJC', 'SFO', 'OAK')

SPLITFLAP_CHARS_PER_LINE = 22
SPLITFLAP_LINE_COUNT = 6

DIRECTIONS_4 = ['N', 'E', 'S', 'W']
DIRECTIONS_8 = ['N', 'NE', 'E', 'SE', 'S', 'SW', 'W', 'NW']
DIRECTIONS_16 = ['N', 'NNE', 'NE', 'ENE', 'E', 'ESE', 'SE', 'SSE',
                 'S', 'SSW', 'SW', 'WSW', 'W', 'WNW', 'NW', 'NNW']





                            <----SKIPPED LINES---->




      newly_nearby_flight_identifiers.append(flight_identifier)
    persistent_nearby_aircraft[flight_identifier] = now

  flights_to_delete = []
  for flight_identifier in persistent_nearby_aircraft:
    if (flight_identifier not in current_nearby_aircraft
        and (now - persistent_nearby_aircraft[flight_identifier]) >
        PERSISTENCE_SECONDS):
      flights_to_delete.append(flight_identifier)
  for flight_identifier in flights_to_delete:
    del persistent_nearby_aircraft[flight_identifier]
  return newly_nearby_flight_identifiers


def ScanForNewFlights(
    persistent_nearby_aircraft, persistent_path, log_jsons, flights):
  """Determines if there are any new aircraft in the radio message.

  The radio is continuously dumping new json messages to the Raspberry pi with
  all the flights currently observed. This function picks up the latest radio
  json, and for any new nearby flights - there should generally be at most one
  new flight on each pass through - gets additional flight data from
  FlightAware and augments the flight definition with the relevant fields to
  keep.

  Args:
    persistent_nearby_aircraft: dictionary where keys are flight numbers, and
      the values are the time the flight was last seen.
    persistent_path: dictionary where keys are flight numbers, and the values
      are a sequential list of the location-attributes in the json file; allows
      for tracking the flight path over time.
    log_jsons: boolean indicating whether we should pickle the JSONs.
    flights: list of flight dictionaries; if no json is returned, used to
      find a recent flight with same flight number to augment this flight with
      origin / destination / airline.

  Returns:
    A tuple:
    - updated persistent_nearby_aircraft
    - (possibly empty) dictionary of flight attributes of the new flight upon
      its first observation.
    - the time of the radio observation if present; None if no radio dump
    - a dictionary of attributes about the dump itself (i.e.: # of flights;
      furthest observed flight, etc.)
    - persistent_path, a data structure containing past details of a flight's
      location as described in ParseDumpJson
    - a text message indicating any errors in querying FlightAware or
      populating flight details
    - timestamp indicating exact time at which FlightAware was queried (or
      attempted to be queried)
  """
  flight_details = {}
  error_message = ''
  now = time.time()
  if SIMULATION:
    (dump_json, json_time) = DUMP_JSONS[SIMULATION_COUNTER]
  else:
    dump_json = ReadFile(DUMP_JSON_FILE, log_exception=True)

  json_desc_dict = {}
  current_nearby_aircraft = {}
  if dump_json:
    (current_nearby_aircraft, now,
     json_desc_dict, persistent_path) = ParseDumpJson(
         dump_json, persistent_path)

    if not SIMULATION and log_jsons:
      PickleObjectToFile((dump_json, now), PICKLE_DUMP_JSON_FILE, True)

    newly_nearby_flight_identifiers = UpdateAircraftList(




                            <----SKIPPED LINES---->




    if newly_nearby_flight_identifiers:

      if len(newly_nearby_flight_identifiers) > 1:
        # newly_nearby_flight_identifiers is a list of 2-tuples, where each
        # tuple is (flight_number, squawk)
        newly_nearby_flight_identifiers_str = ', '.join(
            ['%s/%s ' % (*i,) for i in newly_nearby_flight_identifiers])
        newly_nearby_flight_details_str = '\n'.join([
            str(current_nearby_aircraft[f])
            for f in newly_nearby_flight_identifiers])
        Log('Multiple newly-nearby flights: %s\n%s' % (
            newly_nearby_flight_identifiers_str,
            newly_nearby_flight_details_str))
      flight_identifier = newly_nearby_flight_identifiers[0]

      flight_aware_json = {}
      if SIMULATION:
        json_times = [j[1] for j in FA_JSONS]
        if json_time in json_times:
          flight_aware_json = FA_JSONS[json_times.index(json_time)][0]
        flight_aware_timestamp = time.time()  # "real" timestamp unavailable
      elif flight_identifier[0]:
        flight_number = flight_identifier[0]
        flight_aware_json, error_message, flight_aware_timestamp = (
            GetFlightAwareJson(flight_number))
        if flight_aware_json:
          UpdateStatusLight(GPIO_ERROR_FLIGHT_AWARE_CONNECTION, False)
        else:
          failure_message = 'No json from Flightaware for flight %s: %s' % (
              flight_number, error_message[:500])
          Log(failure_message)
          UpdateStatusLight(
              GPIO_ERROR_FLIGHT_AWARE_CONNECTION, True, failure_message)

      flight_details = {}
      if flight_aware_json:
        flight_details = ParseFlightAwareJson(flight_aware_json)
      elif flight_identifier[0]:  # if there's a flight number but no json
        flight_details, derived_attr_msg = FindAttributesFromSimilarFlights(
            flight_identifier[0], flights)
        error_message += derived_attr_msg

      if not SIMULATION and log_jsons:
        PickleObjectToFile((flight_aware_json, now), PICKLE_FA_JSON_FILE, True)

      # Augment FlightAware details with radio / radio-derived details
      flight_details.update(current_nearby_aircraft[flight_identifier])

      # Augment with the past location data; the [1] is because recall that
      # persistent_path[key] is actually a 2-tuple, the first element being
      # the most recent time seen, and the second element being the actual
      # path. But we do not need to keep around the most recent time seen any
      # more.
      flight_details['persistent_path'] = persistent_path[flight_identifier][1]

  return (
      persistent_nearby_aircraft,
      flight_details,
      now,
      json_desc_dict,
      persistent_path,
      error_message,
      flight_aware_timestamp)


def DescribeDumpJson(parsed):
  """Generates dict with descriptive attributes about the dump json file.

  Args:
    parsed: The parsed json file.

  Returns:
    Dictionary with attributes about radio range, number of flights seen, etc.
  """
  json_desc_dict = {}
  json_desc_dict['now'] = parsed['now']

  aircraft = [a for a in parsed['aircraft'] if a['seen'] < PERSISTENCE_SECONDS]
  json_desc_dict['radio_range_flights'] = len(aircraft)

  aircraft_with_pos = [a for a in aircraft if 'lat' in a and 'lon' in a]
  current_distances = [HaversineDistanceMeters(
      HOME, (a['lat'], a['lon'])) for a in aircraft_with_pos]




                            <----SKIPPED LINES---->




        # just simply need to continue updating this list to keep the
        # dictionary up to date (i.e.: we don't need to directly touch the
        # flights dictionary in main).
        (last_seen, current_path) = persistent_path.get(id_to_use, (None, []))
        if (  # flight position has been updated with this radio signal
            not current_path or
            simplified_aircraft.get('lat') != current_path[-1].get('lat') or
            simplified_aircraft.get('lon') != current_path[-1].get('lon')):
          current_path.append(simplified_aircraft)
        persistent_path[id_to_use] = (now, current_path)

  # if the flight was last seen too far in the past, remove the track info
  for f in list(persistent_path.keys()):
    (last_seen, current_path) = persistent_path[f]
    if last_seen < now - PERSISTENCE_SECONDS:
      persistent_path.pop(f)

  return (nearby_aircraft, now, json_desc_dict, persistent_path)





















last_query_time = 0
def GetFlightAwareJson(flight_number):
  """Scrapes the text json message from FlightAware for a given flight number.

  Given a flight number, loads the corresponding FlightAware webpage for that
  flight and extracts the relevant script that contains all the flight details
  from that page.  But only queries at most once per fixed period of time
  so as to avoid being blocked.

  Args:
    flight_number: text flight number (i.e.: SWA1234)

  Returns:
    Three tuple:
     - Text representation of the json message from FlightAware.
     - Text string of error message, if any
     - Timestamp of attempted query on FlightAware
  """
  min_query_delay_seconds = 90
  url = 'https://flightaware.com/live/flight/' + flight_number

  global last_query_time
  seconds_since_last_query = time.time() - last_query_time
  if last_query_time and seconds_since_last_query < min_query_delay_seconds:
    error_msg = (
        'Unable to query FA for %s at %s since last query to FA was only'
        ' %d seconds ago; min of %d seconds needed: %s' % (
            flight_number,
            EpochDisplayTime(time.time(), format_string='%H:%M:%S'),
            seconds_since_last_query, min_query_delay_seconds, url))

    return '', error_msg, time.time()

  last_query_time = time.time()

  try:
    response = requests.get(url, timeout=5)
    query_time = time.time()
  except requests.exceptions.RequestException as e:
    error_msg = 'Unable to query FA for URL due to %s: %s' % (e, url)

    return '', error_msg, query_time


  soup = bs4.BeautifulSoup(response.text, 'html.parser')
  l = soup.find_all('script')
  flight_script = None
  for script in l:
    if 'trackpollBootstrap' in str(script):
      flight_script = str(script)
      break
  if not flight_script:
    error_msg = (
        'Unable to find trackpollBootstrap script in page: ' + response.text)
    Log(error_msg)
    return '', error_msg, query_time
  first_open_curly_brace = flight_script.find('{')
  last_close_curly_brace = flight_script.rfind('}')
  flight_json = flight_script[first_open_curly_brace:last_close_curly_brace+1]
  return flight_json, '', query_time


def Unidecode(s):
  """Convert a special unicode characters to closest ASCII representation."""
  if s is not None:
    s = unidecode.unidecode(s)
  return s


def FindAttributesFromSimilarFlights(this_flight_number, flights):
  """Returns a dictionary with info about a flight based on other flights.

  We may not get a json from the internet about this flight for any number
  of reasons: internet down; website down; too frequent queries; etc.  However,
  there are still some basic attributes we can derive about this flight
  from past observations of the same flight number, or past observations
  about the flight number prefix.  Specifically, we can get the flight's
  airline, origin, and destination.

  Args:




                            <----SKIPPED LINES---->




        if n/25 == int(n/25):
          print(' - %d' % n)
        CreateFlightInsights(
            flights[:n+1], configuration.get('insights', 'hide'), {})
        PickleObjectToFile(flight, tmp_f, False)

      if mtime == os.path.getmtime(f):
        shutil.move(tmp_f, f)
      else:
        print('Aborted: failed to bootstrap %s: file changed while in process'
              % full_path)
        return


def ResetLogs(config):
  """Clears the non-scrolling logs if reset_logs in config."""
  if 'reset_logs' in config:
    removed_files = []
    for f in (
        STDERR_FILE, BACKUP_FILE, SERVICE_VERIFICATION_FILE,
        NEW_AIRCRAFT_FILE):
      if RemoveFile(f):
        removed_files.append(f)
        open(f, 'a').close()
    Log('Reset logs: cleared files %s' % '; '.join(removed_files))
    config.pop('reset_logs')
    config = BuildSettings(config)
    WriteFile(CONFIG_FILE, config)
  return config


def CheckTemperature(last_logged=0):
  """Turn on fan if temperature exceeds threshold.

  Args:
    last_logged: epoch at which temperature was last logged.

  Returns:
    Epoch at which temperature was last logged.
  """
  if RASPBERRY_PI:




                            <----SKIPPED LINES---->





    ResetLogs(configuration)  # clear the logs if requested
    UpdateRollingLogSize(configuration)

    # if this is a SIMULATION, then process every diff dump. But if it
    # isn't a simulation, then only read & do related processing for the
    # next dump if the last-modified timestamp indicates the file has been
    # updated since it was last read.
    tmp_timestamp = 0
    if not SIMULATION:
      dump_json_exists = os.path.exists(DUMP_JSON_FILE)
      if dump_json_exists:
        tmp_timestamp = os.path.getmtime(DUMP_JSON_FILE)
    if (SIMULATION and DumpJsonChanges()) or (
        not SIMULATION and dump_json_exists and
        tmp_timestamp > last_dump_json_timestamp):

      last_dump_json_timestamp = tmp_timestamp

      (persistent_nearby_aircraft, flight, now, json_desc_dict,
       persistent_path, flight_aware_error_message, flight_aware_timestamp) = (
           ScanForNewFlights(
               persistent_nearby_aircraft,
               persistent_path,
               configuration.get('log_jsons', False),
               flights))

      # Logging: As part of the memory instrumentation, let's track
      # the length of these data structures
      if not iteration % 1000:
        lengths = [len(flights), len(screen_history)]
        Log('Iteration: %d: object lengths: %s' % (iteration, lengths))

      # because this might just be an updated instance of the previous
      # flight as more identifier information (squawk and or flight number)
      # comes in, we only want to process this if its a truly new flight
      new_flight_flag = ConfirmNewFlight(flight, flights)

      if new_flight_flag:
        time_new_flight_found = time.time()

        # Since we no longer need the memory-hogging prior persistent_path
        # stored on the flights, we can remove it
        if flights and 'persistent_path' in flights[-1]:
          del flights[-1]['persistent_path']





                            <----SKIPPED LINES---->





        PickleObjectToFile(
            flight, PICKLE_FLIGHTS, True, timestamp=flight['now'])

        # We record flight number, flight_meets_display_criteria,
        # reason_flight_fails_criteria, flight_aware_error_message, and
        # time stamps of when we confirmed new flight; when we generated
        # message; when we generated insight messages (if any), to a
        # to a pickle file so that we may construct a flight-centric
        # status report
        #
        # Specifically, we record a 3-tuple:
        # - flight number
        # - time stamp of the recording
        # - a dictionary of elements
        data = (
            DisplayFlightNumber(flight),
            time.time(),
            {
                'reason_flight_fails_criteria': reason_flight_fails_criteria,
                'flight_aware_timestamp': flight_aware_timestamp,
                'flight_aware_error_message': flight_aware_error_message,
                'time_new_flight_found': time_new_flight_found,
                'time_flight_message_inserted': time_flight_message_inserted,
                'time_insight_message_inserted': time_insight_message_inserted})
        PickleObjectToFile(data, PICKLE_FLIGHT_DASHBOARD, True)

      else:
        remote, servo = RefreshArduinos(
            remote, servo,
            to_remote_q, to_servo_q, to_main_q, shutdown,
            flights, json_desc_dict, configuration, screen_history)

    message_queue, next_message_time = ProcessArduinoCommmands(
        to_main_q, flights, configuration, message_queue, next_message_time)

    personal_message = PersonalMessage(
        configuration, message_queue, personal_message)

    # MEMORY MANAGEMENT
    # it turns out we only need the last screen in the screen history, not




                            <----SKIPPED LINES---->