<----SKIPPED LINES---->




import statistics
import sys
import textwrap
import time
import tracemalloc
import types

import bs4
import dateutils
import numpy
import matplotlib
import matplotlib.pyplot
import psutil
import pycurl
import pytz
import requests
import tzlocal
import unidecode

# pylint: disable=line-too-long
from constants import RASPBERRY_PI, MESSAGEBOARD_PATH, WEBSERVER_PATH, KEY, SECRET, SUBSCRIPTION_ID, LOCAL_KEY, LOCAL_VESTABOARD_ADDRESS
# pylint: enable=line-too-long

import arduino

if RASPBERRY_PI:
  import gpiozero  # pylint: disable=E0401
  import RPi.GPIO  # pylint: disable=E0401


VERBOSE = False  # additional messages logged

SHUTDOWN_SIGNAL = ''
REBOOT_SIGNAL = False

# to be tracked in the dashboard messages so that we know when a
# restart due to exit (vs. a long delay in some processing) happened
INSTANCE_START_TIME = time.time()

SIMULATION = False
SIMULATION_COUNTER = 0




                            <----SKIPPED LINES---->




# always leveraged by the running software.
CONFIG_FILE = 'secure/settings.txt'
CONFIG_BOOLEANS = (
    'setting_screen_enabled', 'next_flight', 'reset_logs', 'log_jsons')
# A few key settings for the messageboard are its sensitivity to displaying
# flights - though it logs all flights within range, it may not be desirable
# to display all flights to the user. Two key parameters are the maximum
# altitude, and the furthest away we anticipate the flight being at its
# closest point to HOME. As those two parameters are manipulated in the
# settings, a histogram is displayed with one or potentially two series,
# showing the present and potentially prior-set distribution of flights,
# by hour throughout the day, over the last seven days, normalized to
# flights per day. This allows those parameters to be fine-tuned in a
# useful way. This file is the location, on the webserver, of that image,
# which needs to be in alignment with the html page that displays it.
HOURLY_IMAGE_FILE = 'hours.png'

# This is all messages that have been sent to the board since the last time
# the file was manually cleared. Newest messages are at the bottom. It is
# visible at the webserver.
#enumeration of all messages sent to board
ALL_MESSAGE_FILE = 'secure/all_messages.txt'
# This shows the most recent n messages sent to the board. Newest messages
# are at the top for easier viewing of "what did I miss".
ROLLING_MESSAGE_FILE = 'rolling_messages.txt'

STDERR_FILE = 'secure/stderr.txt'
BACKUP_FILE = 'secure/backup.txt'
SERVICE_VERIFICATION_FILE = 'secure/service-verification.txt'
UPTIMES_FILE = 'uptimes.php'
CODE_HISTORY_FILE = 'code_history.php'


# This keeps a log of all aircraft not yet cataloged in the AIRCRAFT_LENGTH dict
NEW_AIRCRAFT_FILE = 'secure/new_aircraft.txt'

FLAG_MSG_FLIGHT = 1  # basic flight details
FLAG_MSG_INSIGHT = 2  # random tidbit about a flight
FLAG_MSG_HISTOGRAM = 3  # histogram message
FLAG_MSG_CLEAR = 4  # a blank message to clear the screen
# user-entered message to display for some duration of time
FLAG_MSG_PERSONAL = 5

FLAG_INSIGHT_LAST_SEEN = 0
FLAG_INSIGHT_DIFF_AIRCRAFT = 1
FLAG_INSIGHT_NTH_FLIGHT = 2
FLAG_INSIGHT_GROUNDSPEED = 3
FLAG_INSIGHT_ALTITUDE = 4
FLAG_INSIGHT_VERTRATE = 5
FLAG_INSIGHT_FIRST_DEST = 6
FLAG_INSIGHT_FIRST_ORIGIN = 7
FLAG_INSIGHT_FIRST_AIRLINE = 8




                            <----SKIPPED LINES---->




    '  function gtag(){dataLayer.push(arguments);}\n'
    "  gtag('js', new Date());\n"
    "  gtag('config', 'UA-99931533-2');\n"
    '</script>\n')

#if running on raspberry, then need to prepend path to file names
if RASPBERRY_PI:
  MEMORY_DIRECTORY = MESSAGEBOARD_PATH + MEMORY_DIRECTORY
  PICKLE_FLIGHTS = MESSAGEBOARD_PATH + PICKLE_FLIGHTS
  PICKLE_SYSTEM_DASHBOARD = MESSAGEBOARD_PATH + PICKLE_SYSTEM_DASHBOARD
  PICKLE_FLIGHT_DASHBOARD = MESSAGEBOARD_PATH + PICKLE_FLIGHT_DASHBOARD
  LOGFILE = MESSAGEBOARD_PATH + LOGFILE
  PICKLE_DUMP_JSON_FILE = MESSAGEBOARD_PATH + PICKLE_DUMP_JSON_FILE
  PICKLE_FA_JSON_FILE = MESSAGEBOARD_PATH + PICKLE_FA_JSON_FILE
  PICKLE_SCREENS = MESSAGEBOARD_PATH + PICKLE_SCREENS
  CODE_REPOSITORY = MESSAGEBOARD_PATH

  HISTOGRAM_CONFIG_FILE = WEBSERVER_PATH + HISTOGRAM_CONFIG_FILE
  CONFIG_FILE = WEBSERVER_PATH + CONFIG_FILE
  ROLLING_MESSAGE_FILE = WEBSERVER_PATH + ROLLING_MESSAGE_FILE
  ALL_MESSAGE_FILE = WEBSERVER_PATH + ALL_MESSAGE_FILE
  ROLLING_LOGFILE = WEBSERVER_PATH + ROLLING_LOGFILE
  STDERR_FILE = WEBSERVER_PATH + STDERR_FILE
  BACKUP_FILE = WEBSERVER_PATH + BACKUP_FILE
  SERVICE_VERIFICATION_FILE = WEBSERVER_PATH + SERVICE_VERIFICATION_FILE
  UPTIMES_FILE = WEBSERVER_PATH + UPTIMES_FILE

  CODE_HISTORY_FILE = WEBSERVER_PATH + CODE_HISTORY_FILE
  NEW_AIRCRAFT_FILE = WEBSERVER_PATH + NEW_AIRCRAFT_FILE


  HISTOGRAM_IMAGE_HTML = WEBSERVER_PATH + HISTOGRAM_IMAGE_HTML
  HOURLY_IMAGE_FILE = (
      WEBSERVER_PATH + WEBSERVER_IMAGE_RELATIVE_FOLDER + HOURLY_IMAGE_FILE)
  VERSION_REPOSITORY = WEBSERVER_PATH + VERSION_REPOSITORY

TIMEZONE = 'US/Pacific' # timezone of display
TZ = pytz.timezone(TIMEZONE)

# iata codes that we don't need to expand
KNOWN_AIRPORTS = ('SJC', 'SFO', 'OAK')

SPLITFLAP_CHARS_PER_LINE = 22
SPLITFLAP_LINE_COUNT = 6

DIRECTIONS_4 = ['N', 'E', 'S', 'W']
DIRECTIONS_8 = ['N', 'NE', 'E', 'SE', 'S', 'SW', 'W', 'NW']
DIRECTIONS_16 = ['N', 'NNE', 'NE', 'ENE', 'E', 'ESE', 'SE', 'SSE',
                 'S', 'SSW', 'SW', 'WSW', 'W', 'WNW', 'NW', 'NNW']





                            <----SKIPPED LINES---->




    persistent_path: dictionary where keys are flight numbers, and the values
      are a sequential list of the location-attributes in the json file; allows
      for tracking the flight path over time.
    log_jsons: boolean indicating whether we should pickle the JSONs.
    flights: list of flight dictionaries; if no json is returned, used to
      find a recent flight with same flight number to augment this flight with
      origin / destination / airline.

  Returns:
    A tuple:
    - updated persistent_nearby_aircraft
    - (possibly empty) dictionary of flight attributes of the new flight upon
      its first observation.
    - the time of the radio observation if present; None if no radio dump
    - a dictionary of attributes about the dump itself (i.e.: # of flights;
      furthest observed flight, etc.)
    - persistent_path, a data structure containing past details of a flight's
      location as described in ParseDumpJson
    - a text message indicating any errors in querying FlightAware or
      populating flight details



    - timestamp indicating exact time at which FlightAware was queried (or
      attempted to be queried), if a query was made in this pass
  """
  flight_details = {}
  error_message = ''
  now = time.time()
  if SIMULATION:
    (dump_json, json_time) = DUMP_JSONS[SIMULATION_COUNTER]
  else:
    dump_json = ReadFile(DUMP_JSON_FILE, log_exception=True)
  # Often there is no flight aware query, so we need to give a default value
  flight_aware_timestamp = 0

  json_desc_dict = {}
  current_nearby_aircraft = {}
  if dump_json:
    (current_nearby_aircraft, now,
     json_desc_dict, persistent_path) = ParseDumpJson(
         dump_json, persistent_path)





                            <----SKIPPED LINES---->




        # newly_nearby_flight_identifiers is a list of 2-tuples, where each
        # tuple is (flight_number, squawk)
        newly_nearby_flight_identifiers_str = ', '.join(
            ['%s/%s ' % (*i,) for i in newly_nearby_flight_identifiers])
        newly_nearby_flight_details_str = '\n'.join([
            str(current_nearby_aircraft[f])
            for f in newly_nearby_flight_identifiers])
        Log('Multiple newly-nearby flights: %s\n%s' % (
            newly_nearby_flight_identifiers_str,
            newly_nearby_flight_details_str))
      flight_identifier = newly_nearby_flight_identifiers[0]

      flight_aware_json = {}
      if SIMULATION:
        json_times = [j[1] for j in FA_JSONS]
        if json_time in json_times:
          flight_aware_json = FA_JSONS[json_times.index(json_time)][0]
        flight_aware_timestamp = time.time()  # "real" timestamp unavailable
      elif flight_identifier[0]:
        flight_number = flight_identifier[0]
        flight_aware_json, error_message, flight_aware_timestamp = (
            GetFlightAwareJson(flight_number))
        if flight_aware_json:
          UpdateStatusLight(GPIO_ERROR_FLIGHT_AWARE_CONNECTION, False)
        else:
          failure_message = 'No json from Flightaware for flight %s: %s' % (
              flight_number, error_message[:500])
          Log(failure_message)
          UpdateStatusLight(
              GPIO_ERROR_FLIGHT_AWARE_CONNECTION, True, failure_message)

      flight_details = {}
      if flight_aware_json:
        flight_details = ParseFlightAwareJson(flight_aware_json)
      elif flight_identifier[0]:  # if there's a flight number but no json
        flight_details, derived_attr_msg = FindAttributesFromSimilarFlights(
            flight_identifier[0], flights)
        error_message = '%s; %s' % (error_message, derived_attr_msg)

      if not SIMULATION and log_jsons:
        PickleObjectToFile((flight_aware_json, now), PICKLE_FA_JSON_FILE, True)

      # Augment FlightAware details with radio / radio-derived details
      flight_details.update(current_nearby_aircraft[flight_identifier])

      # Augment with the past location data; the [1] is because recall that
      # persistent_path[key] is actually a 2-tuple, the first element being
      # the most recent time seen, and the second element being the actual
      # path. But we do not need to keep around the most recent time seen any
      # more.
      flight_details['persistent_path'] = persistent_path[flight_identifier][1]

  return (
      persistent_nearby_aircraft,
      flight_details,
      now,
      json_desc_dict,
      persistent_path,
      error_message,

      flight_aware_timestamp)


def DescribeDumpJson(parsed):
  """Generates dict with descriptive attributes about the dump json file.

  Args:
    parsed: The parsed json file.

  Returns:
    Dictionary with attributes about radio range, number of flights seen, etc.
  """
  json_desc_dict = {}
  json_desc_dict['now'] = parsed['now']

  aircraft = [a for a in parsed['aircraft'] if a['seen'] < PERSISTENCE_SECONDS]
  json_desc_dict['radio_range_flights'] = len(aircraft)

  aircraft_with_pos = [a for a in aircraft if 'lat' in a and 'lon' in a]
  current_distances = [HaversineDistanceMeters(




                            <----SKIPPED LINES---->




    (last_seen, current_path) = persistent_path[f]
    if last_seen < now - PERSISTENCE_SECONDS:
      persistent_path.pop(f)

  return (nearby_aircraft, now, json_desc_dict, persistent_path)


last_query_time = 0
def GetFlightAwareJson(flight_number):
  """Scrapes the text json message from FlightAware for a given flight number.

  Given a flight number, loads the corresponding FlightAware webpage for that
  flight and extracts the relevant script that contains all the flight details
  from that page.  But only queries at most once per fixed period of time
  so as to avoid being blocked.

  Args:
    flight_number: text flight number (i.e.: SWA1234)

  Returns:
    Three tuple:
     - Text representation of the json message from FlightAware.
     - Text string of error message, if any



     - Timestamp of attempted query on FlightAware
  """
  min_query_delay_seconds = 90
  url = 'https://flightaware.com/live/flight/' + flight_number

  global last_query_time
  seconds_since_last_query = time.time() - last_query_time
  if last_query_time and seconds_since_last_query < min_query_delay_seconds:
    error_msg = (
        'Unable to query FA for %s at %s since last query to FA was only'
        ' %d seconds ago; min of %d seconds needed: %s' % (
            flight_number,
            EpochDisplayTime(time.time(), format_string='%H:%M:%S'),
            seconds_since_last_query, min_query_delay_seconds, url))
    return '', error_msg, time.time()


  last_query_time = time.time()

  try:
    response = requests.get(url, timeout=5)
    query_time = time.time()
  except requests.exceptions.RequestException as e:
    query_time = time.time()  # did not get to the query_time assignment above
    error_msg = 'Unable to query FA for URL due to %s: %s' % (e, url)
    return '', error_msg, query_time


  soup = bs4.BeautifulSoup(response.text, 'html.parser')
  l = soup.find_all('script')
  flight_script = None
  for script in l:
    if 'trackpollBootstrap' in str(script):
      flight_script = str(script)
      break
  if not flight_script:
    error_msg = (
        'Unable to find trackpollBootstrap script in page: ' + response.text)
    Log(error_msg)
    return '', error_msg, query_time

  first_open_curly_brace = flight_script.find('{')
  last_close_curly_brace = flight_script.rfind('}')
  flight_json = flight_script[first_open_curly_brace:last_close_curly_brace+1]
  return flight_json, '', query_time



def Unidecode(s):
  """Convert a special unicode characters to closest ASCII representation."""
  if s is not None:
    s = unidecode.unidecode(s)
  return s


def FindAttributesFromSimilarFlights(this_flight_number, flights):
  """Returns a dictionary with info about a flight based on other flights.

  We may not get a json from the internet about this flight for any number
  of reasons: internet down; website down; too frequent queries; etc.  However,
  there are still some basic attributes we can derive about this flight
  from past observations of the same flight number, or past observations
  about the flight number prefix.  Specifically, we can get the flight's
  airline, origin, and destination.

  Args:




                            <----SKIPPED LINES---->




  f.close()


def SimulationSetup():
  """Updates global variable file names and loads JSON data for simulations."""
  # Clear file so that shell tail -f process can continue to point to same file
  def ClearFile(filename):
    if os.path.exists(filename):
      with open(filename, 'w') as f:
        f.write('')

  global SIMULATION
  SIMULATION = True

  global DUMP_JSONS
  DUMP_JSONS = UnpickleObjectFromFile(PICKLE_DUMP_JSON_FILE, True)

  global FA_JSONS
  FA_JSONS = UnpickleObjectFromFile(PICKLE_FA_JSON_FILE, True)

  global ALL_MESSAGE_FILE
  ALL_MESSAGE_FILE = PrependFileName(ALL_MESSAGE_FILE, SIMULATION_PREFIX)
  ClearFile(ALL_MESSAGE_FILE)

  global LOGFILE
  LOGFILE = PrependFileName(LOGFILE, SIMULATION_PREFIX)
  ClearFile(LOGFILE)

  global ROLLING_LOGFILE
  ROLLING_LOGFILE = PrependFileName(ROLLING_LOGFILE, SIMULATION_PREFIX)
  ClearFile(ROLLING_LOGFILE)

  global ROLLING_MESSAGE_FILE
  ROLLING_MESSAGE_FILE = PrependFileName(
      ROLLING_MESSAGE_FILE, SIMULATION_PREFIX)
  ClearFile(ROLLING_MESSAGE_FILE)

  global PICKLE_FLIGHTS
  PICKLE_FLIGHTS = PrependFileName(PICKLE_FLIGHTS, SIMULATION_PREFIX)
  filenames = UnpickleObjectFromFile(
      PICKLE_FLIGHTS, True, max_days=None, filenames=True)
  for file in filenames:
    ClearFile(file)





                            <----SKIPPED LINES---->





  The message is pushed to the vestaboard splitflap display by way of its
  web services; see https://docs.vestaboard.com/introduction for more details;
  if the web service fails, it then reattempts to publish using the local api.

  TODO: rewrite to use the easier-to-follow requests library, more in line
  with PublishMessageLocal.

  Args:
    s: String to publish.
    subscription_id: string subscription id from Vestaboard.
    key: string key from Vestaboard.
    secret: string secret from Vestaboard.
    timeout: Max duration in seconds that we should wait to establish a
      connection.

  Returns:
    Two-tuple:
    - Text string indicating how the message was displayed (web or local
      api), including error messages encountered if any.
    - Status code which is one of the text strings SUCCESS, WARNING, or FAIL,
      indicating whether the web service was used (SUCCESS), the local service
      was used because the web service failed (WARNING), or both failed (FAIL).

  """
  error_code = False
  curl = pycurl.Curl()

  # See https://stackoverflow.com/questions/31826814/
  # curl-post-request-into-pycurl-code
  # Set URL value
  curl.setopt(
      pycurl.URL,
      'https://platform.vestaboard.com/subscriptions/%s/message'
      % subscription_id)
  curl.setopt(pycurl.HTTPHEADER, [
      'X-Vestaboard-Api-Key:%s' % key, 'X-Vestaboard-Api-Secret:%s' % secret])
  curl.setopt(pycurl.TIMEOUT_MS, timeout*1000)
  curl.setopt(pycurl.POST, 1)

  curl.setopt(pycurl.WRITEFUNCTION, lambda x: None) # to keep stdout clean

  # preparing body the way pycurl.READDATA wants it
  body_as_dict = {'characters': StringToCharArray(s)}




                            <----SKIPPED LINES---->




  body_as_file_object = io.StringIO(body_as_json_string)

  # prepare and send. See also: pycurl.READFUNCTION to pass function instead
  curl.setopt(pycurl.READDATA, body_as_file_object)
  curl.setopt(pycurl.POSTFIELDSIZE, len(body_as_json_string))
  web_publish_failure_msg = ''
  try:
    curl.perform()
    status_msg = 'Web service published'
    publish_status_code = 'SUCCESS'
  except pycurl.error as e:
    timing_message = CurlTimingDetailsToString(curl)
    web_publish_failure_msg = (
        'curl.perform() failed with message %s; timing details: %s' %
        (e, timing_message))
    # Using the remote webservice failed, but maybe the local API will
    # be more successful?  If this succeeds, then we should not indicate
    # a failure on the status light / dashboard, but we should still log
    # the remote failure
    Log(web_publish_failure_msg)
    local_publish_error_msg = PublishMessageLocal(
        s, timeout=timeout, update_dashboard=False)
    if not local_publish_error_msg:
      status_msg = (
          'Local service published (1) because web service failed with %s'
          % web_publish_failure_msg)
      publish_status_code = 'WARNING'
    else:
      status_msg = (
          'Local service failed (2) with %s after web service failed with %s' %
          (local_publish_error_msg, web_publish_failure_msg))
      publish_status_code = 'FAILURE'
  else:
    # you may want to check HTTP response code, e.g.
    timing_message = CurlTimingDetailsToString(curl)
    status_code = curl.getinfo(pycurl.RESPONSE_CODE)
    if status_code != 200:
      web_publish_failure_msg = (
          'Server returned HTTP status code %d for message %s; '
          'timing details: %s' % (status_code, s, timing_message))
      Log(web_publish_failure_msg)
      local_publish_error_msg = PublishMessageLocal(
          s, timeout=timeout, update_dashboard=False)
      if not local_publish_error_msg:
        status_msg = (
            'Local service published (3) because web service failed with %s'
            % web_publish_failure_msg)
        publish_status_code = 'WARNING'
      else:
        status_msg = (
            'Local service failed (4) with %s after web service failed with %s'
            % (local_publish_error_msg, web_publish_failure_msg))

        publish_status_code = 'FAILURE'

  # We've logged the error code from the external web service, but the
  # Vestaboard local API was able to recover from the error, so we need not
  # log the failure message / error code in the dashboard.
  if not error_code:
    web_publish_failure_msg = ''

  curl.close()
  UpdateStatusLight(
      GPIO_ERROR_VESTABOARD_CONNECTION,
      publish_status_code=='FAILURE', web_publish_failure_msg)

  return status_msg, publish_status_code


def PublishMessageLocal(
    s,
    local_key=LOCAL_KEY,
    local_address=LOCAL_VESTABOARD_ADDRESS,
    timeout=5,
    update_dashboard=True):
  """Publishes a text string to a Vestaboard via local API.

  The message is pushed to the vestaboard splitflap display by way of its
  local API; see https://docs.vestaboard.com/local for more details.

  Args:
    s: String to publish.
    local_key: string key from Vestaboard for local API access.
    local_address: the address and port to the local Vestaboard service.
    timeout: Max duration in seconds that we should wait to establish a
      connection.
    update_dashboard: Boolean indicating whether this method should update the
      system dashboard, or if that should be left to the calling function.

  Returns:

    False if successful; error message string if failure occurs.

  """
  error_code = False


  data = str(StringToCharArray(s))
  headers = {
      'X-Vestaboard-Local-Api-Key': local_key,
      'Content-Type': 'application/x-www-form-urlencoded',
  }

  response = 'Unable to complete requests.post'
  try:
    response = requests.post(
        local_address,
        headers=headers, data=data, timeout=timeout)
  except requests.exceptions.RequestException as e:
    error_msg = 'Unable to reach %s (%s): %s' % (local_address, response, e)
    Log(error_msg)
    error_code = True
  if not error_code:
    Log('Message sent to Vestaboard by way of local api: %s' % s)


  if update_dashboard:
    UpdateStatusLight(
        GPIO_ERROR_VESTABOARD_CONNECTION, error_code, error_msg)
  return error_code


def CurlTimingDetailsToString(curl):
  """Extracts timing details of a curl request into a readable string."""
  timing = {}
  timing['total-time'] = curl.getinfo(pycurl.TOTAL_TIME)
  timing['namelookup-time'] = curl.getinfo(pycurl.NAMELOOKUP_TIME)
  timing['connect-time'] = curl.getinfo(pycurl.CONNECT_TIME)
  timing['pretransfer-time'] = curl.getinfo(pycurl.PRETRANSFER_TIME)
  timing['redirect-time'] = curl.getinfo(pycurl.REDIRECT_TIME)
  timing['starttransfer-time'] = curl.getinfo(pycurl.STARTTRANSFER_TIME)
  results = [label + '=' + '%.4f' % timing[label] for label in timing]
  results = '; '.join(results)
  return results


def TruncateEscapedLine(s):
  """Formats a single line of the personal message for the Vestaboard.

  The Vestaboard has line length limitations, a limited character set,




                            <----SKIPPED LINES---->




      # There may be one or several insight messages that were added to the
      # message queue along with the flight at a time when the screen was
      # enabled, but by the time it comes to display them, the screen is now
      # disabled.  These should not be displayed.  Note that this check only
      # needs to be done for insight messages because other message types
      # are user initiated and so presumably should be displayed irrespective
      # of when the user triggered it to be displayed.
      if message_type == FLAG_MSG_INSIGHT and not MessageMeetsDisplayCriteria(
          configuration):
        publish_status_msg = (
            'Message purged as no longer meets display criteria')
        publish_status_code = 'WARNING'
        Log('Message %s purged' % message_text)

      else:
        if isinstance(message_text, str):
          message_text = textwrap.wrap(
              message_text,
              width=SPLITFLAP_CHARS_PER_LINE)
        display_message = Screenify(message_text, False)
        Log(display_message, file=ALL_MESSAGE_FILE)

        # Saving this to disk allows us to identify
        # persistently whats currently on the screen
        PickleObjectToFile(message, PICKLE_SCREENS, True)
        screens.append(message)

        MaintainRollingWebLog(display_message, 25)
        if not SIMULATION:
          splitflap_message = Screenify(message_text, True)
          publish_status = PublishMessageWeb(splitflap_message)
          publish_status_msg, publish_status_code = publish_status

      if message_type in (FLAG_MSG_INSIGHT, FLAG_MSG_FLIGHT):
        flight = message[2]
        # We record flight number, time stamp, message, and status
        # to a pickle file so that we may construct a flight-centric
        # status report
        #
        # Specifically, we record a 3-tuple:
        # - flight number




                            <----SKIPPED LINES---->





    ResetLogs(configuration)  # clear the logs if requested
    UpdateRollingLogSize(configuration)

    # if this is a SIMULATION, then process every diff dump. But if it
    # isn't a simulation, then only read & do related processing for the
    # next dump if the last-modified timestamp indicates the file has been
    # updated since it was last read.
    tmp_timestamp = 0
    if not SIMULATION:
      dump_json_exists = os.path.exists(DUMP_JSON_FILE)
      if dump_json_exists:
        tmp_timestamp = os.path.getmtime(DUMP_JSON_FILE)
    if (SIMULATION and DumpJsonChanges()) or (
        not SIMULATION and dump_json_exists and
        tmp_timestamp > last_dump_json_timestamp):

      last_dump_json_timestamp = tmp_timestamp

      (persistent_nearby_aircraft, flight, now, json_desc_dict,
       persistent_path, flight_aware_error_message, flight_aware_timestamp) = (

           ScanForNewFlights(
               persistent_nearby_aircraft,
               persistent_path,
               configuration.get('log_jsons', False),
               flights))

      # Logging: As part of the memory instrumentation, let's track
      # the length of these data structures
      if not iteration % 1000:
        lengths = [len(flights), len(screen_history)]
        Log('Iteration: %d: object lengths: %s' % (iteration, lengths))

      # because this might just be an updated instance of the previous
      # flight as more identifier information (squawk and or flight number)
      # comes in, we only want to process this if its a truly new flight
      new_flight_flag = ConfirmNewFlight(flight, flights)

      if new_flight_flag:
        # now comes directly from the radio json
        time_new_flight_found = now




                            <----SKIPPED LINES---->




            flight, PICKLE_FLIGHTS, True, timestamp=flight['now'])

        # We record flight number, flight_meets_display_criteria,
        # reason_flight_fails_criteria, flight_aware_error_message, and
        # time stamps of when we confirmed new flight; when we generated
        # message; when we generated insight messages (if any), to a
        # to a pickle file so that we may construct a flight-centric
        # status report
        #
        # Specifically, we record a 3-tuple:
        # - flight number
        # - time stamp of the recording
        # - a dictionary of elements
        data = (
            DisplayFlightNumber(flight),
            time.time(),
            {
                'reason_flight_fails_criteria': reason_flight_fails_criteria,
                'flight_aware_timestamp': flight_aware_timestamp,
                'flight_aware_error_message': flight_aware_error_message,

                'time_new_flight_found': time_new_flight_found,
                'time_flight_message_inserted': time_flight_message_inserted,
                'time_insight_message_inserted': time_insight_message_inserted})
        PickleObjectToFile(data, PICKLE_FLIGHT_DASHBOARD, True)

      else:
        remote, servo = RefreshArduinos(
            remote, servo,
            to_remote_q, to_servo_q, to_main_q, shutdown,
            flights, json_desc_dict, configuration, screen_history)

    message_queue, next_message_time = ProcessArduinoCommmands(
        to_main_q, flights, configuration, message_queue, next_message_time)

    personal_message = PersonalMessage(
        configuration, message_queue, personal_message)

    # MEMORY MANAGEMENT
    # it turns out we only need the last screen in the screen history, not
    # the entire history, so we can purge all the rest from active memory




                            <----SKIPPED LINES---->

                            <----SKIPPED LINES---->




import statistics
import sys
import textwrap
import time
import tracemalloc
import types

import bs4
import dateutils
import numpy
import matplotlib
import matplotlib.pyplot
import psutil
import pycurl
import pytz
import requests
import tzlocal
import unidecode

# pylint: disable=line-too-long
from constants import RASPBERRY_PI, MESSAGEBOARD_PATH, WEBSERVER_PATH, SECURE_WEBSERVER_PATH, KEY, SECRET, SUBSCRIPTION_ID, LOCAL_KEY, LOCAL_VESTABOARD_ADDRESS
# pylint: enable=line-too-long

import arduino

if RASPBERRY_PI:
  import gpiozero  # pylint: disable=E0401
  import RPi.GPIO  # pylint: disable=E0401


VERBOSE = False  # additional messages logged

SHUTDOWN_SIGNAL = ''
REBOOT_SIGNAL = False

# to be tracked in the dashboard messages so that we know when a
# restart due to exit (vs. a long delay in some processing) happened
INSTANCE_START_TIME = time.time()

SIMULATION = False
SIMULATION_COUNTER = 0




                            <----SKIPPED LINES---->




# always leveraged by the running software.
CONFIG_FILE = 'secure/settings.txt'
CONFIG_BOOLEANS = (
    'setting_screen_enabled', 'next_flight', 'reset_logs', 'log_jsons')
# A few key settings for the messageboard are its sensitivity to displaying
# flights - though it logs all flights within range, it may not be desirable
# to display all flights to the user. Two key parameters are the maximum
# altitude, and the furthest away we anticipate the flight being at its
# closest point to HOME. As those two parameters are manipulated in the
# settings, a histogram is displayed with one or potentially two series,
# showing the present and potentially prior-set distribution of flights,
# by hour throughout the day, over the last seven days, normalized to
# flights per day. This allows those parameters to be fine-tuned in a
# useful way. This file is the location, on the webserver, of that image,
# which needs to be in alignment with the html page that displays it.
HOURLY_IMAGE_FILE = 'hours.png'

# This is all messages that have been sent to the board since the last time
# the file was manually cleared. Newest messages are at the bottom. It is
# visible at the webserver.


# This shows the most recent n messages sent to the board. Newest messages
# are at the top for easier viewing of "what did I miss".
ROLLING_MESSAGE_FILE = 'rolling_messages.txt'

STDERR_FILE = 'secure/stderr.txt'
BACKUP_FILE = 'secure/backup.txt'
SERVICE_VERIFICATION_FILE = 'secure/service-verification.txt'
UPTIMES_FILE = 'uptimes.php'
CODE_HISTORY_FILE = 'code_history.php'
FLIGHT_STATUS = 'flight_status.php'

# This keeps a log of all aircraft not yet cataloged in the AIRCRAFT_LENGTH dict
NEW_AIRCRAFT_FILE = 'secure/new_aircraft.txt'

FLAG_MSG_FLIGHT = 1  # basic flight details
FLAG_MSG_INSIGHT = 2  # random tidbit about a flight
FLAG_MSG_HISTOGRAM = 3  # histogram message
FLAG_MSG_CLEAR = 4  # a blank message to clear the screen
# user-entered message to display for some duration of time
FLAG_MSG_PERSONAL = 5

FLAG_INSIGHT_LAST_SEEN = 0
FLAG_INSIGHT_DIFF_AIRCRAFT = 1
FLAG_INSIGHT_NTH_FLIGHT = 2
FLAG_INSIGHT_GROUNDSPEED = 3
FLAG_INSIGHT_ALTITUDE = 4
FLAG_INSIGHT_VERTRATE = 5
FLAG_INSIGHT_FIRST_DEST = 6
FLAG_INSIGHT_FIRST_ORIGIN = 7
FLAG_INSIGHT_FIRST_AIRLINE = 8




                            <----SKIPPED LINES---->




    '  function gtag(){dataLayer.push(arguments);}\n'
    "  gtag('js', new Date());\n"
    "  gtag('config', 'UA-99931533-2');\n"
    '</script>\n')

#if running on raspberry, then need to prepend path to file names
if RASPBERRY_PI:
  MEMORY_DIRECTORY = MESSAGEBOARD_PATH + MEMORY_DIRECTORY
  PICKLE_FLIGHTS = MESSAGEBOARD_PATH + PICKLE_FLIGHTS
  PICKLE_SYSTEM_DASHBOARD = MESSAGEBOARD_PATH + PICKLE_SYSTEM_DASHBOARD
  PICKLE_FLIGHT_DASHBOARD = MESSAGEBOARD_PATH + PICKLE_FLIGHT_DASHBOARD
  LOGFILE = MESSAGEBOARD_PATH + LOGFILE
  PICKLE_DUMP_JSON_FILE = MESSAGEBOARD_PATH + PICKLE_DUMP_JSON_FILE
  PICKLE_FA_JSON_FILE = MESSAGEBOARD_PATH + PICKLE_FA_JSON_FILE
  PICKLE_SCREENS = MESSAGEBOARD_PATH + PICKLE_SCREENS
  CODE_REPOSITORY = MESSAGEBOARD_PATH

  HISTOGRAM_CONFIG_FILE = WEBSERVER_PATH + HISTOGRAM_CONFIG_FILE
  CONFIG_FILE = WEBSERVER_PATH + CONFIG_FILE
  ROLLING_MESSAGE_FILE = WEBSERVER_PATH + ROLLING_MESSAGE_FILE

  ROLLING_LOGFILE = WEBSERVER_PATH + ROLLING_LOGFILE
  STDERR_FILE = WEBSERVER_PATH + STDERR_FILE
  BACKUP_FILE = WEBSERVER_PATH + BACKUP_FILE
  SERVICE_VERIFICATION_FILE = WEBSERVER_PATH + SERVICE_VERIFICATION_FILE
  UPTIMES_FILE = WEBSERVER_PATH + UPTIMES_FILE
  FLIGHT_HISTORY = SECURE_WEBSERVER_PATH + UPTIMES_FILE
  CODE_HISTORY_FILE = WEBSERVER_PATH + CODE_HISTORY_FILE
  NEW_AIRCRAFT_FILE = WEBSERVER_PATH + NEW_AIRCRAFT_FILE
  FLIGHT_STATUS = SECURE_WEBSERVER_PATH + FLIGHT_STATUS

  HISTOGRAM_IMAGE_HTML = WEBSERVER_PATH + HISTOGRAM_IMAGE_HTML
  HOURLY_IMAGE_FILE = (
      WEBSERVER_PATH + WEBSERVER_IMAGE_RELATIVE_FOLDER + HOURLY_IMAGE_FILE)
  VERSION_REPOSITORY = WEBSERVER_PATH + VERSION_REPOSITORY

TIMEZONE = 'US/Pacific' # timezone of display
TZ = pytz.timezone(TIMEZONE)

# iata codes that we don't need to expand
KNOWN_AIRPORTS = ('SJC', 'SFO', 'OAK')

SPLITFLAP_CHARS_PER_LINE = 22
SPLITFLAP_LINE_COUNT = 6

DIRECTIONS_4 = ['N', 'E', 'S', 'W']
DIRECTIONS_8 = ['N', 'NE', 'E', 'SE', 'S', 'SW', 'W', 'NW']
DIRECTIONS_16 = ['N', 'NNE', 'NE', 'ENE', 'E', 'ESE', 'SE', 'SSE',
                 'S', 'SSW', 'SW', 'WSW', 'W', 'WNW', 'NW', 'NNW']





                            <----SKIPPED LINES---->




    persistent_path: dictionary where keys are flight numbers, and the values
      are a sequential list of the location-attributes in the json file; allows
      for tracking the flight path over time.
    log_jsons: boolean indicating whether we should pickle the JSONs.
    flights: list of flight dictionaries; if no json is returned, used to
      find a recent flight with same flight number to augment this flight with
      origin / destination / airline.

  Returns:
    A tuple:
    - updated persistent_nearby_aircraft
    - (possibly empty) dictionary of flight attributes of the new flight upon
      its first observation.
    - the time of the radio observation if present; None if no radio dump
    - a dictionary of attributes about the dump itself (i.e.: # of flights;
      furthest observed flight, etc.)
    - persistent_path, a data structure containing past details of a flight's
      location as described in ParseDumpJson
    - a text message indicating any errors in querying FlightAware or
      populating flight details
    - text string of SUCCESS, WARNING, or FAILURE: warning meaning the query
      occurred to soon after the last FA request, FAILURE if the request failed
      for some other reason, and SUCCESS if it was otherwise successful.
    - timestamp indicating exact time at which FlightAware was queried (or
      attempted to be queried), if a query was made in this pass
  """
  flight_details = {}
  error_message = ''
  now = time.time()
  if SIMULATION:
    (dump_json, json_time) = DUMP_JSONS[SIMULATION_COUNTER]
  else:
    dump_json = ReadFile(DUMP_JSON_FILE, log_exception=True)
  # Often there is no flight aware query, so we need to give a default value
  flight_aware_timestamp = 0

  json_desc_dict = {}
  current_nearby_aircraft = {}
  if dump_json:
    (current_nearby_aircraft, now,
     json_desc_dict, persistent_path) = ParseDumpJson(
         dump_json, persistent_path)





                            <----SKIPPED LINES---->




        # newly_nearby_flight_identifiers is a list of 2-tuples, where each
        # tuple is (flight_number, squawk)
        newly_nearby_flight_identifiers_str = ', '.join(
            ['%s/%s ' % (*i,) for i in newly_nearby_flight_identifiers])
        newly_nearby_flight_details_str = '\n'.join([
            str(current_nearby_aircraft[f])
            for f in newly_nearby_flight_identifiers])
        Log('Multiple newly-nearby flights: %s\n%s' % (
            newly_nearby_flight_identifiers_str,
            newly_nearby_flight_details_str))
      flight_identifier = newly_nearby_flight_identifiers[0]

      flight_aware_json = {}
      if SIMULATION:
        json_times = [j[1] for j in FA_JSONS]
        if json_time in json_times:
          flight_aware_json = FA_JSONS[json_times.index(json_time)][0]
        flight_aware_timestamp = time.time()  # "real" timestamp unavailable
      elif flight_identifier[0]:
        flight_number = flight_identifier[0]
        flight_aware_json, error_message, status, flight_aware_timestamp = (
            GetFlightAwareJson(flight_number))
        if flight_aware_json:
          UpdateStatusLight(GPIO_ERROR_FLIGHT_AWARE_CONNECTION, False)
        else:
          failure_message = 'No json from Flightaware for flight %s: %s' % (
              flight_number, error_message[:500])
          Log(failure_message)
          UpdateStatusLight(
              GPIO_ERROR_FLIGHT_AWARE_CONNECTION, True, failure_message)

      flight_details = {}
      if flight_aware_json:
        flight_details = ParseFlightAwareJson(flight_aware_json)
      elif flight_identifier[0]:  # if there's a flight number but no json
        flight_details, derived_attr_msg = FindAttributesFromSimilarFlights(
            flight_identifier[0], flights)
        error_message = '%s; %s' % (error_message, derived_attr_msg)

      if not SIMULATION and log_jsons:
        PickleObjectToFile((flight_aware_json, now), PICKLE_FA_JSON_FILE, True)

      # Augment FlightAware details with radio / radio-derived details
      flight_details.update(current_nearby_aircraft[flight_identifier])

      # Augment with the past location data; the [1] is because recall that
      # persistent_path[key] is actually a 2-tuple, the first element being
      # the most recent time seen, and the second element being the actual
      # path. But we do not need to keep around the most recent time seen any
      # more.
      flight_details['persistent_path'] = persistent_path[flight_identifier][1]

  return (
      persistent_nearby_aircraft,
      flight_details,
      now,
      json_desc_dict,
      persistent_path,
      error_message,
      status,
      flight_aware_timestamp)


def DescribeDumpJson(parsed):
  """Generates dict with descriptive attributes about the dump json file.

  Args:
    parsed: The parsed json file.

  Returns:
    Dictionary with attributes about radio range, number of flights seen, etc.
  """
  json_desc_dict = {}
  json_desc_dict['now'] = parsed['now']

  aircraft = [a for a in parsed['aircraft'] if a['seen'] < PERSISTENCE_SECONDS]
  json_desc_dict['radio_range_flights'] = len(aircraft)

  aircraft_with_pos = [a for a in aircraft if 'lat' in a and 'lon' in a]
  current_distances = [HaversineDistanceMeters(




                            <----SKIPPED LINES---->




    (last_seen, current_path) = persistent_path[f]
    if last_seen < now - PERSISTENCE_SECONDS:
      persistent_path.pop(f)

  return (nearby_aircraft, now, json_desc_dict, persistent_path)


last_query_time = 0
def GetFlightAwareJson(flight_number):
  """Scrapes the text json message from FlightAware for a given flight number.

  Given a flight number, loads the corresponding FlightAware webpage for that
  flight and extracts the relevant script that contains all the flight details
  from that page.  But only queries at most once per fixed period of time
  so as to avoid being blocked.

  Args:
    flight_number: text flight number (i.e.: SWA1234)

  Returns:
    Four tuple:
     - Text representation of the json message from FlightAware.
     - Text string of error message, if any
     - Text string of SUCCESS, WARNING, or FAILURE: warning meaning the query
       occurred to soon after the last FA request, FAILURE if the request failed
       for some other reason, and SUCCESS if it was otherwise successful.
     - Timestamp of attempted query on FlightAware
  """
  min_query_delay_seconds = 90
  url = 'https://flightaware.com/live/flight/' + flight_number

  global last_query_time
  seconds_since_last_query = time.time() - last_query_time
  if last_query_time and seconds_since_last_query < min_query_delay_seconds:
    error_msg = (
        'Unable to query FA for %s at %s since last query to FA was only'
        ' %d seconds ago; min of %d seconds needed: %s' % (
            flight_number,
            EpochDisplayTime(time.time(), format_string='%H:%M:%S'),
            seconds_since_last_query, min_query_delay_seconds, url))
    flight_aware_status_code = 'WARNING'
    return '', error_msg, flight_aware_status_code, time.time()

  last_query_time = time.time()

  try:
    response = requests.get(url, timeout=5)
    query_time = time.time()
  except requests.exceptions.RequestException as e:
    query_time = time.time()  # did not get to the query_time assignment above
    error_msg = 'Unable to query FA for URL due to %s: %s' % (e, url)
    flight_aware_status_code = 'FAILURE'
    return '', error_msg, flight_aware_status_code, query_time

  soup = bs4.BeautifulSoup(response.text, 'html.parser')
  l = soup.find_all('script')
  flight_script = None
  for script in l:
    if 'trackpollBootstrap' in str(script):
      flight_script = str(script)
      break
  if not flight_script:
    error_msg = (
        'Unable to find trackpollBootstrap script in page: ' + response.text)
    Log(error_msg)
    flight_aware_status_code = 'FAILURE'
    return '', error_msg, flight_aware_status_code, query_time
  first_open_curly_brace = flight_script.find('{')
  last_close_curly_brace = flight_script.rfind('}')
  flight_json = flight_script[first_open_curly_brace:last_close_curly_brace+1]
  flight_aware_status_code = 'SUCCESS'
  return flight_json, '', flight_aware_status_code, query_time


def Unidecode(s):
  """Convert a special unicode characters to closest ASCII representation."""
  if s is not None:
    s = unidecode.unidecode(s)
  return s


def FindAttributesFromSimilarFlights(this_flight_number, flights):
  """Returns a dictionary with info about a flight based on other flights.

  We may not get a json from the internet about this flight for any number
  of reasons: internet down; website down; too frequent queries; etc.  However,
  there are still some basic attributes we can derive about this flight
  from past observations of the same flight number, or past observations
  about the flight number prefix.  Specifically, we can get the flight's
  airline, origin, and destination.

  Args:




                            <----SKIPPED LINES---->




  f.close()


def SimulationSetup():
  """Updates global variable file names and loads JSON data for simulations."""
  # Clear file so that shell tail -f process can continue to point to same file
  def ClearFile(filename):
    if os.path.exists(filename):
      with open(filename, 'w') as f:
        f.write('')

  global SIMULATION
  SIMULATION = True

  global DUMP_JSONS
  DUMP_JSONS = UnpickleObjectFromFile(PICKLE_DUMP_JSON_FILE, True)

  global FA_JSONS
  FA_JSONS = UnpickleObjectFromFile(PICKLE_FA_JSON_FILE, True)





  global LOGFILE
  LOGFILE = PrependFileName(LOGFILE, SIMULATION_PREFIX)
  ClearFile(LOGFILE)

  global ROLLING_LOGFILE
  ROLLING_LOGFILE = PrependFileName(ROLLING_LOGFILE, SIMULATION_PREFIX)
  ClearFile(ROLLING_LOGFILE)

  global ROLLING_MESSAGE_FILE
  ROLLING_MESSAGE_FILE = PrependFileName(
      ROLLING_MESSAGE_FILE, SIMULATION_PREFIX)
  ClearFile(ROLLING_MESSAGE_FILE)

  global PICKLE_FLIGHTS
  PICKLE_FLIGHTS = PrependFileName(PICKLE_FLIGHTS, SIMULATION_PREFIX)
  filenames = UnpickleObjectFromFile(
      PICKLE_FLIGHTS, True, max_days=None, filenames=True)
  for file in filenames:
    ClearFile(file)





                            <----SKIPPED LINES---->





  The message is pushed to the vestaboard splitflap display by way of its
  web services; see https://docs.vestaboard.com/introduction for more details;
  if the web service fails, it then reattempts to publish using the local api.

  TODO: rewrite to use the easier-to-follow requests library, more in line
  with PublishMessageLocal.

  Args:
    s: String to publish.
    subscription_id: string subscription id from Vestaboard.
    key: string key from Vestaboard.
    secret: string secret from Vestaboard.
    timeout: Max duration in seconds that we should wait to establish a
      connection.

  Returns:
    Two-tuple:
    - Text string indicating how the message was displayed (web or local
      api), including error messages encountered if any.
    - Status code which is one of the text strings SUCCESS, WARNING, or FAILURE,
      indicating whether the web service was used (SUCCESS), the local service
      was used because the web service failed (WARNING), or both failed
      (FAILURE).
  """
  error_code = False
  curl = pycurl.Curl()

  # See https://stackoverflow.com/questions/31826814/
  # curl-post-request-into-pycurl-code
  # Set URL value
  curl.setopt(
      pycurl.URL,
      'https://platform.vestaboard.com/subscriptions/%s/message'
      % subscription_id)
  curl.setopt(pycurl.HTTPHEADER, [
      'X-Vestaboard-Api-Key:%s' % key, 'X-Vestaboard-Api-Secret:%s' % secret])
  curl.setopt(pycurl.TIMEOUT_MS, timeout*1000)
  curl.setopt(pycurl.POST, 1)

  curl.setopt(pycurl.WRITEFUNCTION, lambda x: None) # to keep stdout clean

  # preparing body the way pycurl.READDATA wants it
  body_as_dict = {'characters': StringToCharArray(s)}




                            <----SKIPPED LINES---->




  body_as_file_object = io.StringIO(body_as_json_string)

  # prepare and send. See also: pycurl.READFUNCTION to pass function instead
  curl.setopt(pycurl.READDATA, body_as_file_object)
  curl.setopt(pycurl.POSTFIELDSIZE, len(body_as_json_string))
  web_publish_failure_msg = ''
  try:
    curl.perform()
    status_msg = 'Web service published'
    publish_status_code = 'SUCCESS'
  except pycurl.error as e:
    timing_message = CurlTimingDetailsToString(curl)
    web_publish_failure_msg = (
        'curl.perform() failed with message %s; timing details: %s' %
        (e, timing_message))
    # Using the remote webservice failed, but maybe the local API will
    # be more successful?  If this succeeds, then we should not indicate
    # a failure on the status light / dashboard, but we should still log
    # the remote failure
    Log(web_publish_failure_msg)
    local_publish_error_code, local_publish_error_msg = PublishMessageLocal(
        s, timeout=timeout, update_dashboard=False)
    if not local_publish_error_code:
      status_msg = (
          '(1) Web service failed with %s so local service used'
          % web_publish_failure_msg)
      publish_status_code = 'WARNING'
    else:
      status_msg = (
          '(2) Web service failed with %s; then local service failed with %s' %
          (web_publish_failure_msg, local_publish_error_msg))
      publish_status_code = 'FAILURE'
  else:
    # you may want to check HTTP response code, e.g.
    timing_message = CurlTimingDetailsToString(curl)
    status_code = curl.getinfo(pycurl.RESPONSE_CODE)
    if status_code != 200:
      web_publish_failure_msg = (
          'Server returned HTTP status code %d for message %s; '
          'timing details: %s' % (status_code, s, timing_message))
      Log(web_publish_failure_msg)
      local_publish_error_code, local_publish_error_msg = PublishMessageLocal(
          s, timeout=timeout, update_dashboard=False)
      if not local_publish_error_code:
        status_msg = (
            '(3) Web service failed with %s so local service used'
            % web_publish_failure_msg)
        publish_status_code = 'WARNING'
      else:
        status_msg = (
            '(4) Web service failed with %s; '
            'then local service failed with %s' %
            (web_publish_failure_msg, local_publish_error_msg))
        publish_status_code = 'FAILURE'

  # We've logged the error code from the external web service, but the
  # Vestaboard local API was able to recover from the error, so we need not
  # log the failure message / error code in the dashboard.
  if not error_code:
    web_publish_failure_msg = ''

  curl.close()
  UpdateStatusLight(
      GPIO_ERROR_VESTABOARD_CONNECTION,
      publish_status_code=='FAILURE', web_publish_failure_msg)

  return status_msg, publish_status_code


def PublishMessageLocal(
    s,
    local_key=LOCAL_KEY,
    local_address=LOCAL_VESTABOARD_ADDRESS,
    timeout=5,
    update_dashboard=True):
  """Publishes a text string to a Vestaboard via local API.

  The message is pushed to the vestaboard splitflap display by way of its
  local API; see https://docs.vestaboard.com/local for more details.

  Args:
    s: String to publish.
    local_key: string key from Vestaboard for local API access.
    local_address: the address and port to the local Vestaboard service.
    timeout: Max duration in seconds that we should wait to establish a
      connection.
    update_dashboard: Boolean indicating whether this method should update the
      system dashboard, or if that should be left to the calling function.

  Returns:
    2-tuple:
    - error_code: False if successful; error message string if failure occurs.
    - status_msg: Text description of status.
  """
  error_code = False
  error_msg = ''

  data = str(StringToCharArray(s))
  headers = {
      'X-Vestaboard-Local-Api-Key': local_key,
      'Content-Type': 'application/x-www-form-urlencoded',
  }

  response = 'Unable to complete requests.post'
  try:
    response = requests.post(
        local_address,
        headers=headers, data=data, timeout=timeout)
  except requests.exceptions.RequestException as e:
    status_msg = 'Unable to reach %s (%s): %s' % (local_address, response, e)
    Log(status_msg)
    error_code = True
  if not error_code:
    status_msg = 'Message sent to Vestaboard by way of local api: %s' % s
    Log(status_msg)

  if update_dashboard:
    UpdateStatusLight(
        GPIO_ERROR_VESTABOARD_CONNECTION, error_code, error_msg)
  return error_code, status_msg


def CurlTimingDetailsToString(curl):
  """Extracts timing details of a curl request into a readable string."""
  timing = {}
  timing['total-time'] = curl.getinfo(pycurl.TOTAL_TIME)
  timing['namelookup-time'] = curl.getinfo(pycurl.NAMELOOKUP_TIME)
  timing['connect-time'] = curl.getinfo(pycurl.CONNECT_TIME)
  timing['pretransfer-time'] = curl.getinfo(pycurl.PRETRANSFER_TIME)
  timing['redirect-time'] = curl.getinfo(pycurl.REDIRECT_TIME)
  timing['starttransfer-time'] = curl.getinfo(pycurl.STARTTRANSFER_TIME)
  results = [label + '=' + '%.4f' % timing[label] for label in timing]
  results = '; '.join(results)
  return results


def TruncateEscapedLine(s):
  """Formats a single line of the personal message for the Vestaboard.

  The Vestaboard has line length limitations, a limited character set,




                            <----SKIPPED LINES---->




      # There may be one or several insight messages that were added to the
      # message queue along with the flight at a time when the screen was
      # enabled, but by the time it comes to display them, the screen is now
      # disabled.  These should not be displayed.  Note that this check only
      # needs to be done for insight messages because other message types
      # are user initiated and so presumably should be displayed irrespective
      # of when the user triggered it to be displayed.
      if message_type == FLAG_MSG_INSIGHT and not MessageMeetsDisplayCriteria(
          configuration):
        publish_status_msg = (
            'Message purged as no longer meets display criteria')
        publish_status_code = 'WARNING'
        Log('Message %s purged' % message_text)

      else:
        if isinstance(message_text, str):
          message_text = textwrap.wrap(
              message_text,
              width=SPLITFLAP_CHARS_PER_LINE)
        display_message = Screenify(message_text, False)


        # Saving this to disk allows us to identify
        # persistently whats currently on the screen
        PickleObjectToFile(message, PICKLE_SCREENS, True)
        screens.append(message)

        MaintainRollingWebLog(display_message, 25)
        if not SIMULATION:
          splitflap_message = Screenify(message_text, True)
          publish_status = PublishMessageWeb(splitflap_message)
          publish_status_msg, publish_status_code = publish_status

      if message_type in (FLAG_MSG_INSIGHT, FLAG_MSG_FLIGHT):
        flight = message[2]
        # We record flight number, time stamp, message, and status
        # to a pickle file so that we may construct a flight-centric
        # status report
        #
        # Specifically, we record a 3-tuple:
        # - flight number




                            <----SKIPPED LINES---->





    ResetLogs(configuration)  # clear the logs if requested
    UpdateRollingLogSize(configuration)

    # if this is a SIMULATION, then process every diff dump. But if it
    # isn't a simulation, then only read & do related processing for the
    # next dump if the last-modified timestamp indicates the file has been
    # updated since it was last read.
    tmp_timestamp = 0
    if not SIMULATION:
      dump_json_exists = os.path.exists(DUMP_JSON_FILE)
      if dump_json_exists:
        tmp_timestamp = os.path.getmtime(DUMP_JSON_FILE)
    if (SIMULATION and DumpJsonChanges()) or (
        not SIMULATION and dump_json_exists and
        tmp_timestamp > last_dump_json_timestamp):

      last_dump_json_timestamp = tmp_timestamp

      (persistent_nearby_aircraft, flight, now, json_desc_dict,
       persistent_path, flight_aware_error_message,
       flight_aware_status_code, flight_aware_timestamp) = (
           ScanForNewFlights(
               persistent_nearby_aircraft,
               persistent_path,
               configuration.get('log_jsons', False),
               flights))

      # Logging: As part of the memory instrumentation, let's track
      # the length of these data structures
      if not iteration % 1000:
        lengths = [len(flights), len(screen_history)]
        Log('Iteration: %d: object lengths: %s' % (iteration, lengths))

      # because this might just be an updated instance of the previous
      # flight as more identifier information (squawk and or flight number)
      # comes in, we only want to process this if its a truly new flight
      new_flight_flag = ConfirmNewFlight(flight, flights)

      if new_flight_flag:
        # now comes directly from the radio json
        time_new_flight_found = now




                            <----SKIPPED LINES---->




            flight, PICKLE_FLIGHTS, True, timestamp=flight['now'])

        # We record flight number, flight_meets_display_criteria,
        # reason_flight_fails_criteria, flight_aware_error_message, and
        # time stamps of when we confirmed new flight; when we generated
        # message; when we generated insight messages (if any), to a
        # to a pickle file so that we may construct a flight-centric
        # status report
        #
        # Specifically, we record a 3-tuple:
        # - flight number
        # - time stamp of the recording
        # - a dictionary of elements
        data = (
            DisplayFlightNumber(flight),
            time.time(),
            {
                'reason_flight_fails_criteria': reason_flight_fails_criteria,
                'flight_aware_timestamp': flight_aware_timestamp,
                'flight_aware_error_message': flight_aware_error_message,
                'flight_aware_status_code': flight_aware_status_code,
                'time_new_flight_found': time_new_flight_found,
                'time_flight_message_inserted': time_flight_message_inserted,
                'time_insight_message_inserted': time_insight_message_inserted})
        PickleObjectToFile(data, PICKLE_FLIGHT_DASHBOARD, True)

      else:
        remote, servo = RefreshArduinos(
            remote, servo,
            to_remote_q, to_servo_q, to_main_q, shutdown,
            flights, json_desc_dict, configuration, screen_history)

    message_queue, next_message_time = ProcessArduinoCommmands(
        to_main_q, flights, configuration, message_queue, next_message_time)

    personal_message = PersonalMessage(
        configuration, message_queue, personal_message)

    # MEMORY MANAGEMENT
    # it turns out we only need the last screen in the screen history, not
    # the entire history, so we can purge all the rest from active memory




                            <----SKIPPED LINES---->