01234567890123456789012345678901234567890123456789012345678901234567890123456789
719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236 73177318731973207321732273237324732573267327732873297330733173327333733473357336 73377338733973407341734273437344734573467347734873497350735173527353735473557356 740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453 |
<----SKIPPED LINES---->
if initial_memory_dump:
tracemalloc.start(initial_frame_count)
startup_time = time.time()
json_desc_dict = {}
init_timing.append((time.time(), 3))
flights = UnpickleObjectFromFile(
PICKLE_FLIGHTS, True, max_days=MAX_INSIGHT_HORIZON_DAYS, heartbeat=True)
# Clear the loaded flight of any cached data, identified by keys
# with a specific suffix, since code fixes may change the values for
# some of those cached elements
init_timing.append((time.time(), 4))
for flight in flights:
for key in list(flight.keys()):
if key.endswith(CACHED_ELEMENT_PREFIX):
flight.pop(key)
init_timing.append((time.time(), 5))
# We have no use for the memory-hogging persistent_path key of historical
# flights, so let's remove all those keys and save a lot of memory
# TODO: On 2022-10-31, commented out anything deleting persistent path
#if flights:
# for flight in flights[:-1]:
# if 'persistent_path' in flight:
# del flight['persistent_path']
screen_history = UnpickleObjectFromFile(PICKLE_SCREENS, True, max_days=2)
# If we're displaying just a single insight message, we want it to be
# something unique, to the extent possible; this dict holds a count of
# the diff types of messages displayed so far
insight_message_distribution = {}
# bootstrap the flight insights distribution from a list of insights on each
# flight (i.e.: flight['insight_types'] for a given flight might look like
# [1, 2, 7, 9], or [], to indicate which insights were identified; this then
# transforms that into {0: 25, 1: 18, ...} summing across all flights.
missing_insights = []
for flight in flights:
if 'insight_types' not in flight:
missing_insights.append('%s on %s' % (
DisplayFlightNumber(flight), DisplayTime(flight, '%x %X')))
distribution = flight.get('insight_types', [])
for key in distribution:
insight_message_distribution[key] = (
<----SKIPPED LINES---->
(persistent_nearby_aircraft, flight, now, json_desc_dict,
persistent_path, flight_aware_error_message) = ScanForNewFlights(
persistent_nearby_aircraft,
persistent_path,
configuration.get('log_jsons', False),
flights)
# Logging: As part of the memory instrumentation, let's track
# the length of these data structures
if not iteration % 1000:
lengths = [len(flights), len(screen_history)]
Log('Iteration: %d: object lengths: %s' % (iteration, lengths))
# because this might just be an updated instance of the previous
# flight as more identifier information (squawk and or flight number)
# comes in, we only want to process this if its a truly new flight
new_flight_flag = ConfirmNewFlight(flight, flights)
if new_flight_flag:
time_new_flight_found = time.time()
flights.append(flight)
remote, servo = RefreshArduinos(
remote, servo,
to_remote_q, to_servo_q, to_main_q, shutdown,
flights, json_desc_dict, configuration, screen_history)
flight_meets_display_criteria, reason_flight_fails_criteria = (
FlightMeetsDisplayCriteria(flight, configuration, log=True))
# Initialize variables for saving in case details not populated by
# later code prior to saving in pickle
flight_aware_error_message = ''
time_flight_message_inserted = 0
time_insight_message_inserted = 0
if flight_meets_display_criteria:
personal_message = None # Any personal message displayed now cleared
flight_message = (
FLAG_MSG_FLIGHT, CreateMessageAboutFlight(flight), flight)
<----SKIPPED LINES---->
'reason_flight_fails_criteria': reason_flight_fails_criteria,
'flight_aware_error_message': flight_aware_error_message,
'time_new_flight_found': time_new_flight_found,
'time_flight_message_inserted': time_flight_message_inserted,
'time_insight_message_inserted': time_insight_message_inserted})
PickleObjectToFile(data, PICKLE_FLIGHT_DASHBOARD, True)
else:
remote, servo = RefreshArduinos(
remote, servo,
to_remote_q, to_servo_q, to_main_q, shutdown,
flights, json_desc_dict, configuration, screen_history)
message_queue, next_message_time = ProcessArduinoCommmands(
to_main_q, flights, configuration, message_queue, next_message_time)
personal_message = PersonalMessage(
configuration, message_queue, personal_message)
# MEMORY MANAGEMENT
# now that we've saved the flight, we have no more need to keep the
# memory-hogging persistent_path key of that flight in live memory
# TODO: On 2022-10-31, commented out anything deleting persistent path
#if 'persistent_path' in flights[-1]:
# del flights[-1]['persistent_path']
# it turns out we only need the last screen in the screen history, not
# the entire history, so we can purge all the rest from active memory
# to eliminate a slow-growing memory hog
if screen_history:
screen_history = [screen_history[-1]]
# we also do not need more than MAX_INSIGHT_HORIZON_DAYS of history
# of the flights, so we can purge flights older than that as the flights
# list slowly grows over time. we can search the entire list every time
# but that's unnecessary; instead we can just pop off the 0-th element
# each time through the list if it's more than 1 + max days in the past.
# Why + 1? Because the histogram generation expects that many *full* days
# of history, so to make sure we don't have a partial day if we were to
# generate the monthly histograms mid-day, we will keep an extra day.
if time.time() - flights[0]['now'] > SECONDS_IN_DAY * (
MAX_INSIGHT_HORIZON_DAYS + 1):
flights.pop(0)
if SIMULATION:
if now:
simulated_hour = EpochDisplayTime(now, '%Y-%m-%d %H:00%z')
<----SKIPPED LINES---->
|
01234567890123456789012345678901234567890123456789012345678901234567890123456789
71927193719471957196719771987199720072017202720372047205720672077208720972107211 721272137214721572167217721872197220722172227223722472257226722772287229723072317232723372347235 7316731773187319732073217322732373247325732673277328732973307331733273337334733573367337733873397340734173427343734473457346734773487349735073517352735373547355735673577358735973607361 74147415741674177418741974207421742274237424742574267427742874297430743174327433 74347435743674377438743974407441744274437444744574467447744874497450745174527453 |
<----SKIPPED LINES---->
if initial_memory_dump:
tracemalloc.start(initial_frame_count)
startup_time = time.time()
json_desc_dict = {}
init_timing.append((time.time(), 3))
flights = UnpickleObjectFromFile(
PICKLE_FLIGHTS, True, max_days=MAX_INSIGHT_HORIZON_DAYS, heartbeat=True)
# Clear the loaded flight of any cached data, identified by keys
# with a specific suffix, since code fixes may change the values for
# some of those cached elements
init_timing.append((time.time(), 4))
for flight in flights:
for key in list(flight.keys()):
if key.endswith(CACHED_ELEMENT_PREFIX):
flight.pop(key)
init_timing.append((time.time(), 5))
# We have no use for the memory-hogging persistent_path key of historical
# flights, so let's remove all those keys and save a lot of memory
if flights:
for flight in flights[:-1]:
if 'persistent_path' in flight:
del flight['persistent_path']
screen_history = UnpickleObjectFromFile(PICKLE_SCREENS, True, max_days=2)
# If we're displaying just a single insight message, we want it to be
# something unique, to the extent possible; this dict holds a count of
# the diff types of messages displayed so far
insight_message_distribution = {}
# bootstrap the flight insights distribution from a list of insights on each
# flight (i.e.: flight['insight_types'] for a given flight might look like
# [1, 2, 7, 9], or [], to indicate which insights were identified; this then
# transforms that into {0: 25, 1: 18, ...} summing across all flights.
missing_insights = []
for flight in flights:
if 'insight_types' not in flight:
missing_insights.append('%s on %s' % (
DisplayFlightNumber(flight), DisplayTime(flight, '%x %X')))
distribution = flight.get('insight_types', [])
for key in distribution:
insight_message_distribution[key] = (
<----SKIPPED LINES---->
(persistent_nearby_aircraft, flight, now, json_desc_dict,
persistent_path, flight_aware_error_message) = ScanForNewFlights(
persistent_nearby_aircraft,
persistent_path,
configuration.get('log_jsons', False),
flights)
# Logging: As part of the memory instrumentation, let's track
# the length of these data structures
if not iteration % 1000:
lengths = [len(flights), len(screen_history)]
Log('Iteration: %d: object lengths: %s' % (iteration, lengths))
# because this might just be an updated instance of the previous
# flight as more identifier information (squawk and or flight number)
# comes in, we only want to process this if its a truly new flight
new_flight_flag = ConfirmNewFlight(flight, flights)
if new_flight_flag:
time_new_flight_found = time.time()
# Since we no longer need the memory-hogging prior persistent_path
# stored on the flights, we can remove it
if flights and 'persistent_path' in flights[-1]:
del flights[-1]['persistent_path']
flights.append(flight)
remote, servo = RefreshArduinos(
remote, servo,
to_remote_q, to_servo_q, to_main_q, shutdown,
flights, json_desc_dict, configuration, screen_history)
flight_meets_display_criteria, reason_flight_fails_criteria = (
FlightMeetsDisplayCriteria(flight, configuration, log=True))
# Initialize variables for saving in case details not populated by
# later code prior to saving in pickle
flight_aware_error_message = ''
time_flight_message_inserted = 0
time_insight_message_inserted = 0
if flight_meets_display_criteria:
personal_message = None # Any personal message displayed now cleared
flight_message = (
FLAG_MSG_FLIGHT, CreateMessageAboutFlight(flight), flight)
<----SKIPPED LINES---->
'reason_flight_fails_criteria': reason_flight_fails_criteria,
'flight_aware_error_message': flight_aware_error_message,
'time_new_flight_found': time_new_flight_found,
'time_flight_message_inserted': time_flight_message_inserted,
'time_insight_message_inserted': time_insight_message_inserted})
PickleObjectToFile(data, PICKLE_FLIGHT_DASHBOARD, True)
else:
remote, servo = RefreshArduinos(
remote, servo,
to_remote_q, to_servo_q, to_main_q, shutdown,
flights, json_desc_dict, configuration, screen_history)
message_queue, next_message_time = ProcessArduinoCommmands(
to_main_q, flights, configuration, message_queue, next_message_time)
personal_message = PersonalMessage(
configuration, message_queue, personal_message)
# MEMORY MANAGEMENT
# it turns out we only need the last screen in the screen history, not
# the entire history, so we can purge all the rest from active memory
# to eliminate a slow-growing memory hog
if screen_history:
screen_history = [screen_history[-1]]
# we also do not need more than MAX_INSIGHT_HORIZON_DAYS of history
# of the flights, so we can purge flights older than that as the flights
# list slowly grows over time. we can search the entire list every time
# but that's unnecessary; instead we can just pop off the 0-th element
# each time through the list if it's more than 1 + max days in the past.
# Why + 1? Because the histogram generation expects that many *full* days
# of history, so to make sure we don't have a partial day if we were to
# generate the monthly histograms mid-day, we will keep an extra day.
if time.time() - flights[0]['now'] > SECONDS_IN_DAY * (
MAX_INSIGHT_HORIZON_DAYS + 1):
flights.pop(0)
if SIMULATION:
if now:
simulated_hour = EpochDisplayTime(now, '%Y-%m-%d %H:00%z')
<----SKIPPED LINES---->
|