01234567890123456789012345678901234567890123456789012345678901234567890123456789
54455446544754485449545054515452545354545455545654575458545954605461546254635464 54655466 54675468546954705471547254735474547554765477547854795480548154825483548454855486 54965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542 554355445545 554655475548 55495550555155525553555455555556555755585559556055615562556355645565556655675568 | <----SKIPPED LINES----> signal.Signals(signalNumber).name) # pylint: disable=E1101 global SHUTDOWN_SIGNAL SHUTDOWN_SIGNAL = msg Log(msg) def PerformGracefulShutdown(queues, shutdown, reboot): """Complete the graceful shutdown process by cleaning up. Args: queues: iterable of queues shared with child processes to be closed shutdown: tuple of shared flags with child processes to initiate shutdown in children reboot: boolean indicating whether we should trigger a reboot """ reboot_msg = '' if reboot: reboot_msg = ' and rebooting' Log('Shutting down self (%d)%s' % (os.getpid(), reboot_msg)) for q in queues: q.close() for v in shutdown: # send the shutdown signal to child processes v.value = 1 if RASPBERRY_PI: RPi.GPIO.cleanup() UpdateDashboard(True, failure_message=SHUTDOWN_SIGNAL) if reboot or REBOOT_SIGNAL: time.sleep(10) # wait 10 seconds for children to shut down as well os.system('sudo reboot') sys.exit() def FindRunningParents(): """Returns proc ids of processes with identically-named python file running. In case there are multiple children processes spawned with the same name, such as via multiprocessing, this will only return the parent id (since a killed child process will likely just be respawned). """ <----SKIPPED LINES----> command_running = any( [this_process_name in s for s in commands['cmdline']]) if command_running: pids.append(commands['pid']) pid_pairs.append((commands['pid'], commands['ppid'])) except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): pass # Exclude those pids that have a parent that is also a pid final_pids = [] for pid_pair in pid_pairs: if pid_pair[1] not in pids: final_pids.append(pid_pair[0]) # Exclude this pid final_pids.pop(final_pids.index(this_process_id)) return sorted(final_pids) def WaitUntilKillComplete(already_running_ids, max_seconds=30): """Prevents main loop from starting until other instancs complete shutdown. A termination command send to any other identically-named process may take a few seconds to complete because that other process is allowed to finish the current iteration in the main loop. Typically, that iteration in the other process will complete before this process finishes the initialization and starts. But in limited scenarios, that might not happen, such as if the other process is in the middle of generating a lot of histogram images, or if this process does not have much data to load. This function ensures that this process does not start the main loop until the other process terminates. If it detects that the other process is still running, it waits for up to max_seconds. If the other process does not terminate before that time limit, then this restarts the RPi. """ still_running_ids = FindRunningParents() if not still_running_ids: return # still_running_ids should at least be a subset of already_running_ids new_processes = sorted( list(set(still_running_ids).difference(set(already_running_ids)))) if new_processes: # uh-oh! at least one new started up in the interim? exit! Log('Kill signal sent to %s from this process %s, but it ' 'seems like there is at least one new process running, %s!' % ( str(already_running_ids), str(os.getpid()), str(new_processes))) sys.exit() # phew - they're a subset; so they probably got the # signal; just wait a few secs elif still_running_ids: n = 0 running_parents = FindRunningParents() while running_parents: if n == max_seconds: Log('Kill signal sent from this process %d to %s, but %s still ' 'running after waiting cume %d seconds; rebooting' % ( os.getpid(), str(already_running_ids), str(running_parents), n+1)) PerformGracefulShutdown((), (), True) if not n % 3: Log('Kill signal sent from this process %d to %s, but %s still ' 'running after waiting cume %d seconds' % ( os.getpid(), str(already_running_ids), str(running_parents), n)) n += 1 time.sleep(1) running_parents = FindRunningParents() def InitArduinoVariables(): """Initializes variables for arduino threads with shared-memory queues.""" to_remote_q = multiprocessing.Queue() to_servo_q = multiprocessing.Queue() <----SKIPPED LINES----> |
01234567890123456789012345678901234567890123456789012345678901234567890123456789
5445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490 55005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540 5541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586 | <----SKIPPED LINES----> signal.Signals(signalNumber).name) # pylint: disable=E1101 global SHUTDOWN_SIGNAL SHUTDOWN_SIGNAL = msg Log(msg) def PerformGracefulShutdown(queues, shutdown, reboot): """Complete the graceful shutdown process by cleaning up. Args: queues: iterable of queues shared with child processes to be closed shutdown: tuple of shared flags with child processes to initiate shutdown in children reboot: boolean indicating whether we should trigger a reboot """ reboot_msg = '' if reboot: reboot_msg = ' and rebooting' Log('Shutting down self (%d)%s' % (os.getpid(), reboot_msg)) # TODO: the q.close() may have been blocking some of the shutdowns, since # it is blocking until the queue is fully drained. Let's see if we have # fewer problems without this code. #for q in queues: # q.close() for v in shutdown: # send the shutdown signal to child processes v.value = 1 if RASPBERRY_PI: RPi.GPIO.cleanup() UpdateDashboard(True, failure_message=SHUTDOWN_SIGNAL) if reboot or REBOOT_SIGNAL: time.sleep(10) # wait 10 seconds for children to shut down as well os.system('sudo reboot') sys.exit() def FindRunningParents(): """Returns proc ids of processes with identically-named python file running. In case there are multiple children processes spawned with the same name, such as via multiprocessing, this will only return the parent id (since a killed child process will likely just be respawned). """ <----SKIPPED LINES----> command_running = any( [this_process_name in s for s in commands['cmdline']]) if command_running: pids.append(commands['pid']) pid_pairs.append((commands['pid'], commands['ppid'])) except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): pass # Exclude those pids that have a parent that is also a pid final_pids = [] for pid_pair in pid_pairs: if pid_pair[1] not in pids: final_pids.append(pid_pair[0]) # Exclude this pid final_pids.pop(final_pids.index(this_process_id)) return sorted(final_pids) def WaitUntilKillComplete(already_running_ids, max_seconds=30): """Prevents main loop from starting until other instances complete shutdown. A termination command send to any other identically-named process may take a few seconds to complete because that other process is allowed to finish the current iteration in the main loop. Typically, that iteration in the other process will complete before this process finishes the initialization and starts. But in limited scenarios, that might not happen, such as if the other process is in the middle of generating a lot of histogram images, or if this process does not have much data to load. This function ensures that this process does not start the main loop until the other process terminates. If it detects that the other process is still running, it waits for up to max_seconds. If the other process does not terminate before that time limit, then this restarts the RPi. """ still_running_ids = FindRunningParents() if not still_running_ids: return # still_running_ids should at least be a subset of already_running_ids new_processes = sorted( list(set(still_running_ids).difference(set(already_running_ids)))) if new_processes: # uh-oh! at least one new started up in the interim? exit! Log('Kill signal sent to %s from this process %s, but it ' 'seems like there is at least one new process running, %s!' % ( str(already_running_ids), str(os.getpid()), str(new_processes))) sys.exit() # phew - they're a subset; so they probably got the # signal; just wait a few secs elif still_running_ids: # handle race condition of two instances both in startup phase; while # this is very unusual, it happened at least once, on July 1 at 17:43:23 # with process IDs 1106 & 1110. if SHUTDOWN_SIGNAL: Log('Race condition? Though this process %s is just in startup, ' 'SHUTDOWN_SIGNAL has already been set to True\n' 'already_running_ids (PIDs detected when process started up): %s\n' 'still_running_ids (PIDs detected at start of function call): %s\n' '\nProcess shutting down' % (str(os.getpid()), already_running_ids, still_running_ids)) sys.exit() n = 0 running_parents = FindRunningParents() while running_parents: if n == max_seconds: Log('Kill signal sent from this process %d to %s, but %s still ' 'running after waiting cume %d seconds; rebooting' % ( os.getpid(), str(already_running_ids), str(running_parents), n+1)) PerformGracefulShutdown((), (), True) if not n % 3: Log('Kill signal sent from this process %d to %s, but %s still ' 'running after waiting cume %d seconds' % ( os.getpid(), str(already_running_ids), str(running_parents), n)) n += 1 time.sleep(1) running_parents = FindRunningParents() def InitArduinoVariables(): """Initializes variables for arduino threads with shared-memory queues.""" to_remote_q = multiprocessing.Queue() to_servo_q = multiprocessing.Queue() <----SKIPPED LINES----> |