smallpond.execution.scheduler.Scheduler#

class smallpond.execution.scheduler.Scheduler(exec_plan: ExecutionPlan, max_retry_count: int = 5, max_fail_count: int = 3, prioritize_retry=False, speculative_exec: Literal['disable', 'enable', 'aggressive'] = 'enable', stop_executor_on_failure=False, nonzero_exitcode_as_oom=False, remove_output_root=False, sched_state_observers=None)#

The task scheduler.

__init__(exec_plan: ExecutionPlan, max_retry_count: int = 5, max_fail_count: int = 3, prioritize_retry=False, speculative_exec: Literal['disable', 'enable', 'aggressive'] = 'enable', stop_executor_on_failure=False, nonzero_exitcode_as_oom=False, remove_output_root=False, sched_state_observers=None) → None#

Methods

__init__(exec_plan[, max_retry_count, ...])
add_state_observer(observer)
clean_temp_files(pool)
clear_cached_executor_lists()
copy_task_for_execution(task)
dispatch_tasks(pool)
export_task_metrics()
export_timeline_figs()
get_retry_task(key)
get_runnable_tasks(finished_task)
log_current_status()
log_overall_progress()
notify_state_observers([force_notify])
probe_executors()
process_finished_tasks(pool)
run()
save_task_final_state(finished_task)
sched_loop(pool)
start_speculative_execution()
stop_executors()
stop_running_tasks(task_key)
suspend_good_executors()
try_boost_resource(item, executor)
try_enqueue(tasks)
try_relax_memory_limit(task, executor)
update_executor_states()

Attributes

StateCallback	alias of Callable[[Scheduler], Any]
abandoned_tasks
alive_executors
elapsed_time
failed_executors
good_executors
large_num_nontrivial_tasks
large_runtime_state
local_executors
low_resource_executors
num_local_running_works
num_pending_nontrivial_tasks
num_pending_tasks
num_running_works
pending_nontrivial_tasks
progress
remote_executors
running_works
stopped_executors
stopping_executors
succeeded_task_ids
success
working_executors