Skip to content

Instantly share code, notes, and snippets.

@wconstab

wconstab/dump.txt

Created February 23, 2024 21:06
Show Gist options
  • Save wconstab/5095af7c25bcfb6993ffd5a55334f641 to your computer and use it in GitHub Desktop.
Save wconstab/5095af7c25bcfb6993ffd5a55334f641 to your computer and use it in GitHub Desktop.
Download ZIP
step_6 torchtrain pp flight dump
Raw
dump.txt
{'pg_id': 0, 'seq_id': 3, 'profiling_name': 'nccl:recv 0<-1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_1', 'first_time': '2024-02-23 12:56:31.060915', 'last_time': '2024-02-23 12:56:40.779100', 'Retired': '[0-1]', 'Active': '[0-1]', 'NotIssued': '[2-511]'}
{'pg_id': 0, 'seq_id': 4, 'profiling_name': 'nccl:recv 0<-1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_1', 'first_time': '2024-02-23 12:56:41.050293', 'last_time': '2024-02-23 12:56:43.105840', 'Retired': '[0-1]', 'Active': '[0-1]', 'NotIssued': '[2-511]'}
{'pg_id': 0, 'seq_id': 5, 'profiling_name': 'nccl:send 0->1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_0', 'first_time': '2024-02-23 12:56:41.118069', 'last_time': '2024-02-23 12:56:43.175982', 'Retired': '[0-1]', 'Active': '[0-1]', 'NotIssued': '[2-511]'}
{'pg_id': 0, 'seq_id': 6, 'profiling_name': 'nccl:send 0->1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_0', 'first_time': '2024-02-23 12:56:41.119742', 'last_time': '2024-02-23 12:56:43.177373', 'Retired': '[0-1]', 'Active': '[0-1]', 'NotIssued': '[2-511]'}
{'pg_id': 0, 'seq_id': 7, 'profiling_name': 'nccl:recv 0<-1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_1', 'first_time': '2024-02-23 12:56:41.161550', 'last_time': '2024-02-23 12:56:43.177581', 'Retired': '[0-1]', 'Active': '[0-1]', 'NotIssued': '[2-511]'}
{'pg_id': 0, 'seq_id': 8, 'profiling_name': 'nccl:recv 0<-1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_1', 'first_time': '2024-02-23 12:56:41.185324', 'last_time': '2024-02-23 12:56:43.178475', 'Retired': '[0-1]', 'Active': '[0-1]', 'NotIssued': '[2-511]'}
{'pg_id': 0, 'seq_id': 9, 'profiling_name': 'nccl:send 0->1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_0', 'first_time': '2024-02-23 12:56:41.250136', 'last_time': '2024-02-23 12:56:43.444962', 'Retired': '[0-1]', 'Active': '[0-1]', 'NotIssued': '[2-511]'}
{'pg_id': 0, 'seq_id': 10, 'profiling_name': 'nccl:send 0->1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_0', 'first_time': '2024-02-23 12:56:41.251639', 'last_time': '2024-02-23 12:56:43.446331', 'Retired': '[0-1]', 'Active': '[0-1]', 'NotIssued': '[2-511]'}
{'pg_id': 0, 'seq_id': 11, 'profiling_name': 'nccl:recv 0<-1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_1', 'first_time': '2024-02-23 12:56:43.219292', 'last_time': '2024-02-23 12:56:43.446548', 'Retired': '[0-1]', 'Active': '[0-1]', 'NotIssued': '[2-511]'}
{'pg_id': 0, 'seq_id': 12, 'profiling_name': 'nccl:recv 0<-1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_1', 'first_time': '2024-02-23 12:56:43.243140', 'last_time': '2024-02-23 12:56:43.447430', 'Retired': '[0-1]', 'Active': '[0-1]', 'NotIssued': '[2-511]'}
{'pg_id': 0, 'seq_id': 13, 'profiling_name': 'nccl:send 0->1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_0', 'first_time': '2024-02-23 12:56:43.307078', 'last_time': '2024-02-23 12:56:43.711056', 'Retired': '[0-1]', 'Active': '[0-1]', 'NotIssued': '[2-511]'}
{'pg_id': 0, 'seq_id': 14, 'profiling_name': 'nccl:send 0->1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_0', 'first_time': '2024-02-23 12:56:43.308692', 'last_time': '2024-02-23 12:56:43.712376', 'Retired': '[0-1]', 'Active': '[0-1]', 'NotIssued': '[2-511]'}
{'pg_id': 0, 'seq_id': 15, 'profiling_name': 'nccl:recv 0<-1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_1', 'first_time': '2024-02-23 12:56:43.350689', 'last_time': '2024-02-23 12:56:43.712610', 'Retired': '[0-1]', 'Active': '[0-1]', 'NotIssued': '[2-511]'}
{'pg_id': 0, 'seq_id': 16, 'profiling_name': 'nccl:recv 0<-1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_1', 'first_time': '2024-02-23 12:56:43.374696', 'last_time': '2024-02-23 12:56:43.713512', 'Retired': '[0-1]', 'Active': '[0-1]', 'NotIssued': '[2-511]'}
{'pg_id': 0, 'seq_id': 17, 'profiling_name': 'nccl:send 0->1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_0', 'first_time': '2024-02-23 12:56:43.441154', 'last_time': '2024-02-23 12:56:43.971789', 'Retired': '[0-1]', 'Active': '[0-1]', 'NotIssued': '[2-511]'}
{'pg_id': 0, 'seq_id': 18, 'profiling_name': 'nccl:send 0->1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_0', 'first_time': '2024-02-23 12:56:43.442784', 'last_time': '2024-02-23 12:56:43.973156', 'Retired': '[0-1]', 'Active': '[0-1]', 'NotIssued': '[2-511]'}
{'pg_id': 0, 'seq_id': 19, 'profiling_name': 'nccl:recv 0<-1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_1', 'first_time': '2024-02-23 12:56:43.488193', 'last_time': '2024-02-23 12:56:43.973361', 'Retired': '[0-1]', 'Active': '[0-1]', 'NotIssued': '[2-511]'}
{'pg_id': 0, 'seq_id': 20, 'profiling_name': 'nccl:recv 0<-1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_1', 'first_time': '2024-02-23 12:56:43.512269', 'last_time': '2024-02-23 12:56:43.974173', 'Retired': '[0-1]', 'Active': '[0-1]', 'NotIssued': '[2-511]'}
{'pg_id': 0, 'seq_id': 21, 'profiling_name': 'nccl:send 0->1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_0', 'first_time': '2024-02-23 12:56:43.574627', 'last_time': '2024-02-23 12:56:44.235678', 'Retired': '[1]', 'Active': '[0,0-1]', 'NotIssued': '[2-511]'}
{'pg_id': 0, 'seq_id': 22, 'profiling_name': 'nccl:send 0->1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_0', 'first_time': '2024-02-23 12:56:43.576240', 'last_time': '2024-02-23 12:56:44.237066', 'Retired': '[1]', 'Active': '[0,0-1]', 'NotIssued': '[2-511]'}
{'pg_id': 0, 'seq_id': 23, 'profiling_name': 'nccl:recv 0<-1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_1', 'first_time': '2024-02-23 12:56:43.618206', 'last_time': '2024-02-23 12:56:44.237301', 'Retired': '[]', 'Active': '[0,0-1,1]', 'NotIssued': '[2-511]'}
{'pg_id': 0, 'seq_id': 24, 'profiling_name': 'nccl:recv 0<-1', 'input_sizes': [[4, 2048, 256]], 'output_sizes': [[4, 2048, 256]], 'state': 'scheduled', 'time_discovered_started_ns': None, 'time_discovered_completed_ns': None, 'frames': 'stack_1', 'first_time': '2024-02-23 12:56:43.642334', 'last_time': '2024-02-23 12:56:44.238177', 'Retired': '[]', 'Active': '[0,0-1,1]', 'NotIssued': '[2-511]'}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment