-
Notifications
You must be signed in to change notification settings - Fork 33
/
Copy pathlog-1019.txt
126 lines (119 loc) · 7.7 KB
/
log-1019.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
/home/leucocyte/pipeswitch/opensource/scripts/config/servers.txt
aws-pipeswitch-opensource> Warm up the server
07d07a0682dab11e055f5b73ef1ed3e8c68cc547c8770f57163d943c337e8837
Training 0 1634681160.9922123 8.037392616271973
Training 1 1634681161.1017268 0.0
Training 2 1634681161.2049365 0.0
Training 3 1634681161.3068137 0.0
Training 4 1634681161.4061377 0.0
Training 5 1634681161.5070531 0.0
Training 6 1634681161.6065385 0.0
Training 7 1634681161.7066014 0.0
Training 8 1634681161.80813 0.0
Training 9 1634681161.909483 0.0
MainThread resnet152 inference >>>>>>>>>> 1634681163.6701558 model status False
resnet152 inference func cost 0.02515411376953125
MainThread inception_v3 inference >>>>>>>>>> 1634681165.8527405 model status False
inception_v3 inference func cost 0.020576953887939453
MainThread bert base inference >>>>>>>>>> 1634681168.8121045 model status False
bert_base inf costs cost 0.038677215576171875
Using cache found in /root/.cache/torch/hub/pytorch_vision_v0.4.2
Using cache found in /root/.cache/torch/hub/pytorch_vision_v0.4.2
Using cache found in /root/.cache/torch/hub/pytorch_vision_v0.4.2
Using cache found in /root/.cache/torch/hub/huggingface_pytorch-transformers_v2.5.0
pipeswitch-dev
aws-pipeswitch-opensource> Complete warming up the server
Complete warming up servers
Plot
Plot figure 5: ready_model, resnet152
Plot figure 5: ready_model, inception_v3
Plot figure 5: ready_model, bert_base
Plot figure 5: pipeswitch, resnet152
Plot figure 5: pipeswitch, inception_v3
Plot figure 5: pipeswitch, bert_base
Plot figure 5: mps, resnet152
Plot figure 5: mps, inception_v3
Plot figure 5: mps, bert_base
Plot figure 5: kill_restart, resnet152
Plot figure 5: kill_restart, inception_v3
Plot figure 5: kill_restart, bert_base
{'ready_model': {'resnet152': 37.06399069892036, 'inception_v3': 40.53577052222358, 'bert_base': 39.33258851369222}, 'pipeswitch': {'resnet152': 43.453288078308105, 'inception_v3': 40.59109687805176, 'bert_base': 53.84514331817627}, 'mps': {'resnet152': 659.041953086853, 'inception_v3': 464.2939805984497, 'bert_base': 522.1973180770874}, 'kill_restart': {'resnet152': 5678.1941175460815, 'inception_v3': 6276.422929763794, 'bert_base': 7243.096494674683}}
Complete ploting
Plot
Plot figure 6: ready_model, 1s
Plot figure 6: ready_model, 2s
Plot figure 6: ready_model, 5s
Plot figure 6: ready_model, 10s
Plot figure 6: ready_model, 30s
Plot figure 6: pipeswitch, 1s
Plot figure 6: pipeswitch, 2s
Plot figure 6: pipeswitch, 5s
Plot figure 6: pipeswitch, 10s
Plot figure 6: pipeswitch, 30s
Plot figure 6: mps, 1s
Plot figure 6: mps, 2s
Plot figure 6: mps, 5s
Plot figure 6: mps, 10s
Plot figure 6: mps, 30s
Plot figure 6: kill_restart, 1s
Plot figure 6: kill_restart, 2s
Plot figure 6: kill_restart, 5s
Plot figure 6: kill_restart, 10s
Plot figure 6: kill_restart, 30s
{'ready_model': {'1s': [26.1, 38.28189473499283, 37.74905204772949, 39.689064025878906], '2s': [27.05, 37.07010864991137, 36.14306449890137, 38.82408142089844], '5s': [25.939999999999998, 38.45897415737602, 37.71662712097168, 40.604352951049805], '10s': [26.99, 36.89031805891423, 35.5982780456543, 38.64550590515137], '30s': [26.36, 37.70939249792181, 34.86943244934082, 40.769338607788086]}, 'pipeswitch': {'1s': [26.75, 37.840731790132615, 36.46254539489746, 49.21555519104004], '2s': [27.125, 36.776533873949184, 35.54129600524902, 47.27005958557129], '5s': [26.4, 37.776902769551135, 35.86697578430176, 51.035165786743164], '10s': [26.55, 37.48260379511084, 36.21268272399902, 51.644086837768555], '30s': [26.233333333333334, 37.85841483173104, 36.667823791503906, 55.48214912414551]}, 'mps': {'1s': [3.5, 356.38771738324846, 104.40444946289062, 659.81125831604], '2s': [2.375, 444.172181581196, 104.42829132080078, 1111.7923259735107], '5s': [3.1, 334.7940098854803, 72.96514511108398, 832.0176601409912], '10s': [2.3, 446.7857998350392, 97.72062301635742, 1411.1425876617432], '30s': [3.0, 335.3376858764225, 38.22636604309082, 975.4421710968018]}, 'kill_restart': {'1s': [0.0, 5521.063446998596, 5495.598316192627, 5572.285890579224], '2s': [0.0, 5540.8729910850525, 5505.0318241119385, 5570.749759674072], '5s': [0.0, 5632.104158401489, 5610.085010528564, 5653.781175613403], '10s': [11.35, 87.96688113443652, 34.96861457824707, 5720.613956451416], '30s': [20.833333333333332, 47.77904901504517, 36.18001937866211, 5707.695007324219]}}
findfont: Font family ['Arial'] not found. Falling back to DejaVu Sans.
findfont: Font family ['Arial'] not found. Falling back to DejaVu Sans.
Complete ploting
Plot
Plot figure 7: pipeswitch, resnet152
Plot figure 7: pipeswitch, inception_v3
Plot figure 7: pipeswitch, bert_base
Plot figure 7: per_layer, resnet152
Plot figure 7: per_layer, inception_v3
Plot figure 7: per_layer, bert_base
Plot figure 7: grouped, resnet152
Plot figure 7: grouped, inception_v3
Plot figure 7: grouped, bert_base
Plot figure 7: per_layer_no_pipeline, resnet152
Plot figure 7: per_layer_no_pipeline, inception_v3
Plot figure 7: per_layer_no_pipeline, bert_base
{'pipeswitch': {'resnet152': 41.063785552978516, 'inception_v3': 39.508891105651855, 'bert_base': 53.6515474319458}, 'per_layer': {'resnet152': 76.35154724121094, 'inception_v3': 53.65896224975586, 'bert_base': 70.32392024993896}, 'grouped': {'resnet152': 60.05537509918213, 'inception_v3': 50.380802154541016, 'bert_base': 83.16259384155273}, 'per_layer_no_pipeline': {'resnet152': 86.45176887512207, 'inception_v3': 58.101725578308105, 'bert_base': 89.94078636169434}}
findfont: Font family ['Arial'] not found. Falling back to DejaVu Sans.
findfont: Font family ['Arial'] not found. Falling back to DejaVu Sans.
Complete ploting
Plot
Plot figure 8: pipeswitch, resnet152
Plot figure 8: pipeswitch, inception_v3
Plot figure 8: pipeswitch, bert_base
Plot figure 8: no_memory_management, resnet152
Plot figure 8: no_memory_management, inception_v3
Plot figure 8: no_memory_management, bert_base
Plot figure 8: no_ipc_optimization, resnet152
Plot figure 8: no_ipc_optimization, inception_v3
Plot figure 8: no_ipc_optimization, bert_base
Plot figure 8: no_pin, resnet152
Plot figure 8: no_pin, inception_v3
Plot figure 8: no_pin, bert_base
Plot figure 8: unified_memory, resnet152
Plot figure 8: unified_memory, inception_v3
Plot figure 8: unified_memory, bert_base
{'pipeswitch': {'resnet152': 42.41344928741455, 'inception_v3': 41.23096466064453, 'bert_base': 53.80122661590576}, 'no_memory_management': {'resnet152': 68.59657764434814, 'inception_v3': 62.008047103881836, 'bert_base': 66.4278507232666}, 'no_ipc_optimization': {'resnet152': 95.24672031402588, 'inception_v3': 65.75343608856201, 'bert_base': 88.65382671356201}, 'no_pin': {'resnet152': 55.82225322723389, 'inception_v3': 39.994168281555176, 'bert_base': 67.72537231445312}, 'unified_memory': {'resnet152': 64.13917541503906, 'inception_v3': 60.88089942932129, 'bert_base': 81.15735054016113}}
findfont: Font family ['Arial'] not found. Falling back to DejaVu Sans.
Complete ploting
Plot
Plot figure 9: pipeswitch, resnet152
Plot figure 9: pipeswitch, inception_v3
Plot figure 9: pipeswitch, bert_base
Plot figure 9: stop_next, resnet152
Plot figure 9: stop_next, inception_v3
Plot figure 9: stop_next, bert_base
Plot figure 9: kill_restart, resnet152
Plot figure 9: kill_restart, inception_v3
Plot figure 9: kill_restart, bert_base
{'pipeswitch': {'resnet152': 43.34447383880615, 'inception_v3': 39.57569599151611, 'bert_base': 55.2584171295166}, 'stop_next': {'resnet152': 224.55966472625732, 'inception_v3': 189.55755233764648, 'bert_base': 221.16706371307373}, 'kill_restart': {'resnet152': 5681.509685516357, 'inception_v3': 6279.901885986328, 'bert_base': 7377.840685844421}}
findfont: Font family ['Arial'] not found. Falling back to DejaVu Sans.
findfont: Font family ['Arial'] not found. Falling back to DejaVu Sans.
Complete ploting
real 275m14.629s
user 0m7.791s
sys 0m1.828s