DisCostiC v2.0.0
Distributed Cost in Cluster: a cross-architecture simulator for massively parallel MPI programs
Loading...
Searching...
No Matches
DisCosTiC.cpp
Go to the documentation of this file.
1
7
8#include <iostream>
9#include <list>
10#include <algorithm>
11#include <cmath>
12#include <mpi.h>
13#include <iomanip>
14#include <numeric>
15
23#include <mutex>
24
25//#define USE_VERBOSE
26// #define USE_DEBUG
27#define USE_CHROMEVIZ
28#define USE_ITACVIZ
29// #define USE_DRAWVIZ
30
36
44
45enum time
46{
47 START = 0,
48 END = 1
49};
50
52{
54 MEMORY = 1
55};
56
57extern int bound;
58extern int scaling_cores;
59extern int bytes_to_send;
60
61// Global variables for setting individual system properties
65int node = 0;
69int socket = 0;
77std::string arch_name = "";
78
80{
81 op_copy.time = op_og.time;
82 op_copy.starttime = op_og.starttime;
83 op_copy.syncstart = op_og.syncstart;
84 op_copy.numOpsInQueue = op_og.numOpsInQueue;
85 op_copy.bufSize = op_og.bufSize;
86 op_copy.target = op_og.target;
87 op_copy.mode = op_og.mode;
88 op_copy.rank = op_og.rank;
89 op_copy.label = op_og.label;
90 op_copy.tag = op_og.tag;
91 op_copy.node = op_og.node;
92 op_copy.network = op_og.network;
93 op_copy.type = op_og.type;
94 return op_copy;
95}
96
97// function to convert operation objects computationally relavent
98double *finalize(double a, double b, double c, double d, double e, double arr[5])
99{
100 arr[0] = a;
101 arr[1] = b;
102 arr[2] = c;
103 arr[3] = d;
104 arr[4] = e;
105
106 return arr;
107}
108
109static std::set<int> ncm_overlap_ranks(AST *ast, int comm_kind, int system_number,
110 int local_node, int local_socket, int peer_node)
111{
112 std::set<int> empty;
113 if (comm_kind == INTERCLUSTER)
114 {
115 auto it = ast->interClusterRanks.find("intercluster");
116 return it == ast->interClusterRanks.end() ? empty : it->second;
117 }
118 auto sys = ast->intraAndInterNodeRanks.find("system" + std::to_string(system_number));
119 if (sys == ast->intraAndInterNodeRanks.end())
120 return empty;
121 auto lookup = [&](const std::string &key) -> std::set<int> {
122 auto it = sys->second.find(key);
123 return it == sys->second.end() ? empty : it->second;
124 };
125 if (comm_kind == INTERNODE)
126 {
127 auto fwd = lookup("internode" + std::to_string(local_node) + std::to_string(peer_node));
128 if (sys->second.count("internode" + std::to_string(local_node) + std::to_string(peer_node)))
129 return fwd;
130 return lookup("internode" + std::to_string(peer_node) + std::to_string(local_node));
131 }
132 if (comm_kind == INTERCHIP)
133 return lookup("intranode" + std::to_string(local_node));
134 return lookup("intrasocket" + std::to_string(local_socket));
135}
136
137int main(int argc, char **argv)
138{
139 if (argc > 1)
140 {
141 if (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "--help") == 0)
142 {
143 help();
144 exit(0);
145 }
146 if (strcmp(argv[1], "-v") == 0 || strcmp(argv[1], "--version") == 0)
147 {
148 version();
149 exit(0);
150 }
151 }
152
154 // Initailize MPI environment //
156 int ierr, process_Rank, size_Of_Cluster, sucesss, N_process_Rank, N_size_Of_Cluster;
157 MPI_Comm newcomm;
158 ierr = MPI_Init(NULL, NULL);
159 ierr = MPI_Comm_rank(MPI_COMM_WORLD, &process_Rank);
160
161 MPI_Comm_split(MPI_COMM_WORLD, (process_Rank) < 1 ? 1 : 2, 1, &newcomm);
162 ierr = MPI_Comm_rank(newcomm, &N_process_Rank);
163 ierr = MPI_Comm_size(newcomm, &N_size_Of_Cluster);
164 ierr = MPI_Comm_rank(MPI_COMM_WORLD, &process_Rank);
165 ierr = MPI_Comm_size(MPI_COMM_WORLD, &size_Of_Cluster);
166 std::queue<int> test;
167 MPI_Status status;
168 MPI_Request request;
169
171 // Initailize config and YAML args //
174
176 DisCosTiC_Datatype collective_mode = CFG_args.getValue<DisCosTiC_Datatype>("collective_mode");
177
178 primary_processes = CFG_args.getValue<DisCosTiC_Datatype>("number_of_processes");
179 secondary_processes = CFG_args.getValue<DisCosTiC_Datatype>("secondary_number_of_processes");
180
181 system_number = N_process_Rank < primary_processes ? 0 : 1;
182
183 virtual_rank = system_number == 0 ? N_process_Rank : N_process_Rank - primary_processes;
184
185 arch_name = system_number == 0 ? CFG_args.getValue<std::string>("micro_architecture") : CFG_args.getValue<std::string>("secondary_micro_architecture");
186
187 //collective mode constraints
188 if (collective_mode == 1) {
189 if (heteregeneous_mode == 1) {
190 if (process_Rank == 0) {
191 std::cerr << "collective_mode=1 requires heteregeneous=0" << std::endl;
192 }
193 MPI_Finalize();
194 exit(1);
195 }
196 if (process_Rank == 0) {
197 std::cout << "collective_mode=1 enabled, forcing 1 timestep execution" << std::endl;
198 }
199 }
200
201 if (heteregeneous_mode == 0)
202 {
203 if (CFG_args.getValue<DisCosTiC_Datatype>("number_of_processes") + 1 != size_Of_Cluster)
204 {
205 std::cout << "Simulator processes != number_of_processes+1" << std::endl;
206 MPI_Finalize();
207 exit(0);
208 }
209 }
210 else
211 {
212 if (CFG_args.getValue<DisCosTiC_Datatype>("number_of_processes") + CFG_args.getValue<DisCosTiC_Datatype>("secondary_number_of_processes") + 1 != size_Of_Cluster)
213 {
214 std::cout << "Simulator processes != number_of_processes+1" << std::endl;
215 MPI_Finalize();
216 exit(0);
217 }
218 }
219
220 UserInterface::YAMLParser YAML_args("nodelevel/machine-files/" + arch_name + ".yml");
221
222 std::string diff_arch_name = arch_name == CFG_args.getValue<std::string>("micro_architecture") ? CFG_args.getValue<std::string>("secondary_micro_architecture") : CFG_args.getValue<std::string>("micro_architecture");
223 UserInterface::YAMLParser YAML_args2("nodelevel/machine-files/" + diff_arch_name + ".yml");
224
226 // Assigning system properties to individual rank //
228
229 task_per_node = system_number == 0 ? CFG_args.getValue<DisCosTiC_Datatype>("task_per_node") : CFG_args.getValue<DisCosTiC_Datatype>("secondary_task_per_node");
231 secondary_task_per_node = CFG_args.getValue<DisCosTiC_Datatype>("secondary_task_per_node");
232
233 node = std::floor(virtual_rank / task_per_node);
237 socket = std::floor(virtual_rank / cores_per_socket);
238 primary_cores_per_socket = system_number == 0 ? YAML_args.cores_per_chip : YAML_args2.cores_per_chip;
239 secondary_cores_per_socket = system_number == 0 ? YAML_args2.cores_per_chip : YAML_args.cores_per_chip;
240
242
243 // sleep(1 * N_process_Rank);
244 // std::cout
245 // << "Actual rank : " << N_process_Rank << std::endl
246 // << "\t Virtual Rank : " << virtual_rank << std::endl
247 // << "\t System Number : " << system_number << std::endl
248 // << "\t task_per_node : " << task_per_node << std::endl
249 // << "\t node : " << node << std::endl
250 // << "\t cores_per_socket : " << cores_per_socket << std::endl
251 // << "\t socket Number : " << socket << std::endl
252 // << "\t cc_numa_domain_per_socket : " << cc_numa_domain_per_socket << std::endl
253 // << "\t cc_numa_domain Number : " << cc_numa_domain << std::endl;
254
256 // Initailize configuration constanqueueOp and binary input environment //
258 DisCosTiC::Benchmark benchmark(&CFG_args);
259 AST *DisCosTiC_AST = new AST(&CFG_args, systemsize);
260
261 DisCosTiC::idNodeTypePair ID = benchmark.Oneway_PositiveDisplacement(DisCosTiC_AST, process_Rank, YAML_args, size_Of_Cluster, MPI_COMM_WORLD);
262
263#ifdef USE_VERBOSE
264 std::cout << "Rank : " << process_Rank << " completed Oneway_PositiveDisplacement()" << std::endl;
265#endif
266
267 DisCosTiC::CompModel compModel(&CFG_args, YAML_args, process_Rank);
268
269 DisCosTiC::Grid_Init AST_args(benchmark.systemsize, benchmark.numOperations, benchmark, ID);
270
271 if (process_Rank == 0)
272 {
273
274 DisCosTiC_AST->printNCMRanks();
275 system(("rm config{0.." + std::to_string(size_Of_Cluster) + "}.cfg 2> /dev/null").c_str());
276 //system(("rm ./nodelevel/configs/" + CFG_args.getValue<std::string>("benchmark_kernel") + "{0.." + std::to_string(size_Of_Cluster) + "}.cfg 2> /dev/null").c_str());
277 }
278
279#ifdef USE_VERBOSE
280 std::cout << "Rank : " << process_Rank << " completed AST generation()" << std::endl;
281#endif
282
283 DisCosTiC::Operations operations_queue;
284
286 // Reading user interface configuration input file //
288
293 UserInterface::NetworkConfigParser IntrachipCFG_args;
294 UserInterface::NetworkConfigParser InterchipCFG_args;
295 UserInterface::NetworkConfigParser InternodeCFG_args;
296 UserInterface::NetworkConfigParser InterclusterCFG_args;
297
298 if (heteregeneous_mode == 0)
299 {
300
301 IntrachipCFG_args.readData("./nodelevel/network-files/" + arch_name + "_intrachip.cfg");
302
303 InterchipCFG_args.readData("./nodelevel/network-files/" + arch_name + "_interchip.cfg");
304
305 InternodeCFG_args.readData("./nodelevel/network-files/" + arch_name + "_internode.cfg");
306
307 // InterclusterCFG_args.readData("./nodelevel/network-files/" + arch_name + "_internode.cfg");
308 }
309 else
310 {
311
312 IntrachipCFG_args.readData("./nodelevel/network-files/" + arch_name + "_waitio_" + CFG_args.getValue<std::string>("waitio_mode") + "_intrachip.cfg");
313
314 InterchipCFG_args.readData("./nodelevel/network-files/" + arch_name + "_waitio_" + CFG_args.getValue<std::string>("waitio_mode") + "_interchip.cfg");
315
316 InternodeCFG_args.readData("./nodelevel/network-files/" + arch_name + "_waitio_" + CFG_args.getValue<std::string>("waitio_mode") + "_internode.cfg");
317
318 InterclusterCFG_args.readData("./nodelevel/network-files/"
319 "waitio_" +
320 CFG_args.getValue<std::string>("waitio_mode") + "_intercluster.cfg");
321 }
322
323 const DisCosTiC_Timetype comm_model = CFG_args.getValue<DisCosTiC_Datatype>("comm_model");
324
325 DisCosTiC::Networktype latency{0, 0, 0, 0};
326
327 DisCosTiC::Networktype inverse_bandwidth{0, 0, 0, 0};
328
329 DisCosTiC::Networktype G{0, 0, 0, 0};
330
331 DisCosTiC::Networktype o{0, 0, 0, 0};
332
333 DisCosTiC::Networktype O{0, 0, 0, 0};
334
335 DisCosTiC::Networktype g{0, 0, 0, 0};
336
337 /*
338 DisCosTiC_Datatype eagerLimit = 0;
339
340 IntrachipCFG_args.setData(1, bytes_to_send, latency, inverse_bandwidth, o, O, g, G, eagerLimit, INTRACHIP);
341 InterchipCFG_args.setData(1, bytes_to_send, latency, inverse_bandwidth, o, O, g, G, eagerLimit, INTERCHIP);
342 InternodeCFG_args.setData(1, bytes_to_send, latency, inverse_bandwidth, o, O, g, G, eagerLimit, INTERNODE);
343 InterclusterCFG_args.setData(1, bytes_to_send, latency, inverse_bandwidth, o, O, g, G, eagerLimit, INTERCLUSTER);
344
345 const DisCosTiC_Datatype nunmberofiter = CFG_args.getValue<DisCosTiC_Datatype>("number_of_timesteps");
346 */
347
348 DisCosTiC_Datatype eagerLimit = 65535; //default 64Kb eager limit, avoiding 0b eager limit
349
350 //use 64KB as default if bytes_to_send is 0
351 int init_msg_size = (bytes_to_send > 0) ? bytes_to_send : 65536;
352
353 IntrachipCFG_args.setData(1, init_msg_size, latency, inverse_bandwidth, o, O, g, G, eagerLimit, INTRACHIP);
354 InterchipCFG_args.setData(1, init_msg_size, latency, inverse_bandwidth, o, O, g, G, eagerLimit, INTERCHIP);
355 InternodeCFG_args.setData(1, init_msg_size, latency, inverse_bandwidth, o, O, g, G, eagerLimit, INTERNODE);
356 InterclusterCFG_args.setData(1, init_msg_size, latency, inverse_bandwidth, o, O, g, G, eagerLimit, INTERCLUSTER);
357
358 DisCosTiC_Datatype nunmberofiter = CFG_args.getValue<DisCosTiC_Datatype>("number_of_timesteps");
359 //if collective_mode is enabled in config.cfg, only 1 timestep is done
360 if (collective_mode == 1) {
361 nunmberofiter = 1;
362 }
363
364 const DisCosTiC_Datatype desync_feature = CFG_args.getValue<DisCosTiC_Datatype>("desync");
365 const DisCosTiC_Datatype idlewave_feature = CFG_args.getValue<DisCosTiC_Datatype>("idlewave");
366 const DisCosTiC_Datatype desync_rank = CFG_args.getValue<DisCosTiC_Datatype>("which_rank");
367 const DisCosTiC_Datatype desync_timestep = CFG_args.getValue<DisCosTiC_Datatype>("at_timestep");
368
369#ifdef USE_VERBOSE
370 std::cout << "Rank : " << process_Rank << " completed network configuration" << std::endl;
371#endif
372
374 // Reading generated AST data //
376 const DisCosTiC_Indextype numRanks = AST_args.graphVec.size();
377
378 const DisCosTiC_Indextype numCores = benchmark.GetNumCores();
379 const DisCosTiC_Indextype numNetworks = benchmark.GetNumNetworks();
380 std::vector<real_t> hosqueueOpync(numRanks);
381
382 if (process_Rank == 0)
383 {
384 std::cout << "\n----------------------------------------------------------------" << std::endl;
385 std::cout << "DisCosTiC (Distributed CosT in Cluster):" << std::endl;
386 std::cout << "----------------------------------------------------------------" << std::endl;
387 std::cout << "Version: v2.0.0" << std::endl;
388 auto timestamp = system_clock::to_time_t(system_clock::now());
389 std::cout << "timestamp: " << ctime(&timestamp);
390 std::cout << "\n\n----------------------------------------------------------------" << std::endl;
391 std::cout << "GENERATED DEPENDENCY GRAPH (Directed Acyclic Benchmark (DisCosTiC)):" << std::endl;
392 std::cout << "----------------------------------------------------------------" << std::endl;
393 std::cout << "number of timesteps: " << nunmberofiter << std::endl;
394 std::cout << "OneWay_PosDist communetworkation pattern: number of Operation_t (send or recv or exec) per timestep per rank: " << 1 << std::endl;
395 std::cout << "\n\n----------------------------------------------------------------" << std::endl;
396 std::cout << "COMMUNICATION PERFORMANCE MODEL (Network+Systemtopology):" << std::endl;
397 std::cout << "----------------------------------------------------------------" << std::endl;
398 std::cout << "number of processes: " << numRanks << std::endl;
399 std::cout << "number of chips: " << numCores << std::endl;
400 std::cout << "SIMPLE LATENCY-BANDWIDTH MODEL:: intra chip latency [ns]: " << latency[INTRACHIP]
401 << ", inter chip latency [ns]: " << latency[INTERCHIP]
402 << ", inter node latency [ns]: " << latency[INTERNODE]
403 << ", inter cluster latency [ns]: " << latency[INTERCLUSTER] << std::endl;
404 std::cout << "SIMPLE LATENCY-BANDWIDTH MODEL:: intra chip inverse bandwidth [s/GB]: " << inverse_bandwidth[INTRACHIP]
405 << ", inter chip inverse bandwidth [s/GB]: " << inverse_bandwidth[INTERCHIP]
406 << ", inter node inverse bandwidth [s/GB]: " << inverse_bandwidth[INTERNODE]
407 << ", inter cluster inverse bandwidth [s/GB]: " << inverse_bandwidth[INTERCLUSTER] << std::endl;
408 std::cout << "LogGP MODEL additional parameters (Intra chip) :: o [ns]: " << o[INTRACHIP] << ", g [ns]: " << g[INTRACHIP] << ", O [ns]: " << O[INTRACHIP] << ", G [ns]: " << G[INTRACHIP] << std::endl;
409 std::cout << "LogGP MODEL additional parameters (Inter chip) :: o [ns]: " << o[INTERCHIP] << ", g [ns]: " << g[INTERCHIP] << ", O [ns]: " << O[INTERCHIP] << ", G [ns]: " << G[INTERCHIP] << std::endl;
410 std::cout << "LogGP MODEL additional parameters (Inter node) :: o [ns]: " << o[INTERNODE] << ", g [ns]: " << g[INTERNODE] << ", O [ns]: " << O[INTERNODE] << ", G [ns]: " << G[INTERNODE] << std::endl;
411 std::cout << "LogGP MODEL additional parameters (Inter cluster) :: o [ns]: " << o[INTERCLUSTER] << ", g [ns]: " << g[INTERCLUSTER] << ", O [ns]: " << O[INTERCLUSTER] << ", G [ns]: " << G[INTERCLUSTER] << std::endl;
412
413 std::cout << "Eager limit: " << eagerLimit << " bytes" << std::endl;
414 std::cout << "num_synchronizations: " << hosqueueOpync.size() << std::endl;
415 }
416
417 MPI_Barrier(MPI_COMM_WORLD);
419 // Initailize user interface (time-rank output) environment and other variables //
421 UserInterface::TimeRankOP traceVisualizer(&CFG_args, N_process_Rank, numRanks);
422 UserInterface::ChromeTraceViz traceVisualizer2(&CFG_args, process_Rank, numRanks);
423#ifdef USE_ITACVIZ
424 UserInterface::ItacVtf traceVisualizerItac(&CFG_args, process_Rank, numRanks);
425#endif
426
428 Timevec2T oSuccessor(numRanks), grecvSuccessor(numRanks), gsendSuccessor(numRanks);
429 for (auto rank : DisCosTiC::getRange(numRanks))
430 {
431 oSuccessor[rank].resize(numCores);
432 grecvSuccessor[rank].resize(numNetworks);
433 gsendSuccessor[rank].resize(numNetworks);
434 std::fill(oSuccessor[rank].begin(), oSuccessor[rank].end(), 0);
435 std::fill(grecvSuccessor[rank].begin(), grecvSuccessor[rank].end(), 0);
436 std::fill(gsendSuccessor[rank].begin(), gsendSuccessor[rank].end(), 0);
437 }
438
439 DisCosTiC_Indextype numOpsInQueue = 0; // ORDER: keep order between comm & comp operations for non-blocking routines
440 DisCosTiC_Indextype rank = 0;
441 DisCosTiC_Indextype DisCosTiCOps = 0;
442 DisCosTiC_Indextype DisCosTiCProgress = 0;
443 DisCosTiC_Indextype computingProcessesCount = 0;
444 DisCosTiC_Timetype COMPCount = 0;
445 DisCosTiC_Timetype timeOld = 0;
446 DisCosTiC_Indextype flow_id = 0;
447
448 bool newOps = true;
449 vec1T finishedRankList;
450 vec1T CompRankList;
451 vec1T new_CompRankList;
454 DisCosTiC::VecListqueueOp recvQueue(numRanks), UMQ(numRanks); // unexpected message queue
455 DisCosTiC_Datatype execTime; // execution run time of computational kernel in micro seconds
456 // std::vector<DisCosTiC_Timetype> buffersize = {3000 , 3200 , 3400 , 4100 , 3700 , 5900 , 5000 , 6000 , 3800 , 3560, 3789 , 3500 , 3450 , 3210 , 3100 , 2300 , 3000 , 4000, 3100 , 2300};
457 static bool initialized;
458
460 // /** \brief Starting Time */ //
462 high_resolution_clock::time_point startTime, endTime;
463 startTime = high_resolution_clock::now();
464 DisCosTiC_Timetype starttimeProcess;
465 srand(time(NULL) + process_Rank + 5);
466
474 DisCosTiC::DisCosTiC_OP operation;
475 DisCosTiC::DisCosTiC_OP operation_og;
476 int iter = 0;
477
478
479 //storage for collective data
480 double collective_data_storage[72][2]; //max 72 ranks, 1 fritz node
481 uint8_t collective_algorithm = 0; //algorithm used for collective
482 DisCosTiC::CollectiveType collective_type = DisCosTiC::COLL_BCAST; //collective type
483
488 if (process_Rank != 0)
489 {
490 for (auto rank : DisCosTiC::getRange(numRanks))
491 { // loop over rank
492 DisCosTiC::Grid *rankLocalIt = &AST_args.graphVec[rank]; // keep iterating over the element till you find all rank and reach the end of AST_args.graphVec vector
493 DisCosTiC::Operations operations;
494 DisCosTiCOps += rankLocalIt->getNumOps(); // rankLocalIt->getNumOps()=15 (x18) , rankLocalIt->num_ranks_in_schedule=18 (x18) DisCosTiCOps: 15, 30, 45, ..., 270 (x18)
495 rankLocalIt->getSortedRootOps(operations);
496
497 iter = nunmberofiter;
498 for (auto &op : operations)
499 {
500 if (rank == N_process_Rank)
501 {
502 operation_og = copy(op, operation_og);
503 operation = op;
504 starttimeProcess = operation.starttime;
505 if (op.bufSize == 0) {
506 std::cerr << "[WARNING] Rank " << N_process_Rank << " has op with zero bufSize. Type: " << +op.type << std::endl;
508 }
509
510 if (op.bufSize != operation_og.bufSize) {
511 std::cerr << "[MISMATCH] Rank " << N_process_Rank
512 << " bufSize mismatch: op.bufSize = " << op.bufSize
513 << ", operation_og.bufSize = " << operation_og.bufSize << std::endl;
514 }
515 }
516#ifdef USE_VERBOSE
517 switch (op.type)
518 {
521 verboseCompInitPrint(rank, op);
522 break;
524 verboseSendInitPrint(rank, op);
525 break;
527 verboseRecvInitPrint(rank, op);
528 break;
530 //new collective operation
531 break;
532 default:
533 std::cerr << "YET NOT IMPLEMENTED!" << std::endl;
534 }
535#endif
536 execTime = op.bufSize;
537 }
538 }
539
540 MPI_Barrier(newcomm);
541
542#ifdef USE_VERBOSE
543 std::cout << "Rank : " << N_process_Rank << " starting simulation" << std::endl;
544#endif
545
551 rank = N_process_Rank;
552
553 // int active_task_per_node = (node == N_size_Of_Cluster / task_per_node ? N_size_Of_Cluster % task_per_node : task_per_node);
554
555 int commTime = 0;
556 int flag_blocking = 0;
557 int flag_blocking_2 = 0;
558 int flag_blocking_3 = 0;
559 int flag_blocking_4 = 0;
560
561 int flag_blocking_continuous = 0;
562
563 double nonoverlap = 0;
564 // std::cout << "Rank : " << rank << " node : " << node << " scaling cores : " << scaling_cores << " active_task_per_node : " << active_task_per_node << std::endl;
565
566 bool cycle = true;
567 int iter_num = 0;
568
569 double gather_buff_comp[2][N_size_Of_Cluster];
570 double gather_buff_optime[N_size_Of_Cluster];
571 double overlap_time[N_size_Of_Cluster];
572 long int last_idle_time = 0, max_idle_time = 0;
573
574 int flag_recv_5 = 0, flag_check = 0, barrier_flag = 0;
575
576 for (int counter = 0; counter < N_size_Of_Cluster; ++counter)
577 {
578 gather_buff_optime[counter] = 0.0;
579 }
580
581#ifdef USE_VERBOSE
582 std::cout << "Rank : " << N_process_Rank << " properties allocated" << std::endl;
583#endif
584
586 // Simulation //
588 do
589 { // Cycle for the number of iterations
590 do
591 { // Cycle for the number of steps per iteration
592
593 if ((timeOld - operation.time) != 0) //}// later: coresPerChip-1
594 {
596 COMPCount = computingProcessesCount;
597
598 new_CompRankList = CompRankList;
599 // print_vec1T(new_CompRankList);
600
601 timeOld = operation.time;
602 }
603
604 // print_vec1T(new_CompRankList);
605
606 // Grow with COMPCount (dual-kernel DSELs exceed the old fixed 2000).
607 const std::size_t buffersize_n =
608 std::max<std::size_t>(2000, static_cast<std::size_t>(COMPCount) + 1);
609 std::vector<DisCosTiC_Timetype> buffersize(buffersize_n, execTime);
610
611 auto it = std::find(new_CompRankList.begin(), new_CompRankList.end(), operation.rank);
612
613 if ((operation.time != 0) && (it != new_CompRankList.end()))
614 {
615 if (COMPCount == 0)
616 {
617 operation.time = operation.time - execTime;
618 }
619 else
620 {
621 operation.time = operation.time - execTime + buffersize[COMPCount - 1];
622 }
623 }
624
625 MPI_Allgather(&operation.time, 1, MPI_DOUBLE, gather_buff_optime, 1, MPI_DOUBLE, newcomm);
626#ifdef USE_VERBOSE
627 std::cout << "Rank : " << N_process_Rank << " in iter : " << iter_num << " MPI_Allgather completed" << std::endl;
628#endif
629
630 //std::cout << "Rank: " << operation.rank << " Before COMP: " << operation.time << " COMPCount: " << COMPCount << " Buffersize: " << buffersize[COMPCount-1] << std::endl;
631 switch (operation.type)
632 {
633
636 {
637
638#ifdef USE_VERBOSE
639 std::cout << "Rank : " << N_process_Rank << " in iter : " << iter_num << " in COMP" << std::endl;
640 verboseCompPrint(operation);
641#endif
642 flag_blocking_continuous = 0;
643 barrier_flag = 0;
644
645 computingProcessesCount++;
646 CompRankList.push_back(operation.rank);
647 if (oSuccessor[operation.rank][operation.node] <= operation.time)
648 {
649
650 oSuccessor[operation.rank][operation.node] = operation.time + operation.bufSize;
651
652 // Gathering data for Bandwidth contention model
653 last_idle_time = operation.time - last_idle_time;
654
655 MPI_Allgather(&operation.time, 1, MPI_DOUBLE, gather_buff_comp[0], 1, MPI_DOUBLE, newcomm);
656 MPI_Allgather(&oSuccessor[operation.rank][operation.node], 1, MPI_DOUBLE, gather_buff_comp[1], 1, MPI_DOUBLE, newcomm);
657
658#ifdef USE_VERBOSE
659 std::cout << "Rank : " << N_process_Rank << " in iter : " << iter_num << " gathered data for BCM" << std::endl;
660#endif
661
662 // Logic to handle overlapping and memory bandwidth
663
664 int startComputingProcess = 0, endComputingProcess = 0, overlappComputingProcess = 0, fasterContentionModel = 0;
665
666 //To handle this for two kernels first kernel should be compute and second should be memory
667 //Make the below if condition as if ((bound == MEMORY) && scaling_cores != -1 && operation.type == DisCosTiC::Operation_t::COMPADD)
668
669 if ((bound == MEMORY) && scaling_cores != -1)
670 {
671 double nonoverlap_penalty_1 = (arch_name.compare("A64FX") == 0) ? 0.1 : (heteregeneous_mode == 1 ? 0.02 : 1.0 / cc_numa_domain_per_socket);
672 double overlap_penalty_2 = (arch_name.compare("A64FX") == 0) ? 0.1 : 0.01;
673 // std::cout << "Timestep : " << iter_num << " "<<operation.bufSize <<std::endl;
674 overlap_time[rank] = operation.bufSize;
675
676 double projected_comp_time = (operation.bufSize) / scaling_cores;
677 // if (virtual_rank == 1)
678 // std::cout << "New projected time : " << projected_comp_time << " old projected time : " << (gather_buff_comp[END][rank] - gather_buff_comp[START][rank]) << std::endl;
679
680 for (int loop = 0; loop < N_size_Of_Cluster && (arch_name.compare("A64FX") == 0); ++loop)
681 {
682
683 int loop_system = loop < primary_processes ? 0 : 1;
684
685 int loop_node = loop < primary_processes ? std::floor(loop / primary_task_per_node) : std::floor((loop - primary_processes) / secondary_task_per_node);
686 int loop_cc_numa_domain = loop < primary_processes ? std::floor(loop / cc_numa_domain_per_socket) : std::floor((loop - primary_processes) / cc_numa_domain_per_socket);
687 int loop_socket = loop < primary_processes ? std::floor(loop / primary_cores_per_socket) : std::floor((loop - primary_processes) / secondary_cores_per_socket);
688
689 if (loop != rank && (loop_node == node) && (loop_cc_numa_domain == cc_numa_domain) && (loop_socket == socket) && (loop_system == system_number))
690 {
691 if ((gather_buff_comp[START][rank] <= gather_buff_comp[START][loop] && (gather_buff_comp[START][rank] + projected_comp_time) >= gather_buff_comp[START][loop]) || (gather_buff_comp[START][rank] <= (gather_buff_comp[END][loop] + projected_comp_time) && (gather_buff_comp[START][rank] + projected_comp_time) >= (gather_buff_comp[END][loop] + projected_comp_time)))
692 {
693 // if (gather_buff_comp[START][rank] <= gather_buff_comp[START][loop] && (gather_buff_comp[START][rank] + projected_comp_time) >= gather_buff_comp[START][loop])
694 // {
695 // overlappComputingProcess++;
696 // }
697 // else
698 if (gather_buff_comp[START][rank] <= (gather_buff_comp[START][loop] + projected_comp_time) && (gather_buff_comp[START][rank] + projected_comp_time) >= (gather_buff_comp[START][loop] + projected_comp_time))
699 {
700 overlappComputingProcess++;
701 }
702 }
703 }
704 }
705
706 if ((overlappComputingProcess <= scaling_cores) && (arch_name.compare("A64FX") == 0))
707 {
708 fasterContentionModel = 1;
709 operation.bufSize = projected_comp_time;
710 }
711
712 for (int loop = 0; loop < N_size_Of_Cluster && (fasterContentionModel == 0); ++loop)
713 {
714
715 int loop_system = loop < primary_processes ? 0 : 1;
716
717 int loop_node = loop < primary_processes ? std::floor(loop / primary_task_per_node) : std::floor((loop - primary_processes) / secondary_task_per_node);
718 int loop_cc_numa_domain = loop < primary_processes ? std::floor(loop / cc_numa_domain_per_socket) : std::floor((loop - primary_processes) / cc_numa_domain_per_socket);
719 int loop_socket = loop < primary_processes ? std::floor(loop / primary_cores_per_socket) : std::floor((loop - primary_processes) / secondary_cores_per_socket);
720
721 if (loop != rank && (loop_node == node) && (loop_cc_numa_domain == cc_numa_domain) && (loop_socket == socket) && (loop_system == system_number))
722 {
723 if ((gather_buff_comp[START][rank] <= gather_buff_comp[START][loop] && gather_buff_comp[END][rank] >= gather_buff_comp[START][loop]) || (gather_buff_comp[START][rank] <= gather_buff_comp[END][loop] && gather_buff_comp[END][rank] >= gather_buff_comp[END][loop]))
724 {
725 if (gather_buff_comp[START][rank] <= gather_buff_comp[START][loop] && gather_buff_comp[END][rank] >= gather_buff_comp[START][loop])
726 {
727 overlap_time[loop] = gather_buff_comp[END][rank] - gather_buff_comp[START][loop] < 0 ? 0 : gather_buff_comp[END][rank] - gather_buff_comp[START][loop];
728 }
729 else if (gather_buff_comp[START][rank] <= gather_buff_comp[END][loop] && gather_buff_comp[END][rank] >= gather_buff_comp[END][loop])
730 {
731 overlap_time[loop] = gather_buff_comp[END][loop] - gather_buff_comp[START][rank] < 0 ? 0 : gather_buff_comp[END][loop] - gather_buff_comp[START][rank];
732 }
733
734 // std::cout << "Rank "<<rank<<" overlapp with Rank " << loop << " is " << overlap_time[loop] / 1e9 << std::endl;
735 nonoverlap = (overlap_time[rank] - overlap_time[loop] < 50000 ? 0 : overlap_time[rank] - overlap_time[loop]);
736 // std::cout << "Rank "<<rank<<" non overlapp with Rank " << loop << " is " << nonoverlap/1e9 << std::endl;
737 operation.bufSize -= nonoverlap * nonoverlap_penalty_1;
738 }
739 else
740 {
741
742 int flag = 0;
743 double loop_computing_time = gather_buff_comp[END][loop] - gather_buff_comp[START][loop];
744 int overlapp_pieces = last_idle_time / loop_computing_time;
745
746 double new_loop_start_time = gather_buff_comp[START][loop];
747 double new_loop_end_time = gather_buff_comp[END][loop];
748 if (gather_buff_comp[START][loop] < gather_buff_comp[START][rank]) // rank in future, loop in past
749 {
750
751 if ((abs(rank - loop) % task_per_node > overlapp_pieces) && last_idle_time > max_idle_time)
752 {
753
754 for (int k = 1; k <= overlapp_pieces; ++k)
755 {
756 double temp = new_loop_start_time;
757 new_loop_start_time = new_loop_end_time;
758 new_loop_end_time += ((new_loop_end_time - temp) * 0.95);
759 }
760
761 new_loop_start_time += overlapp_pieces + 1 * commTime;
762 new_loop_end_time += overlapp_pieces + 1 * commTime;
763 }
764 else
765 {
766 new_loop_start_time = gather_buff_comp[START][loop] + (((int(ceil(abs(loop - rank))) % task_per_node)) * last_idle_time) + (((abs(loop - rank) % task_per_node)) * commTime);
767 new_loop_end_time = gather_buff_comp[END][loop] + (((int(ceil(abs(loop - rank))) % task_per_node)) * last_idle_time) + (((abs(loop - rank) % task_per_node)) * commTime);
768 }
769
770 if ((gather_buff_comp[START][rank] <= new_loop_start_time && gather_buff_comp[END][rank] >= new_loop_start_time) || (gather_buff_comp[START][rank] <= new_loop_end_time && gather_buff_comp[END][rank] >= new_loop_end_time))
771 {
772 if (gather_buff_comp[START][rank] <= new_loop_start_time && gather_buff_comp[END][rank] >= new_loop_start_time)
773 {
774 overlap_time[loop] = gather_buff_comp[END][rank] - new_loop_start_time < 0 ? 0 : gather_buff_comp[END][rank] - new_loop_start_time;
775 }
776 else if (gather_buff_comp[START][rank] <= new_loop_end_time && gather_buff_comp[END][rank] >= new_loop_end_time)
777 {
778 overlap_time[loop] = new_loop_end_time - gather_buff_comp[START][rank] < 0 ? 0 : new_loop_end_time - gather_buff_comp[START][rank];
779 }
780
781 nonoverlap = (overlap_time[rank] - overlap_time[loop] < 50000 ? 0 : overlap_time[rank] - overlap_time[loop]);
782
783 operation.bufSize -= nonoverlap * overlap_penalty_2;
784 }
785 else
786 {
787 operation.bufSize -= operation.bufSize * nonoverlap_penalty_1;
788 }
789 }
790 else if (gather_buff_comp[START][loop] > gather_buff_comp[START][rank]) // rank in past, loop in future
791 {
792
793 if ((abs(rank - loop) % task_per_node > overlapp_pieces) && last_idle_time > max_idle_time)
794 {
795
796 for (int k = 1; k <= overlapp_pieces; ++k)
797 {
798 double temp = new_loop_start_time;
799 new_loop_start_time = new_loop_end_time;
800 new_loop_end_time += ((new_loop_end_time - temp) * 0.95);
801 }
802
803 new_loop_start_time += overlapp_pieces + 1 * commTime;
804 new_loop_end_time += overlapp_pieces + 1 * commTime;
805 }
806 else
807 {
808 new_loop_start_time = gather_buff_comp[START][loop] - (((int(ceil(abs(loop - rank))) % task_per_node)) * last_idle_time) - (((abs(loop - rank) % task_per_node)) * commTime);
809 new_loop_end_time = gather_buff_comp[END][loop] - (((int(ceil(abs(loop - rank))) % task_per_node)) * last_idle_time) - (((abs(loop - rank) % task_per_node)) * commTime);
810 }
811
812 if ((gather_buff_comp[START][rank] <= new_loop_start_time && gather_buff_comp[END][rank] >= new_loop_start_time) || (gather_buff_comp[START][rank] <= new_loop_end_time && gather_buff_comp[END][rank] >= new_loop_end_time))
813 {
814 if (gather_buff_comp[START][rank] <= new_loop_start_time && gather_buff_comp[END][rank] >= new_loop_start_time)
815 {
816 overlap_time[loop] = gather_buff_comp[END][rank] - new_loop_start_time < 0 ? 0 : gather_buff_comp[END][rank] - new_loop_start_time;
817 }
818 else if (gather_buff_comp[START][rank] <= new_loop_end_time && gather_buff_comp[END][rank] >= new_loop_end_time)
819 {
820 overlap_time[loop] = new_loop_end_time - gather_buff_comp[START][rank] < 0 ? 0 : new_loop_end_time - gather_buff_comp[START][rank];
821 }
822
823 nonoverlap = (overlap_time[rank] - overlap_time[loop] < 50000 ? 0 : overlap_time[rank] - overlap_time[loop]);
824
825 operation.bufSize -= nonoverlap * overlap_penalty_2;
826 }
827 else
828 {
829 operation.bufSize -= operation.bufSize * nonoverlap_penalty_1;
830 }
831 }
832 }
833 }
834 }
835 }
836
837#ifdef USE_VERBOSE
838 std::cout << "Rank : " << N_process_Rank << " in iter : " << iter_num << " finished BCM" << std::endl;
839#endif
840
841 oSuccessor[operation.rank][operation.node] = operation.time + operation.bufSize;
842
844 if (!initialized && (operation.rank == desync_rank) && (iter_num == desync_timestep) && (idlewave_feature == 1))
845 {
846
847 initialized = true;
848 oSuccessor[operation.rank][operation.node] = operation.time + operation.bufSize + 25 * operation.bufSize;
849
850#ifdef USE_VERBOSE
851 std::cout << "Rank : " << N_process_Rank << " in iter : " << iter_num << " injected delay to create idle wave" << std::endl;
852#endif
853 }
854
855 AST_args.graphVec[operation.rank].setOp(operation.rank, operation.label, benchmark, ID);
856
857 AST_args.graphVec[operation.rank].unsetOp(operation.rank, operation.label, benchmark, ID);
858
859#ifdef USE_VERBOSE
860 std::cout << "Rank : " << N_process_Rank << " in iter : " << iter_num << " AST graph set and unSet operation done" << std::endl;
861#endif
862 finishedRankList.push_back(operation.rank);
863 operation.numOpsInQueue = numOpsInQueue++;
864
865 std::string eventName;
866 std::string eventColor;
867 if (operation.type == DisCosTiC::Operation_t::COMP)
868 {
869 eventName = CFG_args.getValue<std::string>("part1");
870 eventColor = "cq_build_failed";
871 }
872 else
873 {
874 eventName = CFG_args.getValue<std::string>("part2");
875 eventColor = "cq_build_running";
876 }
877#ifdef USE_CHROMEVIZ
878 // traceVisualizer2.comp(VIS_HOST_DUR, new HostDurationVisEvent("Name:COMP", operation.rank, ("CPU" + std::to_string(operation.node)), operation.time,operation.time+operation.bufSize));
879 traceVisualizer2.args("CPU" + std::to_string(operation.node), operation.rank);
880 traceVisualizer2.completeEvents(eventName, operation.rank, traceVisualizer2.arc[("CPU" + std::to_string(operation.node))], std::to_string(operation.time / 1000), std::to_string(((operation.time + operation.bufSize) - operation.time) / 1000), eventColor);
881#endif
882#ifdef USE_ITACVIZ
883 traceVisualizerItac.comp(operation.rank, operation.time, operation.bufSize, eventName);
884#endif
885
886#ifdef USE_DRAWVIZ
887 traceVisualizer.comp(operation.rank, operation.time, operation.time + operation.bufSize, operation.node);
888#endif
889 }
890 commTime = 0;
891 max_idle_time = std::max(max_idle_time, last_idle_time);
892 last_idle_time = operation.time;
893
894#ifdef USE_VERBOSE
895 std::cout << "Rank : " << N_process_Rank << " in iter : " << iter_num << " finished inserting data in DMS file and completed COMP" << std::endl;
896#endif
897 }
898 break;
899
901 {
902 // Keep count in sync with CompRankList (two COMPs per dual-kernel step).
903 CompRankList.erase(std::remove(CompRankList.begin(), CompRankList.end(), operation.rank), CompRankList.end());
904 computingProcessesCount = static_cast<DisCosTiC_Indextype>(CompRankList.size());
905
906#ifdef USE_VERBOSE
907 std::cout << "Rank : " << N_process_Rank << " in iter : " << iter_num << " starting SEND" << std::endl;
908 // verboseSendPrint(operation);
909#endif
910 int comm_type = 0;
911
912 if ((barrier_flag == 0) && (CFG_args.getValue<int>("barrier_flag") == 1))
913 {
914 barrier_flag = 1;
915 flag_blocking = 0;
916 double max_time = 0;
917 for (int i = 0; i < N_size_Of_Cluster; ++i)
918 {
919 max_time = std::max<double>(gather_buff_optime[i], max_time);
920 }
921
922 operation.time = max_time;
923 oSuccessor[operation.rank][operation.node] = max_time;
924 gsendSuccessor[operation.rank][operation.network] = max_time;
925 }
926
927 if ((gather_buff_optime[primary_processes - 1] < gather_buff_optime[primary_processes]) && (flag_check == 0))
928 {
929 flag_recv_5 = 1;
930 }
931
932 flag_check = 1;
933
934 if (std::max(oSuccessor[operation.rank][operation.node], gsendSuccessor[operation.rank][operation.network]) <= operation.time && (operation.bufSize != 0))
935 {
936 int target_cluster = operation.target < primary_processes ? 0 : 1;
937 int target_node = operation.target < primary_processes ? std::floor(operation.target / primary_task_per_node) : std::floor((operation.target - primary_processes) / secondary_task_per_node);
938 int target_socket = operation.target < primary_processes ? std::floor(operation.target / primary_cores_per_socket) : std::floor((operation.target - primary_processes) / secondary_cores_per_socket);
939
940 // AST_args.graphVec[operation.rank].setOp(operation.rank, operation.label, benchmark, ID); //(CAN COMMENT)
941
942 std::set<int> overlapp_ranks;
943 int overlapp_count = 1;
944
945 if (target_cluster != system_number)
946 {
947 comm_type = INTERCLUSTER;
948 }
949 else if (target_node != node)
950 {
951 comm_type = INTERNODE;
952 }
953 else if (target_socket != socket)
954 {
955 comm_type = INTERCHIP;
956 }
957 else
958 {
959 comm_type = INTRACHIP;
960 }
961 overlapp_ranks = ncm_overlap_ranks(DisCosTiC_AST, comm_type, system_number, node, socket, target_node);
962
963#ifdef USE_VERBOSE
964 std::cout << "Rank : " << N_process_Rank << " in iter : " << iter_num << " communication properties determined" << std::endl;
965#endif
966 for (const auto &elem : overlapp_ranks)
967 {
968 if (((gather_buff_optime[elem] - 100000000) <= gather_buff_optime[rank] && gather_buff_optime[rank] <= (gather_buff_optime[elem] + 100000000)) && (elem != operation.target))
969 {
970 ++overlapp_count;
971 }
972 }
973#ifdef USE_VERBOSE
974 std::cout << "SEND Rank : " << rank << " iteration : " << iter_num << " overlapp count : " << overlapp_count << std::endl;
975#endif
976
977 IntrachipCFG_args.setData(overlapp_count, operation.bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit, INTRACHIP);
978 InterchipCFG_args.setData(overlapp_count, operation.bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit, INTERCHIP);
979 InternodeCFG_args.setData(overlapp_count, operation.bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit, INTERNODE);
980 InterclusterCFG_args.setData(overlapp_count, operation.bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit, INTERCLUSTER);
981
982 if (operation.bufSize > eagerLimit)
983 {
984#ifdef USE_VERBOSE
985 std::cout << "Rank : " << N_process_Rank << " in iter : " << iter_num << " obeying RENDEZVOUS protocol" << std::endl;
986#endif
987
988 //std::cout << "Before Blocking: " << operation.time << std::endl;
989 if (operation.mode == DisCosTiC::Mode_t::BLOCKING)
990 {
991 double OpTime = ((operation.bufSize) * inverse_bandwidth[comm_type]);
992 if (iter_num == 0 && flag_blocking == 0)
993 {
994
995 flag_blocking = 1;
996 if ((gather_buff_optime[primary_processes - 1] > gather_buff_optime[primary_processes]) && (CFG_args.getValue<int>("barrier_flag") == 1))
997 {
998 flag_blocking = 0;
999 }
1000 operation.time += ((N_size_Of_Cluster - rank - 1) * OpTime);
1001 oSuccessor[operation.rank][operation.node] += ((N_size_Of_Cluster - rank - 1) * OpTime);
1002 gsendSuccessor[operation.rank][operation.network] += ((N_size_Of_Cluster - rank - 1) * OpTime);
1003 }
1004
1005 if ((system_number == 1) && (iter_num != 0) && (target_cluster == system_number))
1006 {
1007 if (barrier_flag == 1 && flag_blocking == 0 && (CFG_args.getValue<int>("barrier_flag") == 1))
1008 {
1009 flag_blocking = 1;
1010 operation.time += ((N_size_Of_Cluster - rank - 1) * OpTime);
1011 oSuccessor[operation.rank][operation.node] += ((N_size_Of_Cluster - rank - 1) * OpTime);
1012 gsendSuccessor[operation.rank][operation.network] += ((N_size_Of_Cluster - rank - 1) * OpTime);
1013 }
1014 else if (gather_buff_optime[rank] < gather_buff_optime[operation.target])
1015 {
1016
1017 operation.time = gather_buff_optime[operation.target] + 2 * OpTime;
1018 oSuccessor[operation.rank][operation.node] = gather_buff_optime[operation.target] + 2 * OpTime;
1019 gsendSuccessor[operation.rank][operation.network] = gather_buff_optime[operation.target] + 2 * OpTime;
1020 }
1021 }
1022 if ((gather_buff_optime[primary_processes - 1] < gather_buff_optime[primary_processes]) && (system_number == 0) && (heteregeneous_mode == 1) && (flag_blocking_3 == 0))
1023 {
1024 if (barrier_flag == 1 && flag_blocking == 0 && (CFG_args.getValue<int>("barrier_flag") == 1))
1025 {
1026 flag_blocking = 1;
1027 gather_buff_optime[primary_processes] = operation.time;
1028 }
1029 if (flag_blocking_2 == 0)
1030 {
1031 flag_blocking_2 = 1;
1032 if (rank == (primary_processes - 1))
1033 {
1034
1035 operation.time = gather_buff_optime[primary_processes] + (((N_size_Of_Cluster - rank - 1) * OpTime) * 0.07);
1036 oSuccessor[operation.rank][operation.node] = gather_buff_optime[primary_processes] + (((N_size_Of_Cluster - rank - 1) * OpTime) * 0.07);
1037 gsendSuccessor[operation.rank][operation.network] = gather_buff_optime[primary_processes] + (((N_size_Of_Cluster - rank - 1) * OpTime) * 0.07);
1038 }
1039 else
1040 {
1041
1042 operation.time = gather_buff_optime[primary_processes] + (((N_size_Of_Cluster - rank - 15 + (primary_processes - virtual_rank)) * OpTime));
1043 oSuccessor[operation.rank][operation.node] = gather_buff_optime[primary_processes] + (((N_size_Of_Cluster - rank - 15 + (primary_processes - virtual_rank)) * OpTime));
1044 gsendSuccessor[operation.rank][operation.network] = gather_buff_optime[primary_processes] + (((N_size_Of_Cluster - rank - 15 + (primary_processes - virtual_rank)) * OpTime));
1045 }
1046 }
1047 else if (flag_blocking_continuous == 0)
1048 {
1049 flag_blocking_continuous = 1;
1050 if ((system_number == 0) && (rank != (primary_processes - 1)))
1051 {
1052 operation.time = gather_buff_optime[primary_processes] + ((1.5 * (primary_processes - rank) + 9) * OpTime);
1053 oSuccessor[operation.rank][operation.node] = gather_buff_optime[primary_processes] + ((1.5 * (primary_processes - rank) + 9) * OpTime);
1054 gsendSuccessor[operation.rank][operation.network] = gather_buff_optime[primary_processes] + ((1.5 * (primary_processes - rank) + 9) * OpTime);
1055 }
1056 else
1057 {
1058 operation.time = gather_buff_optime[primary_processes] + (((primary_processes - rank -1) * OpTime));
1059 oSuccessor[operation.rank][operation.node] = gather_buff_optime[primary_processes] + (((primary_processes - rank - 1) * OpTime));
1060 gsendSuccessor[operation.rank][operation.network] = gather_buff_optime[primary_processes] + (((primary_processes - rank - 1) * OpTime));
1061 }
1062 }
1063 else if ((gather_buff_optime[primary_processes - 2] < gather_buff_optime[primary_processes - 1]) && (rank != (primary_processes - 1)))
1064 {
1065 operation.time = gather_buff_optime[primary_processes - 1] + (((primary_processes - rank + 25) * OpTime));
1066
1067 oSuccessor[operation.rank][operation.node] = gather_buff_optime[primary_processes - 1] + (((primary_processes - rank) * OpTime));
1068
1069 gsendSuccessor[operation.rank][operation.network] = gather_buff_optime[primary_processes - 1] + (((primary_processes - rank) * OpTime));
1070 }
1071 }
1072 if ((gather_buff_optime[primary_processes - 1] > gather_buff_optime[primary_processes]) && (heteregeneous_mode == 1) && (flag_blocking_2 == 0))
1073 {
1074
1075 if ((flag_blocking_3 == 0))
1076 {
1077
1078 flag_blocking_3 = 1;
1079 if ((system_number == 0))
1080 {
1081 if (barrier_flag == 1 && flag_blocking == 0 && (CFG_args.getValue<int>("barrier_flag") == 1))
1082 {
1083 flag_blocking = 1;
1084 if (rank != (primary_processes - 1))
1085 {
1086 operation.time = gather_buff_optime[rank] + ((1.5 * (primary_processes - rank) + 25) * OpTime);
1087 oSuccessor[operation.rank][operation.node] = gather_buff_optime[rank] + ((1.5 * (primary_processes - rank) + 25) * OpTime);
1088 gsendSuccessor[operation.rank][operation.network] = gather_buff_optime[rank] + ((1.5 * (primary_processes - rank) + 25) * OpTime);
1089 }
1090 else
1091 {
1092 operation.time = gather_buff_optime[rank] + (((primary_processes - rank) + 2) * OpTime);
1093 oSuccessor[operation.rank][operation.node] = gather_buff_optime[rank] + (((primary_processes - rank) + 2) * OpTime);
1094 gsendSuccessor[operation.rank][operation.network] = gather_buff_optime[rank] + (((primary_processes - rank) + 2) * OpTime);
1095 }
1096 }
1097 else
1098 {
1099 if (rank != (primary_processes - 1))
1100 {
1101 operation.time = gather_buff_optime[rank] + ((1.5 * (primary_processes - rank) + 1) * OpTime);
1102 oSuccessor[operation.rank][operation.node] = gather_buff_optime[rank] + ((1.5 * (primary_processes - rank) + 1) * OpTime);
1103 gsendSuccessor[operation.rank][operation.network] = gather_buff_optime[rank] + ((1.5 * (primary_processes - rank) + 1) * OpTime);
1104 }
1105 else
1106 {
1107 operation.time = gather_buff_optime[rank];
1108 oSuccessor[operation.rank][operation.node] = gather_buff_optime[rank];
1109 gsendSuccessor[operation.rank][operation.network] = gather_buff_optime[rank];
1110 }
1111 }
1112 }
1113 }
1114 else if ((system_number == 0))
1115 {
1116 if (barrier_flag == 1 && flag_blocking == 0 && (CFG_args.getValue<int>("barrier_flag") == 1))
1117 {
1118 flag_blocking = 1;
1119 if (rank != (primary_processes - 1))
1120 {
1121 operation.time += ((1.5 * (primary_processes - rank) + 7) * OpTime);
1122 oSuccessor[operation.rank][operation.node] += ((1.5 * (primary_processes - rank) + 7) * OpTime);
1123 gsendSuccessor[operation.rank][operation.network] += ((1.5 * (primary_processes - rank) + 7) * OpTime);
1124 }
1125 else
1126 {
1127 operation.time += OpTime;
1128 oSuccessor[operation.rank][operation.node] += OpTime;
1129 gsendSuccessor[operation.rank][operation.network] += OpTime;
1130 }
1131 }
1132 else if (rank != (primary_processes - 1) && (gather_buff_optime[primary_processes - 2] <= gather_buff_optime[primary_processes - 1]))
1133 {
1134 operation.time = gather_buff_optime[primary_processes - 1] + ((1.5 * (primary_processes - rank) + 5) * OpTime);
1135 oSuccessor[operation.rank][operation.node] = gather_buff_optime[primary_processes - 1] + ((1.5 * (primary_processes - rank) + 5) * OpTime);
1136 gsendSuccessor[operation.rank][operation.network] = gather_buff_optime[primary_processes - 1] + ((1.5 * (primary_processes - rank) + 5) * OpTime);
1137 }
1138 }
1139 else if (rank == primary_processes)
1140 {
1141
1142 operation.time = gather_buff_optime[primary_processes - 1];
1143 oSuccessor[operation.rank][operation.node] = gather_buff_optime[primary_processes - 1];
1144 gsendSuccessor[operation.rank][operation.network] = gather_buff_optime[primary_processes - 1];
1145 }
1146 }
1147 }
1148 // else if (operation.mode == DisCosTiC::Mode_t::NONBLOCKING)
1149 // {
1150 // if (operation.time < gather_buff_optime[operation.target])
1151 // {
1152 // operation.time = gather_buff_optime[operation.target] + commTime;
1153 // oSuccessor[operation.rank][operation.node] = gather_buff_optime[operation.target] + commTime;
1154 // gsendSuccessor[operation.rank][operation.network] = gather_buff_optime[operation.target] + commTime;
1155 // }
1156 // }
1157 }
1158
1159 //std::cout << "Before Hockney's Time: " << operation.time << std::endl;
1160
1161 if (comm_model == LOGGP)
1162 {
1163
1164 oSuccessor[operation.rank][operation.node] = operation.time + o[comm_type] + ((operation.bufSize - 1) * O[comm_type]);
1165 gsendSuccessor[operation.rank][operation.network] = operation.time + g[comm_type] + ((operation.bufSize - 1) * G[comm_type]); // TODO: G should be charged in network layer only
1166
1167#ifdef USE_CHROMEVIZ
1168 traceVisualizer2.args("CPU" + std::to_string(operation.node), operation.rank);
1169 traceVisualizer2.completeEvents("SEND", operation.rank, traceVisualizer2.arc[("CPU" + std::to_string(operation.node))], std::to_string(operation.time / 1000), std::to_string(((operation.time + o[comm_type] + (operation.bufSize - 1) * O[comm_type]) - operation.time) / 1000), "terrible");
1170#endif
1171#ifdef USE_ITACVIZ
1172 traceVisualizerItac.send(operation.rank, operation.target, operation.time,
1173 o[comm_type] + ((operation.bufSize - 1) * O[comm_type]),
1174 operation.bufSize, operation.tag);
1175#endif
1176
1177#ifdef USE_DRAWVIZ
1178 traceVisualizer.osend(operation.rank, operation.time, operation.time + o + (operation.bufSize - 1) * O, operation.node);
1179#endif
1180 }
1181 if (comm_model == SIMPLELB)
1182 {
1183 oSuccessor[operation.rank][operation.node] = operation.time + ((0.5 * (operation.bufSize) * inverse_bandwidth[comm_type]) + latency[comm_type]);
1184 gsendSuccessor[operation.rank][operation.network] = operation.time + ((0.5 * (operation.bufSize) * inverse_bandwidth[comm_type]) + latency[comm_type]); // TODO: G should be charged in network layer only
1185
1186#ifdef USE_CHROMEVIZ
1187 traceVisualizer2.args("CPU" + std::to_string(operation.node), operation.rank);
1188 traceVisualizer2.completeEvents("SEND", operation.rank, traceVisualizer2.arc[("CPU" + std::to_string(operation.node))], std::to_string(operation.time / 1000), std::to_string(((0.5 * (operation.bufSize) * inverse_bandwidth[comm_type]) + latency[comm_type]) / 1000), "terrible");
1189#endif
1190#ifdef USE_ITACVIZ
1191 traceVisualizerItac.send(operation.rank, operation.target, operation.time,
1192 (0.5 * (operation.bufSize) * inverse_bandwidth[comm_type]) + latency[comm_type],
1193 operation.bufSize, operation.tag);
1194#endif
1195 //std::cout << "Rank: " << operation.rank << "Bufsize: " << operation.bufSize << "Time: " << operation.time << "Bandwith: " << inverse_bandwidth[comm_type] << "Latency: " << latency[comm_type] << std::endl;
1196#ifdef USE_DRAWVIZ
1197 traceVisualizer.osend(operation.rank, operation.time, operation.time + o + (operation.bufSize - 1) * O, operation.node);
1198#endif
1199 }
1200
1201 commTime += (oSuccessor[operation.rank][operation.node] - operation.time);
1202
1203 starttimeProcess = operation.starttime;
1204
1205 operation.starttime = operation.time;
1206
1207 if (comm_model == LOGGP)
1208 {
1209 operation.time += o[comm_type] + latency[comm_type]; // time of first byte only (bandwidth G will be charged at receiver)
1210 }
1211 if (comm_model == SIMPLELB)
1212 {
1213 operation.time += ((0.5 * (operation.bufSize) * inverse_bandwidth[comm_type]) + latency[comm_type]); // time of first byte only (bandwidth G will be charged at receiver)
1214 }
1215
1216 DisCosTiCOps++;
1217 operation.numOpsInQueue = numOpsInQueue++;
1218 int temp = oSuccessor[operation.rank][operation.node];
1219
1220 double arr[13];
1221
1222 // Serialize after operation information is recieved
1223
1224 arr[0] = operation.time;
1225 arr[1] = operation.starttime;
1226 arr[2] = operation.syncstart;
1227 arr[3] = operation.numOpsInQueue;
1228 arr[4] = operation.bufSize;
1229 arr[5] = operation.target;
1230 arr[6] = operation.rank;
1231 arr[7] = operation.label;
1232 arr[8] = operation.tag;
1233 arr[9] = operation.node;
1234 arr[10] = operation.network;
1235 arr[11] = operation.type;
1236 arr[12] = temp;
1237
1238 request = MPI_REQUEST_NULL;
1239
1240 if (operation.mode == DisCosTiC::Mode_t::NONBLOCKING)
1241 {
1242 // MPI_Send(&arr, 13, MPI_DOUBLE, operation.target, operation.tag, newcomm);
1243 MPI_Isend(&arr, 13, MPI_DOUBLE, operation.target, operation.tag, newcomm, &request);
1244 MPI_Wait(&request, &status);
1245 }
1246 else
1247 {
1248 MPI_Send(&arr, 13, MPI_DOUBLE, operation.target, operation.tag, newcomm);
1249 }
1250#ifdef USE_VERBOSE
1251 std::cout << "Rank : " << N_process_Rank << " in iter : " << iter_num << " serialized data and sent to respective rank" << std::endl;
1252#endif
1253
1254 finishedRankList.push_back(operation.rank);
1255
1256 cycle = true;
1257 }
1258#ifdef USE_VERBOSE
1259 std::cout << "Rank : " << N_process_Rank << " in iter : " << iter_num << " finished SEND" << std::endl;
1260#endif
1261 }
1262 break;
1263
1265 {
1266 if (operation.bufSize != 0)
1267 {
1268
1269#ifdef USE_VERBOSE
1270 std::cout << "Rank : " << N_process_Rank << " in iter : " << iter_num << " starting RECV" << std::endl;
1271 // verboseRecvPrint(operation);
1272#endif
1273
1274 int comm_type = 0;
1275
1276 double old_op_time = oSuccessor[operation.rank][operation.node];
1277 double brr[13] = {0};
1278
1279 // AST_args.graphVec[operation.rank].setOp(operation.rank, operation.label, benchmark, ID); //(CAN COMMENT)
1280
1281 request = MPI_REQUEST_NULL;
1282
1283 if (operation.mode == DisCosTiC::Mode_t::NONBLOCKING)
1284 {
1285 // MPI_Recv(&brr, 13, MPI_DOUBLE, MPI_ANY_SOURCE, operation.tag, newcomm, &status);
1286 MPI_Irecv(&brr, 13, MPI_DOUBLE, MPI_ANY_SOURCE, operation.tag, newcomm, &request);
1287 MPI_Wait(&request, &status);
1288 }
1289 else
1290 {
1291
1292 // MPI_Irecv(&brr, 13, MPI_DOUBLE, MPI_ANY_SOURCE, operation.tag, newcomm, &request);
1293
1294 MPI_Recv(&brr, 13, MPI_DOUBLE, MPI_ANY_SOURCE, operation.tag, newcomm, &status);
1295 }
1296 // Deserialize after operation information is recieved
1297
1298 operation.time = std::max(operation.time, brr[0]);
1299 operation.starttime = brr[1];
1300 operation.syncstart = brr[2];
1301 operation.numOpsInQueue = brr[3];
1302 operation.bufSize = brr[4];
1303 operation.target = brr[5];
1304 // operation.rank=brr[6];
1305 operation.label = brr[7];
1306 operation.tag = brr[8];
1307 operation.node = brr[9];
1308 operation.network = brr[10];
1309 operation.type = brr[11];
1310 oSuccessor[operation.rank][operation.node] = brr[12];
1311 cycle = true;
1312
1313#ifdef USE_VERBOSE
1314 std::cout << "Rank : " << N_process_Rank << " in iter : " << iter_num << " received buffer from respective rank and deserialized" << std::endl;
1315#endif
1316
1317 int target_cluster = brr[6] < primary_processes ? 0 : 1;
1318 int target_node = brr[6] < primary_processes ? std::floor(brr[6] / primary_task_per_node) : std::floor((brr[6] - primary_processes) / secondary_task_per_node);
1319 int target_socket = brr[6] < primary_processes ? std::floor(brr[6] / primary_cores_per_socket) : std::floor((brr[6] - primary_processes) / secondary_cores_per_socket);
1320
1321 std::set<int> overlapp_ranks;
1322 int overlapp_count = 1;
1323
1324 if (target_cluster != system_number)
1325 {
1326 comm_type = INTERCLUSTER;
1327 }
1328 else if (target_node != node)
1329 {
1330 comm_type = INTERNODE;
1331 }
1332 else if (target_socket != socket)
1333 {
1334 comm_type = INTERCHIP;
1335 }
1336 else
1337 {
1338 comm_type = INTRACHIP;
1339 }
1340 overlapp_ranks = ncm_overlap_ranks(DisCosTiC_AST, comm_type, system_number, node, socket, target_node);
1341
1342 for (const auto &elem : overlapp_ranks)
1343 {
1344 if (((gather_buff_optime[elem] - 5000) <= gather_buff_optime[rank] && gather_buff_optime[rank] <= (gather_buff_optime[elem] + 5000)) && (elem != brr[6]))
1345 {
1346 ++overlapp_count;
1347 }
1348 }
1349#ifdef USE_VERBOSE
1350 std::cout << "RECV Rank : " << rank << " iteration : " << iter_num << " overlapp count : " << overlapp_count << std::endl;
1351#endif
1352
1353 IntrachipCFG_args.setData(overlapp_count, operation.bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit, INTRACHIP);
1354 InterchipCFG_args.setData(overlapp_count, operation.bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit, INTERCHIP);
1355 InternodeCFG_args.setData(overlapp_count, operation.bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit, INTERNODE);
1356 InterclusterCFG_args.setData(overlapp_count, operation.bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit, INTERCLUSTER);
1357
1358 if (operation.bufSize > eagerLimit)
1359 {
1360#ifdef USE_VERBOSE
1361 std::cout << "Rank : " << N_process_Rank << " in iter : " << iter_num << " obeying RENDEZVOUS protocol" << std::endl;
1362#endif
1363 if (operation.mode == DisCosTiC::Mode_t::BLOCKING)
1364 {
1365 if ((heteregeneous_mode == 1) && (flag_recv_5 == 1))
1366 {
1367 operation.time = old_op_time;
1368 oSuccessor[operation.rank][operation.node] = old_op_time;
1369 gsendSuccessor[operation.rank][operation.network] = old_op_time;
1370 }
1371 if ((heteregeneous_mode == 1) && (system_number == 0) && (iter_num == 0) && (flag_blocking_4 == 0))
1372 {
1373 flag_blocking_4 = 1;
1374 operation.time = old_op_time;
1375 oSuccessor[operation.rank][operation.node] = old_op_time;
1376 gsendSuccessor[operation.rank][operation.network] = old_op_time;
1377 }
1378 }
1379 }
1380
1381 double update_time = 0;
1382 if (comm_type == INTERCLUSTER)
1383 {
1384 update_time = (operation.bufSize * inverse_bandwidth[comm_type]);
1385 }
1386 else
1387 {
1388 update_time = 0.5 * (operation.bufSize * inverse_bandwidth[comm_type]);
1389 }
1390
1391 if (comm_model == LOGGP)
1392 {
1393 oSuccessor[operation.rank][operation.node] = operation.time + o[comm_type] + std::max((operation.bufSize - 1) * O[comm_type], (operation.bufSize - 1) * G[comm_type]);
1394 grecvSuccessor[operation.rank][operation.network] = operation.time + g[comm_type] + ((operation.bufSize - 1) * G[comm_type]);
1395 }
1396 if (comm_model == SIMPLELB)
1397 {
1398 oSuccessor[operation.rank][operation.node] = operation.time + (update_time);
1399 gsendSuccessor[operation.rank][operation.network] = operation.time + (update_time); // TODO: G should be charged in network layer only
1400 }
1401 commTime += (oSuccessor[operation.rank][operation.node] - operation.time);
1402
1403 // print_OpPropertiesNonPointerT(operation);
1404 operation.numOpsInQueue = numOpsInQueue++;
1405 finishedRankList.push_back(operation.rank);
1406
1407#ifdef USE_VERBOSE
1408 std::cout << "[RECV ==> SATISFY] local non-blocking dependency" << std::endl;
1409 std::cout << "Unexpected message queue bufSize: " << UMQ.size() << std::endl;
1410 std::cout << "matching queue search for src " << operation.target << " tag " << operation.tag << std::endl;
1411#endif
1412
1413#ifdef USE_VERBOSE
1414 std::cout << " [RECV ==> FOUND] in local unexpected queue" << std::endl;
1415#endif
1416
1417 if (comm_model == LOGGP)
1418 {
1419#ifdef USE_CHROMEVIZ
1420 traceVisualizer2.args("CPU" + std::to_string(rank), operation.target);
1421 traceVisualizer2.args("CPU" + std::to_string(rank), operation.rank);
1422 traceVisualizer2.flowEventBegin("Name:startMSG", operation.target, traceVisualizer2.arc[("CPU" + std::to_string(0))], std::to_string((operation.starttime + o[comm_type]) / 1000), flow_id);
1423 traceVisualizer2.flowEventEnd("Name:endMSG", operation.rank, traceVisualizer2.arc[("CPU" + std::to_string(0))], std::to_string(operation.time / 1000), flow_id);
1424 flow_id++;
1425
1426 traceVisualizer2.args("CPU" + std::to_string(operation.node), operation.rank);
1427 traceVisualizer2.completeEvents("RECV", operation.rank, traceVisualizer2.arc[("CPU" + std::to_string(operation.node))], std::to_string((operation.time + (operation.bufSize - 1) * G[comm_type] - (operation.bufSize - 1) * O[comm_type]) / 1000), std::to_string(((operation.time + o[comm_type] + std::max((operation.bufSize - 1) * O[comm_type], (operation.bufSize - 1) * G[comm_type])) - (operation.time + (operation.bufSize - 1) * G[comm_type] - (operation.bufSize - 1) * O[comm_type])) / 1000), "terrible");
1428
1429#endif
1430#ifdef USE_ITACVIZ
1431 {
1432 const double recv_start = operation.time + (operation.bufSize - 1) * G[comm_type] - (operation.bufSize - 1) * O[comm_type];
1433 const double recv_end = operation.time + o[comm_type] + std::max((operation.bufSize - 1) * O[comm_type], (operation.bufSize - 1) * G[comm_type]);
1434 // brr[6] is the modeled sender (send.rank). operation.target was overwritten
1435 // with the send destination (this rank), which would look like a self-recv.
1436 const auto src = static_cast<DisCosTiC_Indextype>(brr[6]);
1437 traceVisualizerItac.recv(operation.rank, src, recv_start,
1438 recv_end - recv_start, operation.bufSize, operation.tag);
1439 }
1440#endif
1441
1442#ifdef USE_DRAWVIZ
1443 traceVisualizer.orecv(operation.rank, operation.time, operation.time + o, operation.node);
1444#endif
1445 }
1446 if (comm_model == SIMPLELB)
1447 {
1448#ifdef USE_CHROMEVIZ
1449 traceVisualizer2.args("CPU" + std::to_string(0), operation.target);
1450 traceVisualizer2.args("CPU" + std::to_string(0), operation.rank);
1451 traceVisualizer2.flowEventBegin("Name:startMSG", operation.target, traceVisualizer2.arc[("CPU" + std::to_string(0))], std::to_string((operation.starttime) / 1000), flow_id);
1452 traceVisualizer2.flowEventEnd("Name:endMSG", operation.rank, traceVisualizer2.arc[("CPU" + std::to_string(0))], std::to_string(operation.time / 1000), flow_id);
1453 ++flow_id;
1454
1455 traceVisualizer2.args("CPU" + std::to_string(operation.node), operation.rank);
1456 traceVisualizer2.completeEvents("RECV", operation.rank, traceVisualizer2.arc[("CPU" + std::to_string(operation.node))], std::to_string((operation.time) / 1000), std::to_string((update_time / 1000)), "terrible");
1457
1458#endif
1459#ifdef USE_ITACVIZ
1460 {
1461 const auto src = static_cast<DisCosTiC_Indextype>(brr[6]);
1462 traceVisualizerItac.recv(operation.rank, src, operation.time,
1463 update_time, operation.bufSize, operation.tag);
1464 }
1465#endif
1466
1467#ifdef USE_DRAWVIZ
1468 traceVisualizer.orecv(operation.rank, operation.time, operation.time + o, operation.node);
1469#endif
1470 }
1471
1472 finishedRankList.push_back(operation.rank);
1473
1474 if ((desync_feature == 1) && (iter_num >= 0 && iter_num < 200))
1475 {
1476#ifdef USE_VERBOSE
1477 std::cout << "Rank : " << N_process_Rank << " in iter : " << iter_num << " desynchronizing simulation by injecting noise" << std::endl;
1478#endif
1479 oSuccessor[operation.rank][operation.node] += rand() % 500000;
1480 }
1481 }
1482#ifdef USE_VERBOSE
1483 std::cout << "Rank : " << N_process_Rank << " in iter : " << iter_num << " finished RECV" << std::endl;
1484#endif
1485 }
1486 break;
1487
1489 {
1490#ifdef USE_VERBOSE
1491 std::cout << "Rank : " << N_process_Rank << " in iter : " << iter_num << " starting COLLECTIVE" << std::endl;
1492#endif
1493 //collective operation using CollectiveModel class
1494 //read from collective fields
1495 DisCosTiC_Datatype root = operation.collectiveRoot;
1496 DisCosTiC_Datatype numRanks = operation.collectiveNumRanks;
1497 DisCosTiC::CollectiveType collectiveType = static_cast<DisCosTiC::CollectiveType>(operation.collectiveType);
1498 uint8_t algorithm = operation.algorithm;
1499 collective_algorithm = algorithm;
1500 collective_type = collectiveType;
1501
1502 //sync all ranks entering collective
1503 double collective_entry_time = operation.time;
1504 double local_entry_time = collective_entry_time;
1505 MPI_Allreduce(&local_entry_time, &collective_entry_time, 1, MPI_DOUBLE, MPI_MAX, newcomm);
1506
1507 //determine network params based on collective scope
1508 communication_type comm_type;
1509
1510 //communication type determined based on rank span
1511 if (heteregeneous_mode == 1 &&
1512 ((system_number == 0 && numRanks > primary_processes) ||
1513 (system_number == 1 && root < primary_processes))) {
1514 //collective spans heterogeneous bounds, never used because heteregeneous_mode always 0
1515 comm_type = INTERCLUSTER;
1516 } else if (numRanks > task_per_node) {
1517 //spans multiple nodes
1518 comm_type = INTERNODE;
1519 } else if (numRanks > cores_per_socket) {
1520 //spans multiple sockets but within node
1521 comm_type = INTERCHIP;
1522 } else {
1523 //all ranks within single socket
1524 comm_type = INTRACHIP;
1525 }
1526
1527 //no overlap counting for collectives
1528 //update network params for actual message size
1529 IntrachipCFG_args.setData(1, operation.bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit, INTRACHIP);
1530 InterchipCFG_args.setData(1, operation.bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit, INTERCHIP);
1531 InternodeCFG_args.setData(1, operation.bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit, INTERNODE);
1532 InterclusterCFG_args.setData(1, operation.bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit, INTERCLUSTER);
1533
1534 //actual calculation is done in CollectiveModel class
1535 DisCosTiC_Timetype collective_duration =
1537 collectiveType,
1538 operation.rank,
1539 root,
1540 numRanks,
1541 operation.bufSize,
1542 latency[comm_type], //L
1543 o[comm_type], //o
1544 g[comm_type], //g
1545 G[comm_type], //G
1546 algorithm, // algorithm choice
1547 inverse_bandwidth[comm_type], //inverse bandwidth
1548 eagerLimit //eager limit
1549 );
1550
1551 DisCosTiC_Timetype completion_time = collective_entry_time + collective_duration;
1552
1553 //store collective timing for output at the end
1554 if (collective_mode == 1) {
1555 gather_buff_comp[0][operation.rank] = collective_entry_time;
1556 gather_buff_comp[1][operation.rank] = collective_duration;
1557 }
1558
1559 //each rank completes at its calculated time
1560 operation.time = completion_time;
1561
1562 //update successor time for all ranks
1563 if (oSuccessor[operation.rank][operation.node] < operation.time)
1564 {
1565 oSuccessor[operation.rank][operation.node] = operation.time;
1566 }
1567
1568 finishedRankList.push_back(operation.rank);
1569
1570#ifdef USE_CHROMEVIZ
1571 // viz for collective operations
1572 std::string collectiveName = "Name:COLLECTIVE_";
1573 switch(collectiveType) {
1574 case DisCosTiC::COLL_BCAST: collectiveName += "BCAST"; break;
1575 case DisCosTiC::COLL_REDUCE: collectiveName += "REDUCE"; break;
1576 case DisCosTiC::COLL_ALLREDUCE: collectiveName += "ALLREDUCE"; break;
1577 default: collectiveName += "UNKNOWN"; break;
1578 }
1579
1580 //for collectives, use CPU 0 since operation.node contains algorithm used
1581 traceVisualizer2.args("CPU0", operation.rank);
1582 traceVisualizer2.completeEvents(collectiveName, operation.rank,
1583 traceVisualizer2.arc["CPU0"],
1584 std::to_string(collective_entry_time / 1000),
1585 std::to_string((operation.time - collective_entry_time) / 1000), "terrible");
1586#endif
1587#ifdef USE_ITACVIZ
1588 {
1589 std::string itacColl = "BCAST";
1590 switch (collectiveType)
1591 {
1593 itacColl = "Bcast";
1594 break;
1596 itacColl = "Reduce";
1597 break;
1599 itacColl = "Allreduce";
1600 break;
1601 default:
1602 itacColl = "Collective";
1603 break;
1604 }
1605 traceVisualizerItac.collective(operation.rank, collective_entry_time,
1606 operation.time - collective_entry_time, itacColl);
1607 }
1608#endif
1609
1610#ifdef USE_VERBOSE
1611 std::cout << "Rank : " << N_process_Rank << " in iter : " << iter_num
1612 << " finished COLLECTIVE at time " << operation.time << std::endl;
1613#endif
1614 }
1615 break;
1616
1617 default:
1618 std::cout << "YET NOT IMPLEMENTED! : " << (int)operation.type << std::endl;
1619 break;
1620 }
1621
1622 std::sort(finishedRankList.begin(), finishedRankList.end());
1623 finishedRankList.erase(std::unique(finishedRankList.begin(), finishedRankList.end()), finishedRankList.end());
1624
1625#ifdef USE_VERBOSE
1626 std::cout << "Rank : " << N_process_Rank << " finished iter : " << iter_num << std::endl;
1627#endif
1632 for (auto &rank : finishedRankList)
1633 {
1634 DisCosTiC::Grid *it = &AST_args.graphVec[rank];
1635 DisCosTiC::Operations operations;
1636 it->getTypeSortedOps(operations);
1637
1638 if (operations.size() != 0)
1639 operations_queue.insert(operations_queue.end(), operations.begin(), operations.end());
1640
1641 if (operations_queue.size() != 0)
1642 {
1643 auto op = operations_queue.front();
1644 operations_queue.erase(operations_queue.begin()); // removes the front element.
1645 op.rank = rank;
1646 switch (op.type)
1647 {
1650 op.time = oSuccessor[rank][op.node];
1651#ifdef USE_VERBOSE
1652 verboseCompFinalPrint(rank, op);
1653#endif
1654 break;
1655
1657 op.time = std::max(oSuccessor[rank][op.node], gsendSuccessor[rank][op.network]);
1658#ifdef USE_VERBOSE
1659 verboseSendFinalPrint(rank, op);
1660#endif
1661 break;
1662
1664 op.time = oSuccessor[rank][op.node];
1665#ifdef USE_VERBOSE
1666 verboseRecvFinalPrint(rank, op);
1667#endif
1668 break;
1669
1671 op.time = oSuccessor[rank][op.node];
1672#ifdef USE_VERBOSE
1673 std::cout << "final COLLECTIVE op for rank " << rank << " at time " << op.time << std::endl;
1674#endif
1675 break;
1676
1677 default:
1678 std::cerr << "YET NOT IMPLEMENTED!" << std::endl;
1679 }
1680
1681 operation = op;
1682#ifdef USE_VERBOSE
1683 std::cout << "Rank : " << N_process_Rank << " in iter : " << iter_num << " set new operation" << std::endl;
1684#endif
1685 }
1686 else
1687 {
1688 // double start = MPI_Wtime();
1689
1690 // // I simulate the sleep function. Required for receiving the send and recv in given fashion
1691 // while(MPI_Wtime() - start < 0.05)
1692 // {
1693 // // We keep looping until <waiting_time> seconds have elapsed
1694 // }
1695 iter_num++;
1696 cycle = false;
1697#ifdef USE_VERBOSE
1698 std::cout << "Rank : " << N_process_Rank << " in iter : " << iter_num << " exiting cycle as no new operation in queue" << std::endl;
1699#endif
1700 }
1701 // exit(0);
1702 }
1703
1704 } while (cycle);
1705 cycle = true;
1706 operation_og.time = oSuccessor[operation.rank][operation.node];
1707 operation = copy(operation_og, operation);
1708 } while (iter_num < iter);
1709
1710 //send collective timing data if in collective mode (before final data)
1711 if (collective_mode == 1) {
1712 double collective_timing[4];
1713 collective_timing[0] = gather_buff_comp[0][rank]; //entry time
1714 collective_timing[1] = gather_buff_comp[1][rank];//duration
1715 collective_timing[2] = collective_algorithm; //algorithm
1716 collective_timing[3] = collective_type; //collective type
1717 MPI_Send(&collective_timing, 4, MPI_DOUBLE, 0, 100, MPI_COMM_WORLD);
1718 }
1719
1720 double brr[5];
1721 *finalize(oSuccessor[rank][operation.node], numRanks, numOpsInQueue, rank, operation.node, brr);
1722 MPI_Send(&brr, 5, MPI_DOUBLE, 0, 1, MPI_COMM_WORLD);
1723
1724#ifdef USE_VERBOSE
1725 std::cout << "Rank : " << process_Rank << " send data to MASTER RANK" << std::endl;
1726#endif
1727 }
1732 else if (process_Rank == 0)
1733 {
1734 bool cycle = true;
1735 double brr[5];
1736
1737 if (collective_mode == 1) {
1738 for (int i = 0; i < numRanks; i++) {
1739 double timing_data[4];
1740 MPI_Recv(&timing_data, 4, MPI_DOUBLE, i+1, 100, MPI_COMM_WORLD, &status);
1741 collective_data_storage[i][0] = timing_data[0]; //entry time
1742 collective_data_storage[i][1] = timing_data[1]; //duration
1743 if (i == 0) { //get algo and type from first rank
1744 collective_algorithm = (uint8_t)timing_data[2];
1745 collective_type = static_cast<DisCosTiC::CollectiveType>((int)timing_data[3]);
1746 }
1747 }
1748 }
1749
1750 // receive the regular completion data
1751 while (test.size() < numRanks)
1752 {
1753
1754 MPI_Recv(&brr, 5, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
1755
1756 if (status.MPI_TAG == 1)
1757 {
1758 test.emplace(status.MPI_SOURCE);
1759 oSuccessor[brr[3]][brr[4]] = brr[0];
1760 numOpsInQueue = brr[2];
1761 // print_vec2T(oSuccessor);
1762 }
1763 }
1764#ifdef USE_VERBOSE
1765 std::cout << "Rank : " << process_Rank << " MASTER RANK received data from all simulation processes" << std::endl;
1766#endif
1767 }
1772 ierr = MPI_Comm_rank(MPI_COMM_WORLD, &process_Rank);
1773 MPI_Barrier(MPI_COMM_WORLD);
1774 traceVisualizer2.closeFile();
1775#ifdef USE_ITACVIZ
1776 traceVisualizerItac.closeFile();
1777#endif
1778
1779#ifdef USE_VERBOSE
1780 std::cout << "Rank : " << process_Rank << " All simulation processes completed simulation" << std::endl;
1781#endif
1782
1783 if (process_Rank == 0)
1784 {
1785
1787 // /* \brief Ending Time and Printing Performance information */ //
1789
1790 endTime = high_resolution_clock::now();
1791 DisCosTiC_Timetype DisCosTiC = duration_cast<nanoseconds>(endTime - startTime).count();
1792 ;
1793
1794 std::cout << "\n\nDisCosTiC PERFORMANCE:" << std::endl;
1795 std::cout << "----------------------------------------------------------------" << std::endl;
1796 std::cout << "Number of operations Runtime [s] Speed [operations/ns]" << std::endl;
1797 std::cout << "----------------------------------------------------------------" << std::endl;
1798 std::cout << numOpsInQueue << " \t\t\t\t" << DisCosTiC / 1e9 << " \t\t" << numOpsInQueue / DisCosTiC << std::endl;
1799
1801 // /* \brief check if all queues are empty (debugging the queues) if true //
1802 // * so print finishing time (either for all ranks, or for only maximum rank with enabled batch mode) */ //
1804
1805 if (CFG_args.getValue<int>("batchmode"))
1806 {
1807 slowRankTime(numRanks, oSuccessor);
1808 }
1809 else if (!CFG_args.getValue<int>("batchmode"))
1810 {
1811 allRanksTime(numRanks, oSuccessor);
1812 }
1813 else
1814 {
1815 throw std::runtime_error("ERROR: QUEUES NOT EMPTY!\n");
1816 }
1817
1818 //print collective timings
1819 if (collective_mode == 1) {
1820 std::cout << "\n\nCOLLECTIVE OPERATION TIMING DETAILS:" << std::endl;
1821 std::cout << "-----------------------------------------------------------------" << std::endl;
1822 std::cout << "rank entry_time[ns] duration[ns] completion[ns] stage" << std::endl;
1823 std::cout << "-----------------------------------------------------------------" << std::endl;
1824
1825 //use collective timing data that was already received
1826 double all_collective_data[2 * numRanks];
1827 for (int i = 0; i < numRanks; i++) {
1828 all_collective_data[i] = collective_data_storage[i][0]; //entry time
1829 all_collective_data[numRanks + i] = collective_data_storage[i][1]; //duration
1830 }
1831
1832 //calc stage based on duration (for tree broadcast)
1833 DisCosTiC_Timetype min_duration = *std::min_element(&all_collective_data[numRanks],
1834 &all_collective_data[2 * numRanks]);
1835 DisCosTiC_Timetype stage_time = min_duration; //time for one stage
1836
1837 for (int i = 0; i < numRanks; i++) {
1838 DisCosTiC_Timetype entry_time = all_collective_data[i];
1839 DisCosTiC_Timetype duration = all_collective_data[numRanks + i];
1840 DisCosTiC_Timetype completion = entry_time + duration;
1841 int stage = (stage_time > 0) ? static_cast<int>(duration / stage_time) : 0;
1842
1843 std::cout << std::setw(4) << i << " "
1844 << std::setw(13) << std::fixed << std::setprecision(0) << entry_time << " "
1845 << std::setw(11) << duration << " "
1846 << std::setw(13) << completion << " "
1847 << std::setw(5) << stage << std::endl;
1848 }
1849
1850 //summary
1851 DisCosTiC_Timetype max_duration = *std::max_element(&all_collective_data[numRanks],
1852 &all_collective_data[2 * numRanks]);
1853 DisCosTiC_Timetype avg_duration = std::accumulate(&all_collective_data[numRanks],
1854 &all_collective_data[2 * numRanks], 0.0) / numRanks;
1855
1856 std::cout << "---------------------------------------------------------------" << std::endl;
1857 std::cout << "Collective Summary:" << std::endl;
1858
1859 //print collective type and algorithm
1860 std::string type_name, algo_name;
1861 switch(collective_type) {
1863 type_name = "BROADCAST";
1864 algo_name = (collective_algorithm == DisCosTiC::BCAST_RING) ? "Ring" : "Binomial Tree";
1865 break;
1867 type_name = "REDUCE";
1868 algo_name = "Binomial Tree";
1869 break;
1871 type_name = "ALLREDUCE";
1872 algo_name = "Recursive Doubling";
1873 break;
1874 default:
1875 type_name = "UNKNOWN";
1876 algo_name = "Unknown";
1877 break;
1878 }
1879
1880 std::cout << " Type: " << type_name << " (" << algo_name << " Algorithm)" << std::endl;
1881 std::cout << " Root: 0" << std::endl;
1882 std::cout << " Participating ranks: " << numRanks << std::endl;
1883 std::cout << " Min duration: " << min_duration << " ns" << std::endl;
1884 std::cout << " Max duration: " << max_duration << " ns" << std::endl;
1885 std::cout << " Avg duration: " << avg_duration << " ns" << std::endl;
1886
1887 //calculate stages based on collective type
1888 int stages;
1889 if (collective_type == DisCosTiC::COLL_ALLREDUCE) {
1890 stages = 2 * static_cast<int>(std::ceil(std::log2(numRanks))); //reduce + bcast phases
1891 } else {
1892 stages = static_cast<int>(std::ceil(std::log2(numRanks)));
1893 }
1894 std::cout << " Stages: " << stages << std::endl;
1895 std::cout << "---------------------------------------------------------------" << std::endl;
1896
1897 //print the timings for different message sizes (not part of actual simulation)
1898 std::cout << "\nCollective Performance Analysis (Multiple Sizes):" << std::endl;
1899 std::cout << "---------------------------------------------------------------" << std::endl;
1900 std::cout << "Size (KB) Min (μs) Avg (μs) Max (μs)" << std::endl;
1901 std::cout << "---------------------------------------------------------------" << std::endl;
1902
1903 std::vector<int> test_sizes = {1024, 2048, 4096, 8192, 16384, 32768, 65536,
1904 131072, 262144, 524288, 1048576, 2097152,
1905 4194304, 8388608, 10485760};
1906
1907 for (int msg_size : test_sizes) {
1908 //calculate timing for all ranks with this message size
1909 std::vector<DisCosTiC_Timetype> durations;
1910
1911 //determine comm type like in actual sim
1912 communication_type comm_type_analysis;
1913 if (numRanks <= cores_per_socket) {
1914 comm_type_analysis = INTRACHIP;
1915 } else if (numRanks <= task_per_node) {
1916 comm_type_analysis = INTERCHIP;
1917 } else {
1918 //spans multiple nodes
1919 comm_type_analysis = INTERNODE;
1920 }
1921
1922 //get network params for this message size using temporary variables
1923 DisCosTiC::Networktype temp_latency, temp_inverse_bandwidth, temp_o, temp_O, temp_g, temp_G;
1924 DisCosTiC_Datatype temp_eagerLimit;
1925
1926 if (comm_type_analysis == INTRACHIP) {
1927 IntrachipCFG_args.setData(1, msg_size, temp_latency, temp_inverse_bandwidth, temp_o, temp_O, temp_g, temp_G, temp_eagerLimit, INTRACHIP);
1928 } else if (comm_type_analysis == INTERCHIP) {
1929 InterchipCFG_args.setData(1, msg_size, temp_latency, temp_inverse_bandwidth, temp_o, temp_O, temp_g, temp_G, temp_eagerLimit, INTERCHIP);
1930 } else {
1931 InternodeCFG_args.setData(1, msg_size, temp_latency, temp_inverse_bandwidth, temp_o, temp_O, temp_g, temp_G, temp_eagerLimit, INTERNODE);
1932 }
1933
1934 for (int rank = 0; rank < numRanks; rank++) {
1935 //calc duration and size using the temp parameters
1937 collective_type,
1938 rank, 0, numRanks, msg_size,
1939 temp_latency[comm_type_analysis],
1940 temp_o[comm_type_analysis],
1941 temp_g[comm_type_analysis],
1942 temp_G[comm_type_analysis],
1943 collective_algorithm,
1944 temp_inverse_bandwidth[comm_type_analysis],
1945 eagerLimit
1946 );
1947 durations.push_back(duration);
1948 }
1949
1950 //find min max avg
1951 DisCosTiC_Timetype min_dur = *std::min_element(durations.begin(), durations.end());
1952 DisCosTiC_Timetype max_dur = *std::max_element(durations.begin(), durations.end());
1953 DisCosTiC_Timetype avg_dur = std::accumulate(durations.begin(), durations.end(), 0.0) / numRanks;
1954
1955 //print row with kb size and microsecs timings
1956 std::cout << std::setw(8) << (msg_size / 1024) << " "
1957 << std::setw(7) << std::fixed << std::setprecision(1) << (min_dur / 1000.0) << " "
1958 << std::setw(7) << (avg_dur / 1000.0) << " "
1959 << std::setw(7) << (max_dur / 1000.0) << std::endl;
1960 }
1961
1962 std::cout << "----------------------------------------------------------------" << std::endl;
1963 }
1964
1965#ifdef USE_CHROMEVIZ
1966 system(("rm " + CFG_args.getValue<std::string>("chromevizfilename") + ".dms 2> /dev/null").c_str());
1967
1968 system(("cat " + CFG_args.getValue<std::string>("chromevizfilename") + "{0.." + std::to_string(size_Of_Cluster - 2) + "}.dms > " + CFG_args.getValue<std::string>("chromevizfilename") + ".dms").c_str());
1969
1970 std::string filename = CFG_args.getValue<std::string>("chromevizfilename") + ".dms";
1971 std::ofstream ofs;
1972 ofs.open(filename, std::ios_base::app);
1973
1974 ofs << "{}\n],"
1975 << "\"Name\": \"Trace visualisation ofs DisCosTiC toolkit\", \n "
1976 << "\"Full form\": \"Distributed CosT in Cluster\", \n "
1977 << "\"Version\": \"v2.0.0 (second release)\", \n "
1978 << "\"Author\": \"Ayesha Afzal <ayesha.afzal@fau.de>\", \n "
1979 << "\"Copyright\": \"© 2023 HPC, FAU Erlangen-Nuremberg. All rights reserved\" \n "
1980 << "}\n";
1981 ofs.close();
1982
1983#endif
1984#ifdef USE_ITACVIZ
1985 {
1986 const std::string avt_base = CFG_args.getValue<std::string>("chromevizfilename");
1987 system(("rm " + avt_base + ".avt 2> /dev/null").c_str());
1988 system(("cat " + avt_base + "{0.." + std::to_string(size_Of_Cluster - 2) + "}.avt > " + avt_base + ".avt").c_str());
1989 system(("rm " + avt_base + "{0.." + std::to_string(size_Of_Cluster - 2) + "}.avt 2> /dev/null").c_str());
1990 }
1991#endif
1992#ifdef USE_DRAWVIZ
1993 system("rm DisCosTiC.out 2> /dev/null");
1994
1995 system("cat DisCosTiC*.out >> DisCosTiC.out");
1996
1997#endif
1998 system(("rm DisCosTiC{0.." + std::to_string(size_Of_Cluster - 2) + "}.out 2> /dev/null").c_str());
1999 system(("rm " + CFG_args.getValue<std::string>("chromevizfilename") + "{0.." + std::to_string(size_Of_Cluster - 2) + "}.dms 2> /dev/null").c_str());
2000
2001#ifdef USE_VERBOSE
2002 std::cout << "Rank : " << process_Rank << " wrote data to MASTER file and cleaned up files" << std::endl;
2003#endif
2004 }
2005 delete DisCosTiC_AST;
2006 MPI_Finalize();
2007
2008 return EXIT_SUCCESS;
2009}
UserInterface::ConfigParser CFG_args("config.cfg")
< configuration file parser
std::size_t DisCosTiC_Indextype
Definition DataType.hpp:53
std::vector< std::vector< DisCosTiC_Timetype > > Timevec2T
Definition DataType.hpp:61
std::size_t DisCosTiC_Datatype
Definition DataType.hpp:52
std::vector< DisCosTiC_Indextype > vec1T
Definition DataType.hpp:58
double DisCosTiC_Timetype
Definition DataType.hpp:51
int virtual_rank
Definition DisCosTiC.cpp:62
communication_mode
Definition DisCosTiC.cpp:32
@ SIMPLELB
Definition DisCosTiC.cpp:34
@ LOGGP
Definition DisCosTiC.cpp:33
int secondary_processes
Definition DisCosTiC.cpp:71
int secondary_cores_per_socket
Definition DisCosTiC.cpp:74
int socket
Definition DisCosTiC.cpp:69
int scaling_cores
Definition AST.hpp:36
int main(int argc, char **argv)
int heteregeneous_mode
Definition DisCosTiC.cpp:72
bound_type
Definition DisCosTiC.cpp:52
@ COMPUTE
Definition DisCosTiC.cpp:53
@ MEMORY
Definition DisCosTiC.cpp:54
DisCosTiC::DisCosTiC_OP copy(DisCosTiC::DisCosTiC_OP op_og, DisCosTiC::DisCosTiC_OP op_copy)
Definition DisCosTiC.cpp:79
time
Definition DisCosTiC.cpp:46
@ START
Definition DisCosTiC.cpp:47
@ END
Definition DisCosTiC.cpp:48
int primary_cores_per_socket
Definition DisCosTiC.cpp:73
int secondary_task_per_node
Definition DisCosTiC.cpp:76
int system_number
Definition DisCosTiC.cpp:63
int primary_processes
Definition DisCosTiC.cpp:70
int cores_per_socket
Definition DisCosTiC.cpp:67
int bound
Definition NodeLvlScg.cpp:9
int bytes_to_send
Definition AST.hpp:37
int task_per_node
Definition DisCosTiC.cpp:64
int cc_numa_domain_per_socket
Definition DisCosTiC.cpp:66
communication_type
Definition DisCosTiC.cpp:38
@ INTRACHIP
Definition DisCosTiC.cpp:39
@ INTERCHIP
Definition DisCosTiC.cpp:40
@ INTERNODE
Definition DisCosTiC.cpp:41
@ INTERCLUSTER
Definition DisCosTiC.cpp:42
static std::set< int > ncm_overlap_ranks(AST *ast, int comm_kind, int system_number, int local_node, int local_socket, int peer_node)
std::string arch_name
Definition DisCosTiC.cpp:77
int cc_numa_domain
Definition DisCosTiC.cpp:68
int primary_task_per_node
Definition DisCosTiC.cpp:75
int node
Definition DisCosTiC.cpp:65
double * finalize(double a, double b, double c, double d, double e, double arr[5])
Definition DisCosTiC.cpp:98
end AST_HPP
Definition AST.hpp:58
DisCosTiC::NCMIntraAndInterNodeData intraAndInterNodeRanks
Definition AST.hpp:64
void printNCMRanks()
Definition AST.hpp:394
DisCosTiC::NCMInterClusterData interClusterRanks
Definition AST.hpp:65
end of Benchmark class
DisCosTiC_Datatype systemsize
uint8_t GetNumNetworks()
the maximum number of the network interface controller
DisCosTiC_Datatype numOperations
DisCosTiC::idNodeTypePair Oneway_PositiveDisplacement(AST *DisCosTiC, DisCosTiC_Datatype process_Rank, UserInterface::YAMLParser YAML_args, DisCosTiC_Datatype N_size_Of_Cluster, MPI_Comm comm)
end of Oneway_PositiveDisplacement
uint8_t GetNumCores()
the maximum number of the nodes
static DisCosTiC_Timetype calculateCollectiveTime(CollectiveType collectiveType, DisCosTiC_Datatype rank, DisCosTiC_Datatype root, DisCosTiC_Datatype numRanks, DisCosTiC_Datatype messageSize, DisCosTiC_Timetype L, DisCosTiC_Timetype o, DisCosTiC_Timetype g, DisCosTiC_Timetype G, uint8_t algorithm=0, DisCosTiC_Timetype inverse_bandwidth=0, DisCosTiC_Datatype eagerLimit=65535)
this class exposes all P graphVec and manages dependencies and execution order. It returns a list of ...
Definition GridInit.hpp:23
DisCosTiC::VecSeqGraph_t graphVec
public variables
Definition GridInit.hpp:31
void getTypeSortedOps(DisCosTiC::Operations &opVec)
get the all operations ordered by their type for all ranks and time steps
Definition Grid.hpp:116
void getSortedRootOps(DisCosTiC::Operations &opVec)
get the initial root operations ordered by their type for all ranks and time steps
Definition Grid.hpp:162
DisCosTiC_Datatype getNumOps()
default constructor
Definition Grid.hpp:108
void flowEventEnd(std::string name, DisCosTiC_Indextype dest, DisCosTiC_Indextype dest_tid, std::string time, DisCosTiC_Indextype index)
void args(std::string args_name, DisCosTiC_Indextype rank)
void completeEvents(std::string name, DisCosTiC_Indextype rank, DisCosTiC_Indextype tid, std::string time, std::string duration, std::string color)
std::map< std::string, DisCosTiC_Indextype > arc
void flowEventBegin(std::string name, DisCosTiC_Indextype source, DisCosTiC_Indextype source_tid, std::string time, DisCosTiC_Indextype index)
a wrapper class which contains functions for parsing the configuration file end class configParser
scalarT getValue(const std::string &key, scalarT const &defaultValue=scalarT()) const
a function that retrieves the value of a specific key
void comp(DisCosTiC_Indextype app_rank, double start_ns, double dur_ns, const std::string &name)
Definition ItacVtf.hpp:71
void send(DisCosTiC_Indextype src, DisCosTiC_Indextype dst, double start_ns, double dur_ns, double bytes, DisCosTiC_Indextype tag)
Definition ItacVtf.hpp:80
void collective(DisCosTiC_Indextype app_rank, double start_ns, double dur_ns, const std::string &name)
Definition ItacVtf.hpp:102
void recv(DisCosTiC_Indextype dst, DisCosTiC_Indextype src, double start_ns, double dur_ns, double bytes, DisCosTiC_Indextype tag)
Definition ItacVtf.hpp:91
a wrapper class which contains functions for parsing the configuration file end class configParser
void setData(const int numberOfConcurrentConnection, const int buffersize, DisCosTiC::Networktype &latency, DisCosTiC::Networktype &inverse_bandwidth, DisCosTiC::Networktype &o, DisCosTiC::Networktype &O, DisCosTiC::Networktype &g, DisCosTiC::Networktype &G, DisCosTiC_Datatype &eagerLimit, const int mode)
void readData(const std::string &fileName)
end class TimeRankOP
void orecv(DisCosTiC_Datatype rank, DisCosTiC_Datatype start, DisCosTiC_Datatype end, DisCosTiC_Datatype cpu, float r=0.0, float g=0.0, float b=1.0)
print time taken by overhead at receiver side to output file
void osend(DisCosTiC_Datatype rank, DisCosTiC_Datatype start, DisCosTiC_Datatype end, DisCosTiC_Datatype cpu, float r=0.0, float g=0.0, float b=1.0)
print time taken by overhead at sender side to output file
void comp(DisCosTiC_Datatype rank, DisCosTiC_Datatype start, DisCosTiC_Datatype end, DisCosTiC_Datatype cpu, float r=1.0, float g=0.0, float b=0.0)
print time taken by execution to output file
end class YAMLParser
DisCosTiC_Datatype cores_per_numa_domain
DisCosTiC_Datatype cores_per_chip
@ BLOCKING
send and recv MPI routines, blocking: next operation not be executed before finishing of previous ope...
Definition enum.hpp:33
@ NONBLOCKING
isend and irecv MPI routines, non-blocking: next operation not be executed before starting of previou...
Definition enum.hpp:32
@ COMP
compuation operation type
Definition enum.hpp:19
@ SEND
send operation type
Definition enum.hpp:17
@ COMPADD
Definition enum.hpp:21
@ COLLECTIVE
Definition enum.hpp:22
@ RECV
recv operation type
Definition enum.hpp:18
#define verboseRecvFinalPrint(rank, operation)
Definition macro.hpp:408
#define version()
end P2P_HPP
Definition macro.hpp:569
#define verboseSendFinalPrint(rank, operation)
Definition macro.hpp:390
#define help()
Definition macro.hpp:482
#define slowRankTime(numRanks, nexto)
print final maximum time for only rank taking maximum time
Definition macro.hpp:346
#define allRanksTime(numRanks, nexto)
print final time for all ranks
Definition macro.hpp:332
#define verboseSendInitPrint(rank, operation)
Definition macro.hpp:382
#define verboseCompFinalPrint(rank, operation)
Definition macro.hpp:374
#define verboseRecvInitPrint(rank, operation)
Definition macro.hpp:400
#define print_OpPropertiesNonPointerT(op)
Definition macro.hpp:158
#define verboseCompInitPrint(rank, operation)
print verbose output for computational phases
Definition macro.hpp:368
#define verboseCompPrint(operation)
Definition macro.hpp:418
< benchmark test cases
std::priority_queue< DisCosTiC_OP, Operations, OpTimeComparator > PriorityQueue_t
@ BCAST_RING
Definition enum.hpp:46
CollectiveType
Definition enum.hpp:38
@ COLL_BCAST
Definition enum.hpp:39
@ COLL_ALLREDUCE
Definition enum.hpp:41
@ COLL_REDUCE
Definition enum.hpp:40
std::vector< std::vector< idNodeTypePairT > > idNodeTypePair
std::array< DisCosTiC_Timetype, 4 > Networktype
iteratorRange< scalarT > getRange(scalarT begin, scalarT end)
Definition DataType.hpp:161
std::vector< DisCosTiC_OP > Operations
std::vector< ListqueueOp > VecListqueueOp
char mode
TODO: add blocking/non-blocking mode.
DisCosTiC_Indextype network
uint8_t algorithm
algo choice for collective
uint8_t collectiveType
collective operation type
DisCosTiC_Indextype tag
DisCosTiC_Timetype syncstart
ifdef RANKSYNC
DisCosTiC_Timetype numOpsInQueue
STRICT_ORDER a timestamp that determines the (original) insertion order of elemenqueueOp in the queue...
DisCosTiC_Indextype node
DisCosTiC_Timetype time
DisCosTiC_Datatype bufSize
DisCosTiC_Indextype label
DisCosTiC_Indextype collectiveNumRanks
nubmer of ranks participating in collective
DisCosTiC_Indextype collectiveRoot
root rank for collectives which have root (-1 if no root)
DisCosTiC_Timetype starttime
only used for MSGs to identify start times
DisCosTiC_Indextype target
DisCosTiC_Indextype rank
owning rank of this operation
this matches and removes operations from list if found, otherwise returns false