141 if (strcmp(argv[1],
"-h") == 0 || strcmp(argv[1],
"--help") == 0)
146 if (strcmp(argv[1],
"-v") == 0 || strcmp(argv[1],
"--version") == 0)
156 int ierr, process_Rank, size_Of_Cluster, sucesss, N_process_Rank, N_size_Of_Cluster;
158 ierr = MPI_Init(NULL, NULL);
159 ierr = MPI_Comm_rank(MPI_COMM_WORLD, &process_Rank);
161 MPI_Comm_split(MPI_COMM_WORLD, (process_Rank) < 1 ? 1 : 2, 1, &newcomm);
162 ierr = MPI_Comm_rank(newcomm, &N_process_Rank);
163 ierr = MPI_Comm_size(newcomm, &N_size_Of_Cluster);
164 ierr = MPI_Comm_rank(MPI_COMM_WORLD, &process_Rank);
165 ierr = MPI_Comm_size(MPI_COMM_WORLD, &size_Of_Cluster);
166 std::queue<int> test;
188 if (collective_mode == 1) {
190 if (process_Rank == 0) {
191 std::cerr <<
"collective_mode=1 requires heteregeneous=0" << std::endl;
196 if (process_Rank == 0) {
197 std::cout <<
"collective_mode=1 enabled, forcing 1 timestep execution" << std::endl;
205 std::cout <<
"Simulator processes != number_of_processes+1" << std::endl;
214 std::cout <<
"Simulator processes != number_of_processes+1" << std::endl;
264 std::cout <<
"Rank : " << process_Rank <<
" completed Oneway_PositiveDisplacement()" << std::endl;
271 if (process_Rank == 0)
275 system((
"rm config{0.." + std::to_string(size_Of_Cluster) +
"}.cfg 2> /dev/null").c_str());
280 std::cout <<
"Rank : " << process_Rank <<
" completed AST generation()" << std::endl;
301 IntrachipCFG_args.
readData(
"./nodelevel/network-files/" +
arch_name +
"_intrachip.cfg");
303 InterchipCFG_args.
readData(
"./nodelevel/network-files/" +
arch_name +
"_interchip.cfg");
305 InternodeCFG_args.
readData(
"./nodelevel/network-files/" +
arch_name +
"_internode.cfg");
318 InterclusterCFG_args.
readData(
"./nodelevel/network-files/"
353 IntrachipCFG_args.
setData(1, init_msg_size, latency, inverse_bandwidth, o, O, g, G, eagerLimit,
INTRACHIP);
354 InterchipCFG_args.
setData(1, init_msg_size, latency, inverse_bandwidth, o, O, g, G, eagerLimit,
INTERCHIP);
355 InternodeCFG_args.
setData(1, init_msg_size, latency, inverse_bandwidth, o, O, g, G, eagerLimit,
INTERNODE);
356 InterclusterCFG_args.
setData(1, init_msg_size, latency, inverse_bandwidth, o, O, g, G, eagerLimit,
INTERCLUSTER);
360 if (collective_mode == 1) {
370 std::cout <<
"Rank : " << process_Rank <<
" completed network configuration" << std::endl;
380 std::vector<real_t> hosqueueOpync(numRanks);
382 if (process_Rank == 0)
384 std::cout <<
"\n----------------------------------------------------------------" << std::endl;
385 std::cout <<
"DisCosTiC (Distributed CosT in Cluster):" << std::endl;
386 std::cout <<
"----------------------------------------------------------------" << std::endl;
387 std::cout <<
"Version: v2.0.0" << std::endl;
388 auto timestamp = system_clock::to_time_t(system_clock::now());
389 std::cout <<
"timestamp: " << ctime(×tamp);
390 std::cout <<
"\n\n----------------------------------------------------------------" << std::endl;
391 std::cout <<
"GENERATED DEPENDENCY GRAPH (Directed Acyclic Benchmark (DisCosTiC)):" << std::endl;
392 std::cout <<
"----------------------------------------------------------------" << std::endl;
393 std::cout <<
"number of timesteps: " << nunmberofiter << std::endl;
394 std::cout <<
"OneWay_PosDist communetworkation pattern: number of Operation_t (send or recv or exec) per timestep per rank: " << 1 << std::endl;
395 std::cout <<
"\n\n----------------------------------------------------------------" << std::endl;
396 std::cout <<
"COMMUNICATION PERFORMANCE MODEL (Network+Systemtopology):" << std::endl;
397 std::cout <<
"----------------------------------------------------------------" << std::endl;
398 std::cout <<
"number of processes: " << numRanks << std::endl;
399 std::cout <<
"number of chips: " << numCores << std::endl;
400 std::cout <<
"SIMPLE LATENCY-BANDWIDTH MODEL:: intra chip latency [ns]: " << latency[
INTRACHIP]
401 <<
", inter chip latency [ns]: " << latency[
INTERCHIP]
402 <<
", inter node latency [ns]: " << latency[
INTERNODE]
403 <<
", inter cluster latency [ns]: " << latency[
INTERCLUSTER] << std::endl;
404 std::cout <<
"SIMPLE LATENCY-BANDWIDTH MODEL:: intra chip inverse bandwidth [s/GB]: " << inverse_bandwidth[
INTRACHIP]
405 <<
", inter chip inverse bandwidth [s/GB]: " << inverse_bandwidth[
INTERCHIP]
406 <<
", inter node inverse bandwidth [s/GB]: " << inverse_bandwidth[
INTERNODE]
407 <<
", inter cluster inverse bandwidth [s/GB]: " << inverse_bandwidth[
INTERCLUSTER] << std::endl;
408 std::cout <<
"LogGP MODEL additional parameters (Intra chip) :: o [ns]: " << o[
INTRACHIP] <<
", g [ns]: " << g[
INTRACHIP] <<
", O [ns]: " << O[
INTRACHIP] <<
", G [ns]: " << G[
INTRACHIP] << std::endl;
409 std::cout <<
"LogGP MODEL additional parameters (Inter chip) :: o [ns]: " << o[
INTERCHIP] <<
", g [ns]: " << g[
INTERCHIP] <<
", O [ns]: " << O[
INTERCHIP] <<
", G [ns]: " << G[
INTERCHIP] << std::endl;
410 std::cout <<
"LogGP MODEL additional parameters (Inter node) :: o [ns]: " << o[
INTERNODE] <<
", g [ns]: " << g[
INTERNODE] <<
", O [ns]: " << O[
INTERNODE] <<
", G [ns]: " << G[
INTERNODE] << std::endl;
413 std::cout <<
"Eager limit: " << eagerLimit <<
" bytes" << std::endl;
414 std::cout <<
"num_synchronizations: " << hosqueueOpync.size() << std::endl;
417 MPI_Barrier(MPI_COMM_WORLD);
428 Timevec2T oSuccessor(numRanks), grecvSuccessor(numRanks), gsendSuccessor(numRanks);
431 oSuccessor[rank].resize(numCores);
432 grecvSuccessor[rank].resize(numNetworks);
433 gsendSuccessor[rank].resize(numNetworks);
434 std::fill(oSuccessor[rank].begin(), oSuccessor[rank].end(), 0);
435 std::fill(grecvSuccessor[rank].begin(), grecvSuccessor[rank].end(), 0);
436 std::fill(gsendSuccessor[rank].begin(), gsendSuccessor[rank].end(), 0);
449 vec1T finishedRankList;
451 vec1T new_CompRankList;
457 static bool initialized;
462 high_resolution_clock::time_point startTime, endTime;
463 startTime = high_resolution_clock::now();
465 srand(
time(NULL) + process_Rank + 5);
480 double collective_data_storage[72][2];
481 uint8_t collective_algorithm = 0;
488 if (process_Rank != 0)
494 DisCosTiCOps += rankLocalIt->
getNumOps();
497 iter = nunmberofiter;
498 for (
auto &op : operations)
500 if (rank == N_process_Rank)
502 operation_og =
copy(op, operation_og);
505 if (op.bufSize == 0) {
506 std::cerr <<
"[WARNING] Rank " << N_process_Rank <<
" has op with zero bufSize. Type: " << +op.type << std::endl;
510 if (op.bufSize != operation_og.
bufSize) {
511 std::cerr <<
"[MISMATCH] Rank " << N_process_Rank
512 <<
" bufSize mismatch: op.bufSize = " << op.bufSize
513 <<
", operation_og.bufSize = " << operation_og.
bufSize << std::endl;
533 std::cerr <<
"YET NOT IMPLEMENTED!" << std::endl;
536 execTime = op.bufSize;
540 MPI_Barrier(newcomm);
543 std::cout <<
"Rank : " << N_process_Rank <<
" starting simulation" << std::endl;
551 rank = N_process_Rank;
556 int flag_blocking = 0;
557 int flag_blocking_2 = 0;
558 int flag_blocking_3 = 0;
559 int flag_blocking_4 = 0;
561 int flag_blocking_continuous = 0;
563 double nonoverlap = 0;
569 double gather_buff_comp[2][N_size_Of_Cluster];
570 double gather_buff_optime[N_size_Of_Cluster];
571 double overlap_time[N_size_Of_Cluster];
572 long int last_idle_time = 0, max_idle_time = 0;
574 int flag_recv_5 = 0, flag_check = 0, barrier_flag = 0;
576 for (
int counter = 0; counter < N_size_Of_Cluster; ++counter)
578 gather_buff_optime[counter] = 0.0;
582 std::cout <<
"Rank : " << N_process_Rank <<
" properties allocated" << std::endl;
593 if ((timeOld - operation.
time) != 0)
596 COMPCount = computingProcessesCount;
598 new_CompRankList = CompRankList;
601 timeOld = operation.
time;
607 const std::size_t buffersize_n =
608 std::max<std::size_t>(2000,
static_cast<std::size_t
>(COMPCount) + 1);
609 std::vector<DisCosTiC_Timetype> buffersize(buffersize_n, execTime);
611 auto it = std::find(new_CompRankList.begin(), new_CompRankList.end(), operation.
rank);
613 if ((operation.
time != 0) && (it != new_CompRankList.end()))
617 operation.
time = operation.
time - execTime;
621 operation.
time = operation.
time - execTime + buffersize[COMPCount - 1];
625 MPI_Allgather(&operation.
time, 1, MPI_DOUBLE, gather_buff_optime, 1, MPI_DOUBLE, newcomm);
627 std::cout <<
"Rank : " << N_process_Rank <<
" in iter : " << iter_num <<
" MPI_Allgather completed" << std::endl;
631 switch (operation.
type)
639 std::cout <<
"Rank : " << N_process_Rank <<
" in iter : " << iter_num <<
" in COMP" << std::endl;
642 flag_blocking_continuous = 0;
645 computingProcessesCount++;
646 CompRankList.push_back(operation.
rank);
647 if (oSuccessor[operation.
rank][operation.
node] <= operation.
time)
653 last_idle_time = operation.
time - last_idle_time;
655 MPI_Allgather(&operation.
time, 1, MPI_DOUBLE, gather_buff_comp[0], 1, MPI_DOUBLE, newcomm);
656 MPI_Allgather(&oSuccessor[operation.
rank][operation.
node], 1, MPI_DOUBLE, gather_buff_comp[1], 1, MPI_DOUBLE, newcomm);
659 std::cout <<
"Rank : " << N_process_Rank <<
" in iter : " << iter_num <<
" gathered data for BCM" << std::endl;
664 int startComputingProcess = 0, endComputingProcess = 0, overlappComputingProcess = 0, fasterContentionModel = 0;
672 double overlap_penalty_2 = (
arch_name.compare(
"A64FX") == 0) ? 0.1 : 0.01;
674 overlap_time[rank] = operation.
bufSize;
680 for (
int loop = 0; loop < N_size_Of_Cluster && (
arch_name.compare(
"A64FX") == 0); ++loop)
691 if ((gather_buff_comp[
START][rank] <= gather_buff_comp[
START][loop] && (gather_buff_comp[
START][rank] + projected_comp_time) >= gather_buff_comp[
START][loop]) || (gather_buff_comp[
START][rank] <= (gather_buff_comp[
END][loop] + projected_comp_time) && (gather_buff_comp[
START][rank] + projected_comp_time) >= (gather_buff_comp[
END][loop] + projected_comp_time)))
698 if (gather_buff_comp[
START][rank] <= (gather_buff_comp[
START][loop] + projected_comp_time) && (gather_buff_comp[
START][rank] + projected_comp_time) >= (gather_buff_comp[
START][loop] + projected_comp_time))
700 overlappComputingProcess++;
708 fasterContentionModel = 1;
709 operation.
bufSize = projected_comp_time;
712 for (
int loop = 0; loop < N_size_Of_Cluster && (fasterContentionModel == 0); ++loop)
723 if ((gather_buff_comp[
START][rank] <= gather_buff_comp[
START][loop] && gather_buff_comp[
END][rank] >= gather_buff_comp[
START][loop]) || (gather_buff_comp[
START][rank] <= gather_buff_comp[
END][loop] && gather_buff_comp[
END][rank] >= gather_buff_comp[
END][loop]))
725 if (gather_buff_comp[
START][rank] <= gather_buff_comp[
START][loop] && gather_buff_comp[
END][rank] >= gather_buff_comp[
START][loop])
727 overlap_time[loop] = gather_buff_comp[
END][rank] - gather_buff_comp[
START][loop] < 0 ? 0 : gather_buff_comp[
END][rank] - gather_buff_comp[
START][loop];
729 else if (gather_buff_comp[
START][rank] <= gather_buff_comp[
END][loop] && gather_buff_comp[
END][rank] >= gather_buff_comp[
END][loop])
731 overlap_time[loop] = gather_buff_comp[
END][loop] - gather_buff_comp[
START][rank] < 0 ? 0 : gather_buff_comp[
END][loop] - gather_buff_comp[
START][rank];
735 nonoverlap = (overlap_time[rank] - overlap_time[loop] < 50000 ? 0 : overlap_time[rank] - overlap_time[loop]);
737 operation.
bufSize -= nonoverlap * nonoverlap_penalty_1;
743 double loop_computing_time = gather_buff_comp[
END][loop] - gather_buff_comp[
START][loop];
744 int overlapp_pieces = last_idle_time / loop_computing_time;
746 double new_loop_start_time = gather_buff_comp[
START][loop];
747 double new_loop_end_time = gather_buff_comp[
END][loop];
748 if (gather_buff_comp[
START][loop] < gather_buff_comp[
START][rank])
751 if ((abs(rank - loop) %
task_per_node > overlapp_pieces) && last_idle_time > max_idle_time)
754 for (
int k = 1; k <= overlapp_pieces; ++k)
756 double temp = new_loop_start_time;
757 new_loop_start_time = new_loop_end_time;
758 new_loop_end_time += ((new_loop_end_time - temp) * 0.95);
761 new_loop_start_time += overlapp_pieces + 1 * commTime;
762 new_loop_end_time += overlapp_pieces + 1 * commTime;
766 new_loop_start_time = gather_buff_comp[
START][loop] + (((int(ceil(abs(loop - rank))) %
task_per_node)) * last_idle_time) + (((abs(loop - rank) %
task_per_node)) * commTime);
767 new_loop_end_time = gather_buff_comp[
END][loop] + (((int(ceil(abs(loop - rank))) %
task_per_node)) * last_idle_time) + (((abs(loop - rank) %
task_per_node)) * commTime);
770 if ((gather_buff_comp[
START][rank] <= new_loop_start_time && gather_buff_comp[
END][rank] >= new_loop_start_time) || (gather_buff_comp[
START][rank] <= new_loop_end_time && gather_buff_comp[
END][rank] >= new_loop_end_time))
772 if (gather_buff_comp[
START][rank] <= new_loop_start_time && gather_buff_comp[
END][rank] >= new_loop_start_time)
774 overlap_time[loop] = gather_buff_comp[
END][rank] - new_loop_start_time < 0 ? 0 : gather_buff_comp[
END][rank] - new_loop_start_time;
776 else if (gather_buff_comp[
START][rank] <= new_loop_end_time && gather_buff_comp[
END][rank] >= new_loop_end_time)
778 overlap_time[loop] = new_loop_end_time - gather_buff_comp[
START][rank] < 0 ? 0 : new_loop_end_time - gather_buff_comp[
START][rank];
781 nonoverlap = (overlap_time[rank] - overlap_time[loop] < 50000 ? 0 : overlap_time[rank] - overlap_time[loop]);
783 operation.
bufSize -= nonoverlap * overlap_penalty_2;
790 else if (gather_buff_comp[
START][loop] > gather_buff_comp[
START][rank])
793 if ((abs(rank - loop) %
task_per_node > overlapp_pieces) && last_idle_time > max_idle_time)
796 for (
int k = 1; k <= overlapp_pieces; ++k)
798 double temp = new_loop_start_time;
799 new_loop_start_time = new_loop_end_time;
800 new_loop_end_time += ((new_loop_end_time - temp) * 0.95);
803 new_loop_start_time += overlapp_pieces + 1 * commTime;
804 new_loop_end_time += overlapp_pieces + 1 * commTime;
808 new_loop_start_time = gather_buff_comp[
START][loop] - (((int(ceil(abs(loop - rank))) %
task_per_node)) * last_idle_time) - (((abs(loop - rank) %
task_per_node)) * commTime);
809 new_loop_end_time = gather_buff_comp[
END][loop] - (((int(ceil(abs(loop - rank))) %
task_per_node)) * last_idle_time) - (((abs(loop - rank) %
task_per_node)) * commTime);
812 if ((gather_buff_comp[
START][rank] <= new_loop_start_time && gather_buff_comp[
END][rank] >= new_loop_start_time) || (gather_buff_comp[
START][rank] <= new_loop_end_time && gather_buff_comp[
END][rank] >= new_loop_end_time))
814 if (gather_buff_comp[
START][rank] <= new_loop_start_time && gather_buff_comp[
END][rank] >= new_loop_start_time)
816 overlap_time[loop] = gather_buff_comp[
END][rank] - new_loop_start_time < 0 ? 0 : gather_buff_comp[
END][rank] - new_loop_start_time;
818 else if (gather_buff_comp[
START][rank] <= new_loop_end_time && gather_buff_comp[
END][rank] >= new_loop_end_time)
820 overlap_time[loop] = new_loop_end_time - gather_buff_comp[
START][rank] < 0 ? 0 : new_loop_end_time - gather_buff_comp[
START][rank];
823 nonoverlap = (overlap_time[rank] - overlap_time[loop] < 50000 ? 0 : overlap_time[rank] - overlap_time[loop]);
825 operation.
bufSize -= nonoverlap * overlap_penalty_2;
838 std::cout <<
"Rank : " << N_process_Rank <<
" in iter : " << iter_num <<
" finished BCM" << std::endl;
844 if (!initialized && (operation.
rank == desync_rank) && (iter_num == desync_timestep) && (idlewave_feature == 1))
851 std::cout <<
"Rank : " << N_process_Rank <<
" in iter : " << iter_num <<
" injected delay to create idle wave" << std::endl;
860 std::cout <<
"Rank : " << N_process_Rank <<
" in iter : " << iter_num <<
" AST graph set and unSet operation done" << std::endl;
862 finishedRankList.push_back(operation.
rank);
865 std::string eventName;
866 std::string eventColor;
870 eventColor =
"cq_build_failed";
875 eventColor =
"cq_build_running";
879 traceVisualizer2.
args(
"CPU" + std::to_string(operation.
node), operation.
rank);
880 traceVisualizer2.
completeEvents(eventName, operation.
rank, traceVisualizer2.
arc[(
"CPU" + std::to_string(operation.
node))], std::to_string(operation.
time / 1000), std::to_string(((operation.
time + operation.
bufSize) - operation.
time) / 1000), eventColor);
891 max_idle_time = std::max(max_idle_time, last_idle_time);
892 last_idle_time = operation.
time;
895 std::cout <<
"Rank : " << N_process_Rank <<
" in iter : " << iter_num <<
" finished inserting data in DMS file and completed COMP" << std::endl;
903 CompRankList.erase(std::remove(CompRankList.begin(), CompRankList.end(), operation.
rank), CompRankList.end());
907 std::cout <<
"Rank : " << N_process_Rank <<
" in iter : " << iter_num <<
" starting SEND" << std::endl;
917 for (
int i = 0; i < N_size_Of_Cluster; ++i)
919 max_time = std::max<double>(gather_buff_optime[i], max_time);
922 operation.
time = max_time;
923 oSuccessor[operation.
rank][operation.
node] = max_time;
924 gsendSuccessor[operation.
rank][operation.
network] = max_time;
934 if (std::max(oSuccessor[operation.
rank][operation.
node], gsendSuccessor[operation.
rank][operation.
network]) <= operation.
time && (operation.
bufSize != 0))
942 std::set<int> overlapp_ranks;
943 int overlapp_count = 1;
949 else if (target_node !=
node)
953 else if (target_socket !=
socket)
964 std::cout <<
"Rank : " << N_process_Rank <<
" in iter : " << iter_num <<
" communication properties determined" << std::endl;
966 for (
const auto &elem : overlapp_ranks)
968 if (((gather_buff_optime[elem] - 100000000) <= gather_buff_optime[rank] && gather_buff_optime[rank] <= (gather_buff_optime[elem] + 100000000)) && (elem != operation.
target))
974 std::cout <<
"SEND Rank : " << rank <<
" iteration : " << iter_num <<
" overlapp count : " << overlapp_count << std::endl;
977 IntrachipCFG_args.
setData(overlapp_count, operation.
bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit,
INTRACHIP);
978 InterchipCFG_args.
setData(overlapp_count, operation.
bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit,
INTERCHIP);
979 InternodeCFG_args.
setData(overlapp_count, operation.
bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit,
INTERNODE);
980 InterclusterCFG_args.
setData(overlapp_count, operation.
bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit,
INTERCLUSTER);
982 if (operation.
bufSize > eagerLimit)
985 std::cout <<
"Rank : " << N_process_Rank <<
" in iter : " << iter_num <<
" obeying RENDEZVOUS protocol" << std::endl;
991 double OpTime = ((operation.
bufSize) * inverse_bandwidth[comm_type]);
992 if (iter_num == 0 && flag_blocking == 0)
1000 operation.
time += ((N_size_Of_Cluster - rank - 1) * OpTime);
1001 oSuccessor[operation.
rank][operation.
node] += ((N_size_Of_Cluster - rank - 1) * OpTime);
1002 gsendSuccessor[operation.
rank][operation.
network] += ((N_size_Of_Cluster - rank - 1) * OpTime);
1007 if (barrier_flag == 1 && flag_blocking == 0 && (
CFG_args.
getValue<
int>(
"barrier_flag") == 1))
1010 operation.
time += ((N_size_Of_Cluster - rank - 1) * OpTime);
1011 oSuccessor[operation.
rank][operation.
node] += ((N_size_Of_Cluster - rank - 1) * OpTime);
1012 gsendSuccessor[operation.
rank][operation.
network] += ((N_size_Of_Cluster - rank - 1) * OpTime);
1014 else if (gather_buff_optime[rank] < gather_buff_optime[operation.
target])
1017 operation.
time = gather_buff_optime[operation.
target] + 2 * OpTime;
1018 oSuccessor[operation.
rank][operation.
node] = gather_buff_optime[operation.
target] + 2 * OpTime;
1019 gsendSuccessor[operation.
rank][operation.
network] = gather_buff_optime[operation.
target] + 2 * OpTime;
1024 if (barrier_flag == 1 && flag_blocking == 0 && (
CFG_args.
getValue<
int>(
"barrier_flag") == 1))
1029 if (flag_blocking_2 == 0)
1031 flag_blocking_2 = 1;
1035 operation.
time = gather_buff_optime[
primary_processes] + (((N_size_Of_Cluster - rank - 1) * OpTime) * 0.07);
1036 oSuccessor[operation.
rank][operation.
node] = gather_buff_optime[
primary_processes] + (((N_size_Of_Cluster - rank - 1) * OpTime) * 0.07);
1037 gsendSuccessor[operation.
rank][operation.
network] = gather_buff_optime[
primary_processes] + (((N_size_Of_Cluster - rank - 1) * OpTime) * 0.07);
1047 else if (flag_blocking_continuous == 0)
1049 flag_blocking_continuous = 1;
1075 if ((flag_blocking_3 == 0))
1078 flag_blocking_3 = 1;
1081 if (barrier_flag == 1 && flag_blocking == 0 && (
CFG_args.
getValue<
int>(
"barrier_flag") == 1))
1087 oSuccessor[operation.
rank][operation.
node] = gather_buff_optime[rank] + ((1.5 * (
primary_processes - rank) + 25) * OpTime);
1102 oSuccessor[operation.
rank][operation.
node] = gather_buff_optime[rank] + ((1.5 * (
primary_processes - rank) + 1) * OpTime);
1107 operation.
time = gather_buff_optime[rank];
1108 oSuccessor[operation.
rank][operation.
node] = gather_buff_optime[rank];
1109 gsendSuccessor[operation.
rank][operation.
network] = gather_buff_optime[rank];
1116 if (barrier_flag == 1 && flag_blocking == 0 && (
CFG_args.
getValue<
int>(
"barrier_flag") == 1))
1127 operation.
time += OpTime;
1128 oSuccessor[operation.
rank][operation.
node] += OpTime;
1129 gsendSuccessor[operation.
rank][operation.
network] += OpTime;
1161 if (comm_model ==
LOGGP)
1164 oSuccessor[operation.
rank][operation.
node] = operation.
time + o[comm_type] + ((operation.
bufSize - 1) * O[comm_type]);
1165 gsendSuccessor[operation.
rank][operation.
network] = operation.
time + g[comm_type] + ((operation.
bufSize - 1) * G[comm_type]);
1168 traceVisualizer2.
args(
"CPU" + std::to_string(operation.
node), operation.
rank);
1169 traceVisualizer2.
completeEvents(
"SEND", operation.
rank, traceVisualizer2.
arc[(
"CPU" + std::to_string(operation.
node))], std::to_string(operation.
time / 1000), std::to_string(((operation.
time + o[comm_type] + (operation.
bufSize - 1) * O[comm_type]) - operation.
time) / 1000),
"terrible");
1173 o[comm_type] + ((operation.
bufSize - 1) * O[comm_type]),
1183 oSuccessor[operation.
rank][operation.
node] = operation.
time + ((0.5 * (operation.
bufSize) * inverse_bandwidth[comm_type]) + latency[comm_type]);
1184 gsendSuccessor[operation.
rank][operation.
network] = operation.
time + ((0.5 * (operation.
bufSize) * inverse_bandwidth[comm_type]) + latency[comm_type]);
1187 traceVisualizer2.
args(
"CPU" + std::to_string(operation.
node), operation.
rank);
1188 traceVisualizer2.
completeEvents(
"SEND", operation.
rank, traceVisualizer2.
arc[(
"CPU" + std::to_string(operation.
node))], std::to_string(operation.
time / 1000), std::to_string(((0.5 * (operation.
bufSize) * inverse_bandwidth[comm_type]) + latency[comm_type]) / 1000),
"terrible");
1192 (0.5 * (operation.
bufSize) * inverse_bandwidth[comm_type]) + latency[comm_type],
1201 commTime += (oSuccessor[operation.
rank][operation.
node] - operation.
time);
1207 if (comm_model ==
LOGGP)
1209 operation.
time += o[comm_type] + latency[comm_type];
1213 operation.
time += ((0.5 * (operation.
bufSize) * inverse_bandwidth[comm_type]) + latency[comm_type]);
1218 int temp = oSuccessor[operation.
rank][operation.
node];
1224 arr[0] = operation.
time;
1229 arr[5] = operation.
target;
1230 arr[6] = operation.
rank;
1231 arr[7] = operation.
label;
1232 arr[8] = operation.
tag;
1233 arr[9] = operation.
node;
1235 arr[11] = operation.
type;
1238 request = MPI_REQUEST_NULL;
1243 MPI_Isend(&arr, 13, MPI_DOUBLE, operation.
target, operation.
tag, newcomm, &request);
1244 MPI_Wait(&request, &status);
1248 MPI_Send(&arr, 13, MPI_DOUBLE, operation.
target, operation.
tag, newcomm);
1251 std::cout <<
"Rank : " << N_process_Rank <<
" in iter : " << iter_num <<
" serialized data and sent to respective rank" << std::endl;
1254 finishedRankList.push_back(operation.
rank);
1259 std::cout <<
"Rank : " << N_process_Rank <<
" in iter : " << iter_num <<
" finished SEND" << std::endl;
1270 std::cout <<
"Rank : " << N_process_Rank <<
" in iter : " << iter_num <<
" starting RECV" << std::endl;
1276 double old_op_time = oSuccessor[operation.
rank][operation.
node];
1277 double brr[13] = {0};
1281 request = MPI_REQUEST_NULL;
1286 MPI_Irecv(&brr, 13, MPI_DOUBLE, MPI_ANY_SOURCE, operation.
tag, newcomm, &request);
1287 MPI_Wait(&request, &status);
1294 MPI_Recv(&brr, 13, MPI_DOUBLE, MPI_ANY_SOURCE, operation.
tag, newcomm, &status);
1298 operation.
time = std::max(operation.
time, brr[0]);
1303 operation.
target = brr[5];
1305 operation.
label = brr[7];
1306 operation.
tag = brr[8];
1307 operation.
node = brr[9];
1309 operation.
type = brr[11];
1310 oSuccessor[operation.
rank][operation.
node] = brr[12];
1314 std::cout <<
"Rank : " << N_process_Rank <<
" in iter : " << iter_num <<
" received buffer from respective rank and deserialized" << std::endl;
1321 std::set<int> overlapp_ranks;
1322 int overlapp_count = 1;
1328 else if (target_node !=
node)
1332 else if (target_socket !=
socket)
1342 for (
const auto &elem : overlapp_ranks)
1344 if (((gather_buff_optime[elem] - 5000) <= gather_buff_optime[rank] && gather_buff_optime[rank] <= (gather_buff_optime[elem] + 5000)) && (elem != brr[6]))
1350 std::cout <<
"RECV Rank : " << rank <<
" iteration : " << iter_num <<
" overlapp count : " << overlapp_count << std::endl;
1353 IntrachipCFG_args.
setData(overlapp_count, operation.
bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit,
INTRACHIP);
1354 InterchipCFG_args.
setData(overlapp_count, operation.
bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit,
INTERCHIP);
1355 InternodeCFG_args.
setData(overlapp_count, operation.
bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit,
INTERNODE);
1356 InterclusterCFG_args.
setData(overlapp_count, operation.
bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit,
INTERCLUSTER);
1358 if (operation.
bufSize > eagerLimit)
1361 std::cout <<
"Rank : " << N_process_Rank <<
" in iter : " << iter_num <<
" obeying RENDEZVOUS protocol" << std::endl;
1367 operation.
time = old_op_time;
1368 oSuccessor[operation.
rank][operation.
node] = old_op_time;
1369 gsendSuccessor[operation.
rank][operation.
network] = old_op_time;
1373 flag_blocking_4 = 1;
1374 operation.
time = old_op_time;
1375 oSuccessor[operation.
rank][operation.
node] = old_op_time;
1376 gsendSuccessor[operation.
rank][operation.
network] = old_op_time;
1381 double update_time = 0;
1384 update_time = (operation.
bufSize * inverse_bandwidth[comm_type]);
1388 update_time = 0.5 * (operation.
bufSize * inverse_bandwidth[comm_type]);
1391 if (comm_model ==
LOGGP)
1393 oSuccessor[operation.
rank][operation.
node] = operation.
time + o[comm_type] + std::max((operation.
bufSize - 1) * O[comm_type], (operation.
bufSize - 1) * G[comm_type]);
1394 grecvSuccessor[operation.
rank][operation.
network] = operation.
time + g[comm_type] + ((operation.
bufSize - 1) * G[comm_type]);
1398 oSuccessor[operation.
rank][operation.
node] = operation.
time + (update_time);
1399 gsendSuccessor[operation.
rank][operation.
network] = operation.
time + (update_time);
1401 commTime += (oSuccessor[operation.
rank][operation.
node] - operation.
time);
1405 finishedRankList.push_back(operation.
rank);
1408 std::cout <<
"[RECV ==> SATISFY] local non-blocking dependency" << std::endl;
1409 std::cout <<
"Unexpected message queue bufSize: " << UMQ.size() << std::endl;
1410 std::cout <<
"matching queue search for src " << operation.
target <<
" tag " << operation.
tag << std::endl;
1414 std::cout <<
" [RECV ==> FOUND] in local unexpected queue" << std::endl;
1417 if (comm_model ==
LOGGP)
1420 traceVisualizer2.
args(
"CPU" + std::to_string(rank), operation.
target);
1421 traceVisualizer2.
args(
"CPU" + std::to_string(rank), operation.
rank);
1422 traceVisualizer2.
flowEventBegin(
"Name:startMSG", operation.
target, traceVisualizer2.
arc[(
"CPU" + std::to_string(0))], std::to_string((operation.
starttime + o[comm_type]) / 1000), flow_id);
1423 traceVisualizer2.
flowEventEnd(
"Name:endMSG", operation.
rank, traceVisualizer2.
arc[(
"CPU" + std::to_string(0))], std::to_string(operation.
time / 1000), flow_id);
1426 traceVisualizer2.
args(
"CPU" + std::to_string(operation.
node), operation.
rank);
1427 traceVisualizer2.
completeEvents(
"RECV", operation.
rank, traceVisualizer2.
arc[(
"CPU" + std::to_string(operation.
node))], std::to_string((operation.
time + (operation.
bufSize - 1) * G[comm_type] - (operation.
bufSize - 1) * O[comm_type]) / 1000), std::to_string(((operation.
time + o[comm_type] + std::max((operation.
bufSize - 1) * O[comm_type], (operation.
bufSize - 1) * G[comm_type])) - (operation.
time + (operation.
bufSize - 1) * G[comm_type] - (operation.
bufSize - 1) * O[comm_type])) / 1000),
"terrible");
1432 const double recv_start = operation.
time + (operation.
bufSize - 1) * G[comm_type] - (operation.
bufSize - 1) * O[comm_type];
1433 const double recv_end = operation.
time + o[comm_type] + std::max((operation.
bufSize - 1) * O[comm_type], (operation.
bufSize - 1) * G[comm_type]);
1437 traceVisualizerItac.
recv(operation.
rank, src, recv_start,
1438 recv_end - recv_start, operation.
bufSize, operation.
tag);
1449 traceVisualizer2.
args(
"CPU" + std::to_string(0), operation.
target);
1450 traceVisualizer2.
args(
"CPU" + std::to_string(0), operation.
rank);
1451 traceVisualizer2.
flowEventBegin(
"Name:startMSG", operation.
target, traceVisualizer2.
arc[(
"CPU" + std::to_string(0))], std::to_string((operation.
starttime) / 1000), flow_id);
1452 traceVisualizer2.
flowEventEnd(
"Name:endMSG", operation.
rank, traceVisualizer2.
arc[(
"CPU" + std::to_string(0))], std::to_string(operation.
time / 1000), flow_id);
1455 traceVisualizer2.
args(
"CPU" + std::to_string(operation.
node), operation.
rank);
1456 traceVisualizer2.
completeEvents(
"RECV", operation.
rank, traceVisualizer2.
arc[(
"CPU" + std::to_string(operation.
node))], std::to_string((operation.
time) / 1000), std::to_string((update_time / 1000)),
"terrible");
1462 traceVisualizerItac.
recv(operation.
rank, src, operation.
time,
1463 update_time, operation.
bufSize, operation.
tag);
1472 finishedRankList.push_back(operation.
rank);
1474 if ((desync_feature == 1) && (iter_num >= 0 && iter_num < 200))
1477 std::cout <<
"Rank : " << N_process_Rank <<
" in iter : " << iter_num <<
" desynchronizing simulation by injecting noise" << std::endl;
1479 oSuccessor[operation.
rank][operation.
node] += rand() % 500000;
1483 std::cout <<
"Rank : " << N_process_Rank <<
" in iter : " << iter_num <<
" finished RECV" << std::endl;
1491 std::cout <<
"Rank : " << N_process_Rank <<
" in iter : " << iter_num <<
" starting COLLECTIVE" << std::endl;
1498 uint8_t algorithm = operation.
algorithm;
1499 collective_algorithm = algorithm;
1500 collective_type = collectiveType;
1503 double collective_entry_time = operation.
time;
1504 double local_entry_time = collective_entry_time;
1505 MPI_Allreduce(&local_entry_time, &collective_entry_time, 1, MPI_DOUBLE, MPI_MAX, newcomm);
1529 IntrachipCFG_args.
setData(1, operation.
bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit,
INTRACHIP);
1530 InterchipCFG_args.
setData(1, operation.
bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit,
INTERCHIP);
1531 InternodeCFG_args.
setData(1, operation.
bufSize, latency, inverse_bandwidth, o, O, g, G, eagerLimit,
INTERNODE);
1547 inverse_bandwidth[comm_type],
1554 if (collective_mode == 1) {
1555 gather_buff_comp[0][operation.
rank] = collective_entry_time;
1556 gather_buff_comp[1][operation.
rank] = collective_duration;
1560 operation.
time = completion_time;
1563 if (oSuccessor[operation.
rank][operation.
node] < operation.
time)
1565 oSuccessor[operation.
rank][operation.
node] = operation.
time;
1568 finishedRankList.push_back(operation.
rank);
1572 std::string collectiveName =
"Name:COLLECTIVE_";
1573 switch(collectiveType) {
1577 default: collectiveName +=
"UNKNOWN";
break;
1581 traceVisualizer2.
args(
"CPU0", operation.
rank);
1583 traceVisualizer2.
arc[
"CPU0"],
1584 std::to_string(collective_entry_time / 1000),
1585 std::to_string((operation.
time - collective_entry_time) / 1000),
"terrible");
1589 std::string itacColl =
"BCAST";
1590 switch (collectiveType)
1596 itacColl =
"Reduce";
1599 itacColl =
"Allreduce";
1602 itacColl =
"Collective";
1605 traceVisualizerItac.
collective(operation.
rank, collective_entry_time,
1606 operation.
time - collective_entry_time, itacColl);
1611 std::cout <<
"Rank : " << N_process_Rank <<
" in iter : " << iter_num
1612 <<
" finished COLLECTIVE at time " << operation.
time << std::endl;
1618 std::cout <<
"YET NOT IMPLEMENTED! : " << (int)operation.
type << std::endl;
1622 std::sort(finishedRankList.begin(), finishedRankList.end());
1623 finishedRankList.erase(std::unique(finishedRankList.begin(), finishedRankList.end()), finishedRankList.end());
1626 std::cout <<
"Rank : " << N_process_Rank <<
" finished iter : " << iter_num << std::endl;
1632 for (
auto &rank : finishedRankList)
1638 if (operations.size() != 0)
1639 operations_queue.insert(operations_queue.end(), operations.begin(), operations.end());
1641 if (operations_queue.size() != 0)
1643 auto op = operations_queue.front();
1644 operations_queue.erase(operations_queue.begin());
1650 op.time = oSuccessor[rank][op.node];
1657 op.time = std::max(oSuccessor[rank][op.node], gsendSuccessor[rank][op.network]);
1664 op.time = oSuccessor[rank][op.node];
1671 op.time = oSuccessor[rank][op.node];
1673 std::cout <<
"final COLLECTIVE op for rank " << rank <<
" at time " << op.time << std::endl;
1678 std::cerr <<
"YET NOT IMPLEMENTED!" << std::endl;
1683 std::cout <<
"Rank : " << N_process_Rank <<
" in iter : " << iter_num <<
" set new operation" << std::endl;
1698 std::cout <<
"Rank : " << N_process_Rank <<
" in iter : " << iter_num <<
" exiting cycle as no new operation in queue" << std::endl;
1706 operation_og.
time = oSuccessor[operation.
rank][operation.
node];
1707 operation =
copy(operation_og, operation);
1708 }
while (iter_num < iter);
1711 if (collective_mode == 1) {
1712 double collective_timing[4];
1713 collective_timing[0] = gather_buff_comp[0][rank];
1714 collective_timing[1] = gather_buff_comp[1][rank];
1715 collective_timing[2] = collective_algorithm;
1716 collective_timing[3] = collective_type;
1717 MPI_Send(&collective_timing, 4, MPI_DOUBLE, 0, 100, MPI_COMM_WORLD);
1721 *
finalize(oSuccessor[rank][operation.
node], numRanks, numOpsInQueue, rank, operation.
node, brr);
1722 MPI_Send(&brr, 5, MPI_DOUBLE, 0, 1, MPI_COMM_WORLD);
1725 std::cout <<
"Rank : " << process_Rank <<
" send data to MASTER RANK" << std::endl;
1732 else if (process_Rank == 0)
1737 if (collective_mode == 1) {
1738 for (
int i = 0; i < numRanks; i++) {
1739 double timing_data[4];
1740 MPI_Recv(&timing_data, 4, MPI_DOUBLE, i+1, 100, MPI_COMM_WORLD, &status);
1741 collective_data_storage[i][0] = timing_data[0];
1742 collective_data_storage[i][1] = timing_data[1];
1744 collective_algorithm = (uint8_t)timing_data[2];
1751 while (test.size() < numRanks)
1754 MPI_Recv(&brr, 5, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
1756 if (status.MPI_TAG == 1)
1758 test.emplace(status.MPI_SOURCE);
1759 oSuccessor[brr[3]][brr[4]] = brr[0];
1760 numOpsInQueue = brr[2];
1765 std::cout <<
"Rank : " << process_Rank <<
" MASTER RANK received data from all simulation processes" << std::endl;
1772 ierr = MPI_Comm_rank(MPI_COMM_WORLD, &process_Rank);
1773 MPI_Barrier(MPI_COMM_WORLD);
1780 std::cout <<
"Rank : " << process_Rank <<
" All simulation processes completed simulation" << std::endl;
1783 if (process_Rank == 0)
1790 endTime = high_resolution_clock::now();
1794 std::cout <<
"\n\nDisCosTiC PERFORMANCE:" << std::endl;
1795 std::cout <<
"----------------------------------------------------------------" << std::endl;
1796 std::cout <<
"Number of operations Runtime [s] Speed [operations/ns]" << std::endl;
1797 std::cout <<
"----------------------------------------------------------------" << std::endl;
1798 std::cout << numOpsInQueue <<
" \t\t\t\t" <<
DisCosTiC / 1e9 <<
" \t\t" << numOpsInQueue /
DisCosTiC << std::endl;
1815 throw std::runtime_error(
"ERROR: QUEUES NOT EMPTY!\n");
1819 if (collective_mode == 1) {
1820 std::cout <<
"\n\nCOLLECTIVE OPERATION TIMING DETAILS:" << std::endl;
1821 std::cout <<
"-----------------------------------------------------------------" << std::endl;
1822 std::cout <<
"rank entry_time[ns] duration[ns] completion[ns] stage" << std::endl;
1823 std::cout <<
"-----------------------------------------------------------------" << std::endl;
1826 double all_collective_data[2 * numRanks];
1827 for (
int i = 0; i < numRanks; i++) {
1828 all_collective_data[i] = collective_data_storage[i][0];
1829 all_collective_data[numRanks + i] = collective_data_storage[i][1];
1834 &all_collective_data[2 * numRanks]);
1837 for (
int i = 0; i < numRanks; i++) {
1841 int stage = (stage_time > 0) ?
static_cast<int>(duration / stage_time) : 0;
1843 std::cout << std::setw(4) << i <<
" "
1844 << std::setw(13) << std::fixed << std::setprecision(0) << entry_time <<
" "
1845 << std::setw(11) << duration <<
" "
1846 << std::setw(13) << completion <<
" "
1847 << std::setw(5) << stage << std::endl;
1852 &all_collective_data[2 * numRanks]);
1854 &all_collective_data[2 * numRanks], 0.0) / numRanks;
1856 std::cout <<
"---------------------------------------------------------------" << std::endl;
1857 std::cout <<
"Collective Summary:" << std::endl;
1860 std::string type_name, algo_name;
1861 switch(collective_type) {
1863 type_name =
"BROADCAST";
1867 type_name =
"REDUCE";
1868 algo_name =
"Binomial Tree";
1871 type_name =
"ALLREDUCE";
1872 algo_name =
"Recursive Doubling";
1875 type_name =
"UNKNOWN";
1876 algo_name =
"Unknown";
1880 std::cout <<
" Type: " << type_name <<
" (" << algo_name <<
" Algorithm)" << std::endl;
1881 std::cout <<
" Root: 0" << std::endl;
1882 std::cout <<
" Participating ranks: " << numRanks << std::endl;
1883 std::cout <<
" Min duration: " << min_duration <<
" ns" << std::endl;
1884 std::cout <<
" Max duration: " << max_duration <<
" ns" << std::endl;
1885 std::cout <<
" Avg duration: " << avg_duration <<
" ns" << std::endl;
1890 stages = 2 *
static_cast<int>(std::ceil(std::log2(numRanks)));
1892 stages =
static_cast<int>(std::ceil(std::log2(numRanks)));
1894 std::cout <<
" Stages: " << stages << std::endl;
1895 std::cout <<
"---------------------------------------------------------------" << std::endl;
1898 std::cout <<
"\nCollective Performance Analysis (Multiple Sizes):" << std::endl;
1899 std::cout <<
"---------------------------------------------------------------" << std::endl;
1900 std::cout <<
"Size (KB) Min (μs) Avg (μs) Max (μs)" << std::endl;
1901 std::cout <<
"---------------------------------------------------------------" << std::endl;
1903 std::vector<int> test_sizes = {1024, 2048, 4096, 8192, 16384, 32768, 65536,
1904 131072, 262144, 524288, 1048576, 2097152,
1905 4194304, 8388608, 10485760};
1907 for (
int msg_size : test_sizes) {
1909 std::vector<DisCosTiC_Timetype> durations;
1927 IntrachipCFG_args.
setData(1, msg_size, temp_latency, temp_inverse_bandwidth, temp_o, temp_O, temp_g, temp_G, temp_eagerLimit,
INTRACHIP);
1928 }
else if (comm_type_analysis ==
INTERCHIP) {
1929 InterchipCFG_args.
setData(1, msg_size, temp_latency, temp_inverse_bandwidth, temp_o, temp_O, temp_g, temp_G, temp_eagerLimit,
INTERCHIP);
1931 InternodeCFG_args.
setData(1, msg_size, temp_latency, temp_inverse_bandwidth, temp_o, temp_O, temp_g, temp_G, temp_eagerLimit,
INTERNODE);
1934 for (
int rank = 0; rank < numRanks; rank++) {
1938 rank, 0, numRanks, msg_size,
1939 temp_latency[comm_type_analysis],
1940 temp_o[comm_type_analysis],
1941 temp_g[comm_type_analysis],
1942 temp_G[comm_type_analysis],
1943 collective_algorithm,
1944 temp_inverse_bandwidth[comm_type_analysis],
1947 durations.push_back(duration);
1953 DisCosTiC_Timetype avg_dur = std::accumulate(durations.begin(), durations.end(), 0.0) / numRanks;
1956 std::cout << std::setw(8) << (msg_size / 1024) <<
" "
1957 << std::setw(7) << std::fixed << std::setprecision(1) << (min_dur / 1000.0) <<
" "
1958 << std::setw(7) << (avg_dur / 1000.0) <<
" "
1959 << std::setw(7) << (max_dur / 1000.0) << std::endl;
1962 std::cout <<
"----------------------------------------------------------------" << std::endl;
1966 system((
"rm " +
CFG_args.
getValue<std::string>(
"chromevizfilename") +
".dms 2> /dev/null").c_str());
1968 system((
"cat " +
CFG_args.
getValue<std::string>(
"chromevizfilename") +
"{0.." + std::to_string(size_Of_Cluster - 2) +
"}.dms > " +
CFG_args.
getValue<std::string>(
"chromevizfilename") +
".dms").c_str());
1970 std::string filename =
CFG_args.
getValue<std::string>(
"chromevizfilename") +
".dms";
1972 ofs.open(filename, std::ios_base::app);
1975 <<
"\"Name\": \"Trace visualisation ofs DisCosTiC toolkit\", \n "
1976 <<
"\"Full form\": \"Distributed CosT in Cluster\", \n "
1977 <<
"\"Version\": \"v2.0.0 (second release)\", \n "
1978 <<
"\"Author\": \"Ayesha Afzal <ayesha.afzal@fau.de>\", \n "
1979 <<
"\"Copyright\": \"© 2023 HPC, FAU Erlangen-Nuremberg. All rights reserved\" \n "
1986 const std::string avt_base =
CFG_args.
getValue<std::string>(
"chromevizfilename");
1987 system((
"rm " + avt_base +
".avt 2> /dev/null").c_str());
1988 system((
"cat " + avt_base +
"{0.." + std::to_string(size_Of_Cluster - 2) +
"}.avt > " + avt_base +
".avt").c_str());
1989 system((
"rm " + avt_base +
"{0.." + std::to_string(size_Of_Cluster - 2) +
"}.avt 2> /dev/null").c_str());
1993 system(
"rm DisCosTiC.out 2> /dev/null");
1995 system(
"cat DisCosTiC*.out >> DisCosTiC.out");
1998 system((
"rm DisCosTiC{0.." + std::to_string(size_Of_Cluster - 2) +
"}.out 2> /dev/null").c_str());
1999 system((
"rm " +
CFG_args.
getValue<std::string>(
"chromevizfilename") +
"{0.." + std::to_string(size_Of_Cluster - 2) +
"}.dms 2> /dev/null").c_str());
2002 std::cout <<
"Rank : " << process_Rank <<
" wrote data to MASTER file and cleaned up files" << std::endl;
2005 delete DisCosTiC_AST;
2008 return EXIT_SUCCESS;