VA: log NUMA node of allocated rank(s)

3 files changed, 52 insertions, 32 deletions

diff --git a/VA/dimes-hetsim-hbm.sh b/VA/dimes-hetsim-hbm.sh
index 4b651c8..acb27b1 100755
--- a/VA/dimes-hetsim-hbm.sh
+++ b/VA/dimes-hetsim-hbm.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 
 cd baselines/cpu
 make -B NUMA=1
@@ -6,29 +6,23 @@ make -B NUMA=1
 mkdir -p log/$(hostname)
 fn=log/$(hostname)/$(date +%Y%m%d)-baseline
 
-# upstream uses 167772160 * int32 == 1.25 GiB for DPU version
-
-run_benchmark() {
-	eval "$@"
-	./va -i ${input_size} -a ${ram} -b ${ram} -c ${cpu} -t ${nr_threads} -w 0 -e 40
-	return $?
-}
-
-export -f run_benchmark
+# upstream uses 167772160 * 2 * int32 == 2.5 GiB input and 1.25 GiB output for DPU version
 
 (
 
+echo "single-node execution (1/2)" >&2
+
 parallel -j1 --eta --joblog ${fn}.1.joblog --resume --header : \
-	run_benchmark nr_threads={nr_threads} ram={ram} cpu={cpu} \
-	input_size={input_size} \
+	./va -i {input_size} -a {ram} -b {ram} -c {cpu} -t {nr_threads} -w 0 -e 20 \
 	::: nr_threads 1 2 4 8 12 16 \
 	::: cpu $(seq 0 7) \
 	::: ram $(seq 0 15) \
 	::: input_size 167772160
 
+echo "multi-node execution (2/2)" >&2
+
 parallel -j1 --eta --joblog ${fn}.2.joblog --resume --header : \
-	run_benchmark nr_threads={nr_threads} ram={ram} cpu={cpu} \
-	input_size={input_size} \
+	./va -i {input_size} -a {ram} -b {ram} -c {cpu} -t {nr_threads} -w 0 -e 20 \
 	::: nr_threads 32 48 64 96 128 \
 	::: cpu -1 \
 	::: ram $(seq 0 15) \
diff --git a/VA/dimes-hetsim-nmc.sh b/VA/dimes-hetsim-nmc.sh
index 97aef94..3d94fba 100755
--- a/VA/dimes-hetsim-nmc.sh
+++ b/VA/dimes-hetsim-nmc.sh
@@ -3,10 +3,11 @@
 mkdir -p log/$(hostname) baselines/cpu/log/$(hostname)
 fn=log/$(hostname)/$(date +%Y%m%d)
 
-# upstream uses 167772160 * int32 == 1.25 GiB for DPU version
+# upstream uses 167772160 * 2 * int32 == 2.5 GiB input and 1.25 GiB output for DPU version
 
 run_benchmark_nmc() {
-	eval "$@"
+	local "$@"
+	sudo limit_ranks_to_numa_node ${numa_rank}
 	if make -B NR_DPUS=${nr_dpus} NR_TASKLETS=${nr_tasklets} BL=10 WITH_ALLOC_OVERHEAD=1 WITH_LOAD_OVERHEAD=1 WITH_FREE_OVERHEAD=1; then
 		bin/host_code -w 0 -e 100 -i ${input_size} -x 1
 	fi
@@ -15,19 +16,22 @@ run_benchmark_nmc() {
 
 export -f run_benchmark_nmc
 
-run_benchmark_baseline() {
-	eval "$@"
-	./va -i ${input_size} -a ${ram} -b ${ram} -c ${cpu} -t $nr_threads -w 0 -e 40
-	return $?
-}
+(
 
-export -f run_benchmark_baseline
+echo "NMC single-node operation (1/2)" >&2
 
-(
+parallel -j1 --eta --joblog ${fn}.1.joblog --resume --header : \
+	run_benchmark_nmc nr_dpus={nr_dpus} nr_tasklets=16 input_size={input_size} numa_rank={numa_rank} \
+	::: numa_rank 0 1 \
+	::: nr_dpus 64 128 256 512 768 1024 \
+	::: input_size 167772160
+
+echo "NMC multi-node operation (2/2)" >&2
 
-parallel -j1 --eta --joblog ${fn}.joblog --resume --header : \
-	run_benchmark_nmc nr_dpus={nr_dpus} nr_tasklets=16 input_size={input_size} \
-	::: nr_dpus 64 128 256 512 768 1024 1536 2048 2304 \
+parallel -j1 --eta --joblog ${fn}.2.joblog --resume --header : \
+	run_benchmark_nmc nr_dpus={nr_dpus} nr_tasklets=16 input_size={input_size} numa_rank={numa_rank} \
+	::: numa_rank -1 \
+	::: nr_dpus 1536 2048 2304 \
 	::: input_size 167772160
 
 ) > ${fn}.txt
@@ -39,15 +43,19 @@ make -B NUMA=1
 
 (
 
+echo "CPU single-node operation (1/2)" >&2
+
 parallel -j1 --eta --joblog ${fn}.1.joblog --resume --header : \
-	run_benchmark_baseline nr_threads={nr_threads} input_size={input_size} ram={ram} cpu={cpu} \
+	./va -i {input_size} -a {ram} -b {ram} -c {cpu} -t {nr_threads} -w 0 -e 40 \
 	::: ram 0 1 \
 	::: cpu 0 1 \
 	::: nr_threads 1 2 4 8 12 16 32 \
 	::: input_size 167772160
 
+echo "CPU multi-node operation (1/2)" >&2
+
 parallel -j1 --eta --joblog ${fn}.2.joblog --resume --header : \
-	run_benchmark_baseline nr_threads={nr_threads} input_size={input_size} ram={ram} cpu={cpu} \
+	./va -i {input_size} -a {ram} -b {ram} -c {cpu} -t {nr_threads} -w 0 -e 40 \
 	::: ram any \
 	::: cpu -1 \
 	::: nr_threads 48 64 \
diff --git a/VA/host/app.c b/VA/host/app.c
index d363514..5fe3f61 100644
--- a/VA/host/app.c
+++ b/VA/host/app.c
@@ -29,10 +29,8 @@
 #include <dpu_probe.h>
 #endif
 
-#if WITH_DPUINFO
 #include <dpu_management.h>
 #include <dpu_target_macros.h>
-#endif
 
 // Pointer declaration
 static T* A;
@@ -73,6 +71,8 @@ int main(int argc, char **argv) {
     // Timer declaration
     Timer timer;
 
+    int numa_node_rank = -2;
+
     // Allocate DPUs and load binary
 #if !WITH_ALLOC_OVERHEAD
     DPU_ASSERT(dpu_alloc(NR_DPUS, NULL, &dpu_set));
@@ -144,6 +144,24 @@ int main(int argc, char **argv) {
         assert(nr_of_dpus == NR_DPUS);
 #endif
 
+        // int prev_rank_id = -1;
+        int rank_id = -1;
+        DPU_FOREACH (dpu_set, dpu) {
+            rank_id = dpu_get_rank_id(dpu_get_rank(dpu_from_set(dpu))) & DPU_TARGET_MASK;
+            if ((numa_node_rank != -2) && numa_node_rank != dpu_get_rank_numa_node(dpu_get_rank(dpu_from_set(dpu)))) {
+                numa_node_rank = -1;
+            } else {
+                numa_node_rank = dpu_get_rank_numa_node(dpu_get_rank(dpu_from_set(dpu)));
+            }
+            /*
+            if (rank_id != prev_rank_id) {
+                printf("/dev/dpu_rank%d @ NUMA node %d\n", rank_id, numa_node_rank);
+                prev_rank_id = rank_id;
+            }
+            */
+        }
+
+
         // Compute output on CPU (performance comparison and verification purposes)
         if(rep >= p.n_warmup) {
             start(&timer, 2, 0);
@@ -257,8 +275,8 @@ int main(int argc, char **argv) {
             if (rep >= p.n_warmup) {
                 printf("[::] VA-UPMEM | n_dpus=%d n_ranks=%d n_tasklets=%d e_type=%s block_size_B=%d n_elements=%d n_elements_per_dpu=%d",
                     nr_of_dpus, nr_of_ranks, NR_TASKLETS, XSTR(T), BLOCK_SIZE, input_size, input_size / NR_DPUS);
-                printf(" b_with_alloc_overhead=%d b_with_load_overhead=%d b_with_free_overhead=%d ",
-                    WITH_ALLOC_OVERHEAD, WITH_LOAD_OVERHEAD, WITH_FREE_OVERHEAD);
+                printf(" b_with_alloc_overhead=%d b_with_load_overhead=%d b_with_free_overhead=%d numa_node_rank=%d ",
+                    WITH_ALLOC_OVERHEAD, WITH_LOAD_OVERHEAD, WITH_FREE_OVERHEAD, numa_node_rank);
                 printf("| latency_alloc_us=%f latency_load_us=%f latency_cpu_us=%f latency_write_us=%f latency_kernel_us=%f latency_read_us=%f latency_free_us=%f",
                     timer.time[0],
                     timer.time[1],