BS NMC: log NUMA data

3 files changed, 68 insertions, 12 deletions

diff --git a/BS/dimes-hetsim-hbm.sh b/BS/dimes-hetsim-hbm.sh
index af1a124..b4a1d6d 100755
--- a/BS/dimes-hetsim-hbm.sh
+++ b/BS/dimes-hetsim-hbm.sh
@@ -28,34 +28,42 @@ export -f run_benchmark
 
 (
 
+echo "single-node execution, DPU ref (1/4)" >&2
+
 parallel -j1 --eta --joblog ${fn}.1.joblog --resume --header : \
 	run_benchmark i={i} nr_threads={nr_threads} ram={ram} cpu={cpu} \
 	input_size=${input_size_dpu} num_queries=${num_queries_dpu} \
-	::: i $(seq 1 20) \
+	::: i $(seq 1 5) \
 	::: nr_threads 1 2 4 8 12 16 \
 	::: cpu $(seq 0 7) \
 	::: ram $(seq 0 15)
 
+echo "single-node execution, HBM ref (2/4)" >&2
+
 parallel -j1 --eta --joblog ${fn}.2.joblog --resume --header : \
 	run_benchmark i={i} nr_threads={nr_threads} ram={ram} cpu={cpu} \
 	input_size=${input_size_hbm} num_queries=${num_queries_hbm} \
-	::: i $(seq 1 20) \
+	::: i $(seq 1 5) \
 	::: nr_threads 1 2 4 8 12 16 \
 	::: cpu $(seq 0 7) \
 	::: ram $(seq 0 15)
 
+echo "multi-node execution, DPU ref (3/4)" >&2
+
 parallel -j1 --eta --joblog ${fn}.3.joblog --resume --header : \
 	run_benchmark i={i} nr_threads={nr_threads} ram={ram} cpu={cpu} \
 	input_size=${input_size_dpu} num_queries=${num_queries_dpu} \
-	::: i $(seq 1 20) \
+	::: i $(seq 1 5) \
 	::: nr_threads 32 48 64 96 128 \
 	::: cpu -1 \
 	::: ram $(seq 0 15)
 
+echo "multi-node execution, HBM ref (4/4)" >&2
+
 parallel -j1 --eta --joblog ${fn}.4.joblog --resume --header : \
 	run_benchmark i={i} nr_threads={nr_threads} ram={ram} cpu={cpu} \
 	input_size=${input_size_hbm} num_queries=${num_queries_hbm} \
-	::: i $(seq 1 20) \
+	::: i $(seq 1 5) \
 	::: nr_threads 32 48 64 96 128 \
 	::: cpu -1 \
 	::: ram $(seq 0 15)
diff --git a/BS/dimes-hetsim-nmc.sh b/BS/dimes-hetsim-nmc.sh
index ec27ceb..7a2b12a 100755
--- a/BS/dimes-hetsim-nmc.sh
+++ b/BS/dimes-hetsim-nmc.sh
@@ -13,6 +13,7 @@ num_queries_dpu=1048576
 
 run_benchmark_nmc() {
 	local "$@"
+	sudo limit_ranks_to_numa_node ${numa_rank}
 	if make -B NR_DPUS=${nr_dpus} NR_TASKLETS=${nr_tasklets} WITH_ALLOC_OVERHEAD=1 WITH_LOAD_OVERHEAD=1 WITH_FREE_OVERHEAD=1 INPUT_SIZE=${input_size} PROBLEM_SIZE=${num_queries}; then
 		bin/bs_host -w 0 -e 100
 	fi
@@ -31,15 +32,37 @@ export -f run_benchmark_baseline
 
 (
 
+echo "NMC single-node upstream-ref (1/4)" >&2
+
 parallel -j1 --eta --joblog ${fn}.1.joblog --resume --header : \
-	run_benchmark_nmc nr_dpus={nr_dpus} nr_tasklets=16 \
+	run_benchmark_nmc nr_dpus={nr_dpus} nr_tasklets=16 numa_rank={numa_rank} \
 	num_queries=${num_queries_upstream} input_size=${input_size_upstream} \
-	::: nr_dpus 64 128 256 512 768 1024 1536 2048 2304
+	::: numa_rank 0 1 \
+	::: nr_dpus 64 128 256 512 768 1024
+
+echo "NMC multi-node upstream-ref (2/4)" >&2
+
+parallel -j1 --eta --joblog ${fn}.1.joblog --resume --header : \
+	run_benchmark_nmc nr_dpus={nr_dpus} nr_tasklets=16 numa_rank={numa_rank} \
+	num_queries=${num_queries_upstream} input_size=${input_size_upstream} \
+	::: numa_rank -1 \
+	::: nr_dpus 1536 2048 2304
+
+echo "NMC single-node DPU-ref (3/4)" >&2
+
+parallel -j1 --eta --joblog ${fn}.2.joblog --resume --header : \
+	run_benchmark_nmc nr_dpus={nr_dpus} nr_tasklets=16 numa_rank={numa_rank} \
+	num_queries=${num_queries_dpu} input_size=${input_size_dpu} \
+	::: numa_rank 0 1 \
+	::: nr_dpus 64 128 256 512 768 1024
+
+echo "NMC multi-node DPU-ref (4/4)" >&2
 
 parallel -j1 --eta --joblog ${fn}.2.joblog --resume --header : \
-	run_benchmark_nmc nr_dpus={nr_dpus} nr_tasklets=16 \
+	run_benchmark_nmc nr_dpus={nr_dpus} nr_tasklets=16 numa_rank={numa_rank} \
 	num_queries=${num_queries_dpu} input_size=${input_size_dpu} \
-	::: nr_dpus 64 128 256 512 768 1024 1536 2048 2304
+	::: numa_rank -1 \
+	::: nr_dpus 1536 2048 2304
 
 ) > ${fn}.txt
 
@@ -50,6 +73,8 @@ make -B NUMA=1
 
 (
 
+echo "CPU single-node upstream-ref (1/4)" >&2
+
 parallel -j1 --eta --joblog ${fn}.1.joblog --resume --header : \
 	run_benchmark_baseline i={i} nr_threads={nr_threads} ram={ram} cpu={cpu} \
 	num_queries=${num_queries_upstream} input_size=${input_size_upstream} \
@@ -58,6 +83,8 @@ parallel -j1 --eta --joblog ${fn}.1.joblog --resume --header : \
 	::: ram 0 1 \
 	::: nr_threads 1 2 4 8 12 16 32
 
+echo "CPU single-node DPU-ref (2/4)" >&2
+
 parallel -j1 --eta --joblog ${fn}.2.joblog --resume --header : \
 	run_benchmark_baseline i={i} nr_threads={nr_threads} ram={ram} cpu={cpu} \
 	num_queries=${num_queries_dpu} input_size=${input_size_dpu} \
@@ -66,6 +93,8 @@ parallel -j1 --eta --joblog ${fn}.2.joblog --resume --header : \
 	::: ram 0 1 \
 	::: nr_threads 1 2 4 8 12 16 32
 
+echo "CPU multi-node upstream-ref (3/4)" >&2
+
 parallel -j1 --eta --joblog ${fn}.3.joblog --resume --header : \
 	run_benchmark_baseline i={i} nr_threads={nr_threads} ram={ram} cpu={cpu} \
 	num_queries=${num_queries_upstream} input_size=${input_size_upstream} \
@@ -74,6 +103,8 @@ parallel -j1 --eta --joblog ${fn}.3.joblog --resume --header : \
 	::: ram 0 1 \
 	::: nr_threads 48 64
 
+echo "CPU multi-node DPU-ref (4/4)" >&2
+
 parallel -j1 --eta --joblog ${fn}.4.joblog --resume --header : \
 	run_benchmark_baseline i={i} nr_threads={nr_threads} ram={ram} cpu={cpu} \
 	num_queries=${num_queries_dpu} input_size=${input_size_dpu} \
diff --git a/BS/host/app.c b/BS/host/app.c
index 6d8db6a..85eb883 100644
--- a/BS/host/app.c
+++ b/BS/host/app.c
@@ -18,10 +18,8 @@
 #include <dpu_probe.h>
 #endif
 
-#if WITH_DPUINFO
 #include <dpu_management.h>
 #include <dpu_target_macros.h>
-#endif
 
 #define XSTR(x) STR(x)
 #define STR(x) #x
@@ -89,6 +87,8 @@ int main(int argc, char **argv) {
     // Timer declaration
     Timer timer;
 
+    int numa_node_rank = -2;
+
     // Allocate DPUs and load binary
 #if !WITH_ALLOC_OVERHEAD
     DPU_ASSERT(dpu_alloc(NR_DPUS, NULL, &dpu_set));
@@ -162,6 +162,23 @@ int main(int argc, char **argv) {
 		assert(nr_of_dpus == NR_DPUS);
 #endif
 
+		// int prev_rank_id = -1;
+		int rank_id = -1;
+		DPU_FOREACH (dpu_set, dpu) {
+			rank_id = dpu_get_rank_id(dpu_get_rank(dpu_from_set(dpu))) & DPU_TARGET_MASK;
+			if ((numa_node_rank != -2) && numa_node_rank != dpu_get_rank_numa_node(dpu_get_rank(dpu_from_set(dpu)))) {
+				numa_node_rank = -1;
+			} else {
+				numa_node_rank = dpu_get_rank_numa_node(dpu_get_rank(dpu_from_set(dpu)));
+			}
+			/*
+			if (rank_id != prev_rank_id) {
+				printf("/dev/dpu_rank%d @ NUMA node %d\n", rank_id, numa_node_rank);
+				prev_rank_id = rank_id;
+			}
+			*/
+		}
+
 		// Compute host solution
 		if(rep >= p.n_warmup) {
 			start(&timer, 2, 0);
@@ -283,8 +300,8 @@ int main(int argc, char **argv) {
 			if (rep >= p.n_warmup) {
 				printf("[::] BS-UPMEM | n_dpus=%d n_ranks=%d n_tasklets=%d e_type=%s block_size_B=%d n_elements=%lu",
 					NR_DPUS, nr_of_ranks, NR_TASKLETS, XSTR(DTYPE), BLOCK_SIZE, input_size);
-				printf(" with_alloc_overhead=%d b_with_load_overhead=%d b_with_free_overhead=%d ",
-					WITH_ALLOC_OVERHEAD, WITH_LOAD_OVERHEAD, WITH_FREE_OVERHEAD);
+				printf(" with_alloc_overhead=%d b_with_load_overhead=%d b_with_free_overhead=%d numa_node_rank=%d ",
+					WITH_ALLOC_OVERHEAD, WITH_LOAD_OVERHEAD, WITH_FREE_OVERHEAD, numa_node_rank);
 				printf("| latency_alloc_us=%f latency_load_us=%f latency_cpu_us=%f latency_write_us=%f latency_kernel_us=%f latency_read_us=%f latency_free_us=%f",
 					timer.time[0],
 					timer.time[1],