TRNS: Add NUMA and overhead measurement support

6 files changed, 252 insertions, 9 deletions

diff --git a/TRNS/Makefile b/TRNS/Makefile
index f58eb0c..fd3f493 100644
--- a/TRNS/Makefile
+++ b/TRNS/Makefile
@@ -1,14 +1,17 @@
 NR_DPUS ?= 1
 NR_TASKLETS ?= 16
 ENERGY ?= 0
+WITH_ALLOC_OVERHEAD ?= 0
+WITH_LOAD_OVERHEAD ?= 0
+WITH_FREE_OVERHEAD ?= 0
 
 COMMON_INCLUDES := support
 HOST_SOURCES := $(wildcard host/*.c)
 DPU_SOURCES := $(wildcard dpu/*.c)
 
 COMMON_FLAGS := -Wall -Wextra -g -I${COMMON_INCLUDES}
-HOST_FLAGS := ${COMMON_FLAGS} -std=c11 -O3 `dpu-pkg-config --cflags --libs dpu` -DNR_TASKLETS=${NR_TASKLETS} -DNR_DPUS=${NR_DPUS} -DENERGY=${ENERGY} 
-DPU_FLAGS := ${COMMON_FLAGS} -O2 -DNR_TASKLETS=${NR_TASKLETS} 
+HOST_FLAGS := ${COMMON_FLAGS} -std=c11 -O3 `dpu-pkg-config --cflags --libs dpu` -DNR_TASKLETS=${NR_TASKLETS} -DNR_DPUS=${NR_DPUS} -DENERGY=${ENERGY} -DWITH_ALLOC_OVERHEAD=${WITH_ALLOC_OVERHEAD} -DWITH_LOAD_OVERHEAD=${WITH_LOAD_OVERHEAD} -DWITH_FREE_OVERHEAD=${WITH_FREE_OVERHEAD}
+DPU_FLAGS := ${COMMON_FLAGS} -O2 -DNR_TASKLETS=${NR_TASKLETS}
 
 QUIET = @
 
diff --git a/TRNS/baselines/cpu/Makefile b/TRNS/baselines/cpu/Makefile
index 781e2be..a5a1635 100644
--- a/TRNS/baselines/cpu/Makefile
+++ b/TRNS/baselines/cpu/Makefile
@@ -32,8 +32,15 @@
 #  THE SOFTWARE.
 # 
 
+NUMA ?= 0
+FLAGS =
+
+ifeq (${NUMA}, 1)
+	FLAGS += -lnuma
+endif
+
 CXX=g++
-CXX_FLAGS=-std=c++11
+CXX_FLAGS=-std=c++11 -Wall -Wextra -pedantic -DNUMA=${NUMA}
 
 LIB=-L/usr/lib/ -lm -pthread
 
@@ -44,7 +51,7 @@ EXE=trns
 all: trns
 
 trns: ${SRC}
-	$(CXX) -O2 $(CXX_FLAGS) $(SRC) $(LIB) -o $(EXE)
+	$(CXX) -O2 $(CXX_FLAGS) $(SRC) $(LIB) -o $(EXE) $(FLAGS)
 
 trns_O0: ${SRC}
 	$(CXX) $(CXX_FLAGS) $(SRC) $(LIB) -o $(EXE)_O0
diff --git a/TRNS/baselines/cpu/main.cpp b/TRNS/baselines/cpu/main.cpp
index 2c481df..e1a3601 100644
--- a/TRNS/baselines/cpu/main.cpp
+++ b/TRNS/baselines/cpu/main.cpp
@@ -47,6 +47,19 @@
 #define XSTR(x) STR(x)
 #define STR(x) #x
 
+#if NUMA
+#include <numaif.h>
+#include <numa.h>
+
+void* mp_pages[1];
+int mp_status[1];
+int mp_nodes[1];
+int numa_node_in = -1;
+int numa_node_out = -1;
+int numa_node_cpu = -1;
+#endif
+
+
 // Params ---------------------------------------------------------------------
 struct Params {
 
@@ -57,6 +70,11 @@ struct Params {
     int m;
     int N_;
     int n;
+#if NUMA
+    struct bitmask* bitmask_in;
+    struct bitmask* bitmask_out;
+    int numa_node_cpu;
+#endif
 
     Params(int argc, char **argv) {
         n_threads     = 4;
@@ -66,8 +84,13 @@ struct Params {
         m             = 16;
         N_            = 128;
         n             = 8;
+#if NUMA
+        bitmask_in    = NULL;
+        bitmask_out   = NULL;
+        numa_node_cpu = -1;
+#endif
         int opt;
-        while((opt = getopt(argc, argv, "ht:w:r:m:n:o:p:")) >= 0) {
+        while((opt = getopt(argc, argv, "ht:w:r:m:n:o:p:a:b:c:")) >= 0) {
             switch(opt) {
             case 'h':
                 usage();
@@ -80,6 +103,11 @@ struct Params {
             case 'n': n             = atoi(optarg); break;
             case 'o': M_            = atoi(optarg); break;
             case 'p': N_            = atoi(optarg); break;
+#if NUMA
+            case 'a': bitmask_in    = numa_parse_nodestring(optarg); break;
+            case 'b': bitmask_out   = numa_parse_nodestring(optarg); break;
+            case 'c': numa_node_cpu = atoi(optarg); break;
+#endif
             default:
                 fprintf(stderr, "\nUnrecognized option!\n");
                 usage();
@@ -133,11 +161,41 @@ int main(int argc, char **argv) {
     int n       = p.n;
     int in_size       = M_ * m * N_ * n;
     int finished_size = M_ * m * N_;
+
+#if NUMA
+    if (p.bitmask_in) {
+        numa_set_membind(p.bitmask_in);
+        numa_free_nodemask(p.bitmask_in);
+    }
+    T *              h_in_out = (T *)numa_alloc(in_size * sizeof(T));
+    std::atomic_int *h_finished =
+        (std::atomic_int *)numa_alloc(sizeof(std::atomic_int) * finished_size);
+#else
     T *              h_in_out = (T *)malloc(in_size * sizeof(T));
     std::atomic_int *h_finished =
         (std::atomic_int *)malloc(sizeof(std::atomic_int) * finished_size);
+#endif
+
+#if NUMA
+    if (p.bitmask_out) {
+        numa_set_membind(p.bitmask_out);
+        numa_free_nodemask(p.bitmask_out);
+    }
+    std::atomic_int *h_head = (std::atomic_int *)numa_alloc(N_ * sizeof(std::atomic_int));
+#else
     std::atomic_int *h_head = (std::atomic_int *)malloc(N_ * sizeof(std::atomic_int));
+#endif
+
     ALLOC_ERR(h_in_out, h_finished, h_head);
+
+
+#if NUMA
+    struct bitmask *bitmask_all = numa_allocate_nodemask();
+    numa_bitmask_setall(bitmask_all);
+    numa_set_membind(bitmask_all);
+    numa_free_nodemask(bitmask_all);
+#endif
+
     T *h_in_backup = (T *)malloc(in_size * sizeof(T));
     ALLOC_ERR(h_in_backup);
     timer.stop("Allocation");
@@ -153,6 +211,40 @@ int main(int argc, char **argv) {
     //timer.print("Initialization", 1);
     memcpy(h_in_backup, h_in_out, in_size * sizeof(T)); // Backup for reuse across iterations
 
+#if NUMA
+    mp_pages[0] = h_in_out;
+    if (move_pages(0, 1, mp_pages, NULL, mp_status, 0) == -1) {
+        perror("move_pages(A)");
+    }
+    else if (mp_status[0] < 0) {
+        printf("move_pages error: %d", mp_status[0]);
+    }
+    else {
+        numa_node_in = mp_status[0];
+    }
+
+    mp_pages[0] = h_finished;
+    if (move_pages(0, 1, mp_pages, NULL, mp_status, 0) == -1) {
+        perror("move_pages(C)");
+    }
+    else if (mp_status[0] < 0) {
+        printf("move_pages error: %d", mp_status[0]);
+    }
+    else {
+        numa_node_out = mp_status[0];
+    }
+
+    numa_node_cpu = p.numa_node_cpu;
+    if (numa_node_cpu != -1) {
+        if (numa_run_on_node(numa_node_cpu) == -1) {
+            perror("numa_run_on_node");
+            numa_node_cpu = -1;
+        }
+    }
+#endif
+
+
+
     // Loop over main kernel
     for(int rep = 0; rep < p.n_warmup + p.n_reps; rep++) {
 
@@ -202,9 +294,15 @@ int main(int argc, char **argv) {
             timer.stop("Step 3");
 
         if (rep >= p.n_warmup) {
-            printf("[::] TRNS CPU | n_threads=%d e_type=%s n_elements=%d "
-                "| throughput_MBps=%f",
+            printf("[::] TRNS CPU | n_threads=%d e_type=%s n_elements=%d"
+#if NUMA
+                " numa_node_in=%d numa_node_out=%d numa_node_cpu=%d numa_distance_in_cpu=%d numa_distance_cpu_out=%d"
+#endif
+                " | throughput_MBps=%f",
                 p.n_threads, XSTR(T), in_size,
+#if NUMA
+                numa_node_in, numa_node_out, numa_node_cpu, numa_distance(numa_node_in, numa_node_cpu), numa_distance(numa_node_cpu, numa_node_out),
+#endif
                 in_size * sizeof(T) / (timer.get("Step 1") + timer.get("Step 2") + timer.get("Step 3")));
             printf(" throughput_MOpps=%f",
                 in_size / (timer.get("Step 1") + timer.get("Step 2") + timer.get("Step 3")));
@@ -221,10 +319,17 @@ int main(int argc, char **argv) {
 
     // Free memory
     timer.start("Deallocation");
+#if NUMA
+    numa_free(h_in_out, in_size * sizeof(T));
+    numa_free(h_finished, sizeof(std::atomic_int) * finished_size);
+    numa_free(h_head, N_ * sizeof(std::atomic_int));
+    numa_free(h_in_backup, in_size * sizeof(T));
+#else
     free(h_in_out);
     free(h_finished);
     free(h_head);
     free(h_in_backup);
+#endif
     timer.stop("Deallocation");
     //timer.print("Deallocation", 1);
 
diff --git a/TRNS/dimes-hetsim-hbm.sh b/TRNS/dimes-hetsim-hbm.sh
new file mode 100755
index 0000000..a7240a9
--- /dev/null
+++ b/TRNS/dimes-hetsim-hbm.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+
+cd baselines/cpu
+make -B NUMA=1
+
+mkdir -p log/$(hostname)
+fn=log/$(hostname)/$(date +%Y%m%d)
+
+# DPU version: -m m -n n -o M_ -p N_
+# input size: sizeof(T) * M_ * m * N_ * n
+# Upstream DPU version uses int64_t, -p 2048 -o 12288 -x 1 [implicit -m 16 -n 8]
+# Upstream CPU version uses double,  -p 2556 -o 4096 -m 16 -n 8 and fails with -o 12288 (allocation error)
+#
+# Here: -p 2048 -o 2048 -m 16 -n 8 for both
+
+(
+
+echo "CPU single-node operation (1/2)" >&2
+
+parallel -j1 --eta --joblog ${fn}.1.joblog --header : \
+	./trns -w 0 -r 40 -p 2048 -o 2048 -m 16 -n 8 -t {nr_threads} -a {ram} -b {ram} -c {cpu} \
+	::: ram $(seq 0 15) \
+	::: cpu $(seq 0 7) \
+	::: nr_threads 1 2 4 8 12 16
+
+echo "CPU multi-node operation (1/2)" >&2
+
+parallel -j1 --eta --joblog ${fn}.2.joblog --header : \
+	./trns -w 0 -r 40 -p 2048 -o 2048 -m 16 -n 8 -t {nr_threads} -a {ram} -b {ram} -c {cpu} \
+	::: ram $(seq 0 15) \
+	::: cpu -1 \
+	::: nr_threads 32 48 64 96 128
+
+) > ${fn}.txt
+
+xz -f -v -9 -M 800M ${fn}.txt
diff --git a/TRNS/dimes-hetsim-nmc.sh b/TRNS/dimes-hetsim-nmc.sh
new file mode 100755
index 0000000..583fed3
--- /dev/null
+++ b/TRNS/dimes-hetsim-nmc.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+
+mkdir -p log/$(hostname) baselines/cpu/log/$(hostname)
+fn=log/$(hostname)/$(date +%Y%m%d)
+
+# DPU version: -m m -n n -o M_ -p N_
+# input size: sizeof(T) * M_ * m * N_ * n
+# Upstream DPU version uses int64_t, -p 2048 -o 12288 -x 1 [implicit -m 16 -n 8]
+# Upstream CPU version uses double,  -p 2556 -o 4096 -m 16 -n 8 and fails with -o 12288 (allocation error)
+#
+# Here: -p 2048 -o 2048 -m 16 -n 8 for both
+
+# upstream uses 167772160 * 2 * int32 == 2.5 GiB input and 1.25 GiB output for DPU version
+
+run_benchmark_nmc() {
+	local "$@"
+	sudo limit_ranks_to_numa_node ${numa_rank}
+	if make -B NR_DPUS=${nr_dpus} NR_TASKLETS=${nr_tasklets} WITH_ALLOC_OVERHEAD=1 WITH_LOAD_OVERHEAD=1 WITH_FREE_OVERHEAD=1; then
+		bin/host_code -w 0 -e 100 -p 2048 -o 2048 -m 16 -n 8 -x 1
+	fi
+	return $?
+}
+
+export -f run_benchmark_nmc
+
+(
+
+echo "NMC single-node operation (1/2)" >&2
+
+parallel -j1 --eta --joblog ${fn}.1.joblog --header : \
+	run_benchmark_nmc nr_dpus={nr_dpus} nr_tasklets=16 input_size={input_size} numa_rank={numa_rank} \
+	::: numa_rank 0 1 \
+	::: nr_dpus 64 128 256 512 768 1024
+
+echo "NMC multi-node operation (2/2)" >&2
+
+parallel -j1 --eta --joblog ${fn}.2.joblog --header : \
+	run_benchmark_nmc nr_dpus={nr_dpus} nr_tasklets=16 input_size={input_size} numa_rank={numa_rank} \
+	::: numa_rank any \
+	::: nr_dpus 1536 2048 2304
+
+) > ${fn}.txt
+
+xz -f -v -9 -M 800M ${fn}.txt
+
+cd baselines/cpu
+make -B NUMA=1
+
+(
+
+echo "CPU single-node operation (1/2)" >&2
+
+parallel -j1 --eta --joblog ${fn}.1.joblog --header : \
+	./trns -w 0 -r 40 -p 2048 -o 2048 -m 16 -n 8 -t {nr_threads} \
+	::: ram 0 1 \
+	::: cpu 0 1 \
+	::: nr_threads 1 2 4 8 12 16 32
+
+echo "CPU multi-node operation (1/2)" >&2
+
+parallel -j1 --eta --joblog ${fn}.2.joblog --header : \
+	./trns -w 0 -r 40 -p 2048 -o 2048 -m 16 -n 8 -t {nr_threads} \
+	::: ram 0 1 \
+	::: cpu -1 \
+	::: nr_threads 48 64
+
+) > ${fn}.txt
+
+xz -f -v -9 -M 800M ${fn}.txt
diff --git a/TRNS/host/app.c b/TRNS/host/app.c
index 90807a9..33ce848 100644
--- a/TRNS/host/app.c
+++ b/TRNS/host/app.c
@@ -30,6 +30,9 @@
 #include <dpu_probe.h>
 #endif
 
+#include <dpu_management.h>
+#include <dpu_target_macros.h>
+
 // Pointer declaration
 static T* A_host;
 static T* A_backup;
@@ -94,6 +97,8 @@ int main(int argc, char **argv) {
     // Timer declaration
     Timer timer;
 
+    int numa_node_rank = -2;
+
     // Loop over main kernel
     for(int rep = 0; rep < p.n_warmup + p.n_reps; rep++) {
 
@@ -250,6 +255,24 @@ int main(int argc, char **argv) {
                 first_round = 0;
             }
         }
+
+        // int prev_rank_id = -1;
+        int rank_id = -1;
+        DPU_FOREACH (dpu_set, dpu) {
+            rank_id = dpu_get_rank_id(dpu_get_rank(dpu_from_set(dpu))) & DPU_TARGET_MASK;
+            if ((numa_node_rank != -2) && numa_node_rank != dpu_get_rank_numa_node(dpu_get_rank(dpu_from_set(dpu)))) {
+                numa_node_rank = -1;
+            } else {
+                numa_node_rank = dpu_get_rank_numa_node(dpu_get_rank(dpu_from_set(dpu)));
+            }
+            /*
+            if (rank_id != prev_rank_id) {
+                printf("/dev/dpu_rank%d @ NUMA node %d\n", rank_id, numa_node_rank);
+                prev_rank_id = rank_id;
+            }
+            */
+        }
+
         start(&timer, 1, 1);
         DPU_ASSERT(dpu_free(dpu_set));
         stop(&timer, 1);
@@ -268,8 +291,8 @@ int main(int argc, char **argv) {
             printf("[" ANSI_COLOR_GREEN "OK" ANSI_COLOR_RESET "] Outputs are equal\n");
             unsigned long input_size = M_ * m * N_ * n;
             if (rep >= p.n_warmup) {
-                printf("[::] TRNS UPMEM | n_dpus=%d n_ranks=%d n_tasklets=%d e_type=%s n_elements=%lu ",
-                    NR_DPUS, nr_of_ranks, NR_TASKLETS, XSTR(T), input_size);
+                printf("[::] TRNS UPMEM | n_dpus=%d n_ranks=%d n_tasklets=%d e_type=%s n_elements=%lu numa_node_rank=%d ",
+                    NR_DPUS, nr_of_ranks, NR_TASKLETS, XSTR(T), input_size, numa_node_rank);
                 printf("| latency_cpu_us=%f latency_realloc_us=%f latency_load_us=%f latency_write_us=%f latency_kernel_us=%f latency_read_us=%f",
                     timer.time[0], // CPU
                     timer.time[1], // free + alloc