From 3633e0409cb244a951a251047bd97a2e35986084 Mon Sep 17 00:00:00 2001
From: Birte Kristina Friesel <birte.friesel@uos.de>
Date: Mon, 8 Jul 2024 16:35:46 +0200
Subject: SCAN-RSS: NUMA support

---
 SCAN-RSS/host/app.c | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)

(limited to 'SCAN-RSS/host/app.c')

diff --git a/SCAN-RSS/host/app.c b/SCAN-RSS/host/app.c
index 93c2c47..6771207 100644
--- a/SCAN-RSS/host/app.c
+++ b/SCAN-RSS/host/app.c
@@ -29,10 +29,12 @@
 #include <dpu_probe.h>
 #endif
 
+#include <dpu_management.h>
+#include <dpu_target_macros.h>
+
 // Pointer declaration
 static T* A;
 static T* C;
-static T* C2;
 
 // Create input arrays
 static void read_input(T* A, unsigned int nr_elements, unsigned int nr_elements_round) {
@@ -71,6 +73,8 @@ int main(int argc, char **argv) {
     // Timer declaration
     Timer timer;
 
+    int numa_node_rank = -2;
+
     // Allocate DPUs and load binary
 #if !WITH_ALLOC_OVERHEAD
     DPU_ASSERT(dpu_alloc(NR_DPUS, NULL, &dpu_set));
@@ -98,9 +102,8 @@ int main(int argc, char **argv) {
     // Input/output allocation
     A = malloc(input_size_dpu_round * NR_DPUS * sizeof(T));
     C = malloc(input_size_dpu_round * NR_DPUS * sizeof(T));
-    C2 = malloc(input_size_dpu_round * NR_DPUS * sizeof(T));
     T *bufferA = A;
-    T *bufferC = C2;
+    T *bufferC = C;
 
     // Create an input file with arbitrary data
     read_input(A, input_size, input_size_dpu_round * NR_DPUS);
@@ -132,6 +135,23 @@ int main(int argc, char **argv) {
         assert(nr_of_dpus == NR_DPUS);
 #endif
 
+        // int prev_rank_id = -1;
+        int rank_id = -1;
+        DPU_FOREACH (dpu_set, dpu) {
+            rank_id = dpu_get_rank_id(dpu_get_rank(dpu_from_set(dpu))) & DPU_TARGET_MASK;
+            if ((numa_node_rank != -2) && numa_node_rank != dpu_get_rank_numa_node(dpu_get_rank(dpu_from_set(dpu)))) {
+                numa_node_rank = -1;
+            } else {
+                numa_node_rank = dpu_get_rank_numa_node(dpu_get_rank(dpu_from_set(dpu)));
+            }
+            /*
+            if (rank_id != prev_rank_id) {
+                printf("/dev/dpu_rank%d @ NUMA node %d\n", rank_id, numa_node_rank);
+                prev_rank_id = rank_id;
+            }
+            */
+        }
+
         // Compute output on CPU (performance comparison and verification purposes)
         if(rep >= p.n_warmup) {
             start(&timer, 2, 0);
@@ -314,8 +334,8 @@ int main(int argc, char **argv) {
             printf("[" ANSI_COLOR_GREEN "OK" ANSI_COLOR_RESET "] Outputs are equal\n");
             printf("[::] SCAN-RSS UPMEM | n_dpus=%d n_ranks=%d n_tasklets=%d e_type=%s block_size_B=%d b_unroll=%d n_elements=%d",
                 NR_DPUS, nr_of_ranks, NR_TASKLETS, XSTR(T), BLOCK_SIZE, UNROLL, input_size);
-            printf(" b_with_alloc_overhead=%d b_with_load_overhead=%d b_with_free_overhead=%d ",
-                WITH_ALLOC_OVERHEAD, WITH_LOAD_OVERHEAD, WITH_FREE_OVERHEAD);
+            printf(" b_with_alloc_overhead=%d b_with_load_overhead=%d b_with_free_overhead=%d numa_node_rank=%d ",
+                WITH_ALLOC_OVERHEAD, WITH_LOAD_OVERHEAD, WITH_FREE_OVERHEAD, numa_node_rank);
             printf("| latency_alloc_us=%f latency_load_us=%f latency_cpu_us=%f latency_write_us=%f latency_kernel_us=%f latency_sync_us=%f latency_read_us=%f latency_free_us=%f",
                 timer.time[0],
                 timer.time[1],
@@ -359,7 +379,6 @@ int main(int argc, char **argv) {
     // Deallocation
     free(A);
     free(C);
-    free(C2);
 
 #if !WITH_ALLOC_OVERHEAD
     DPU_ASSERT(dpu_free(dpu_set));
-- 
cgit v1.2.3