HST-S: alloc overhead

2 files changed, 139 insertions, 59 deletions

diff --git a/HST-S/Makefile b/HST-S/Makefile
index de234d1..fdea674 100644
--- a/HST-S/Makefile
+++ b/HST-S/Makefile
@@ -2,13 +2,17 @@ NR_DPUS ?= 1
 NR_TASKLETS ?= 16
 BL ?= 10
 ENERGY ?= 0
+WITH_ALLOC_OVERHEAD ?= 0
+WITH_LOAD_OVERHEAD ?= 0
+WITH_FREE_OVERHEAD ?= 0
+WITH_DPUINFO ?= 0
 
 COMMON_INCLUDES := support
 HOST_SOURCES := $(wildcard host/*.c)
 DPU_SOURCES := $(wildcard dpu/*.c)
 
 COMMON_FLAGS := -Wall -Wextra -g -I${COMMON_INCLUDES} -DNR_TASKLETS=${NR_TASKLETS} -DNR_DPUS=${NR_DPUS} -DBL=${BL}
-HOST_FLAGS := ${COMMON_FLAGS} -std=c11 -O3 `dpu-pkg-config --cflags --libs dpu` -DENERGY=${ENERGY}
+HOST_FLAGS := ${COMMON_FLAGS} -std=c11 -O3 `dpu-pkg-config --cflags --libs dpu` -DENERGY=${ENERGY} -DWITH_ALLOC_OVERHEAD=${WITH_ALLOC_OVERHEAD} -DWITH_LOAD_OVERHEAD=${WITH_LOAD_OVERHEAD} -DWITH_FREE_OVERHEAD=${WITH_FREE_OVERHEAD} -DWITH_DPUINFO=${WITH_DPUINFO}
 DPU_FLAGS := ${COMMON_FLAGS} -O2
 
 QUIET = @
diff --git a/HST-S/host/app.c b/HST-S/host/app.c
index 7fe87c1..df728a8 100644
--- a/HST-S/host/app.c
+++ b/HST-S/host/app.c
@@ -30,6 +30,17 @@
 #include <dpu_probe.h>
 #endif
 
+#if WITH_DPUINFO
+#include <dpu_management.h>
+#include <dpu_target_macros.h>
+#endif
+
+#if SDK_SINGLETHREADED
+#define DPU_ALLOC_PROFILE "nrThreadsPerRank=0"
+#else
+#define DPU_ALLOC_PROFILE NULL
+#endif
+
 // Pointer declaration
 static T* A;
 static unsigned int* histo_host;
@@ -83,23 +94,39 @@ int main(int argc, char **argv) {
 
     struct dpu_set_t dpu_set, dpu;
     uint32_t nr_of_dpus;
-    
+    uint32_t nr_of_ranks;
+
 #if ENERGY
     struct dpu_probe_t probe;
     DPU_ASSERT(dpu_probe_init("energy_probe", &probe));
 #endif
 
+    printf("WITH_ALLOC_OVERHEAD=%d WITH_LOAD_OVERHEAD=%d WITH_FREE_OVERHEAD=%d\n", WITH_ALLOC_OVERHEAD, WITH_LOAD_OVERHEAD, WITH_FREE_OVERHEAD);
+
+    // Timer declaration
+    Timer timer;
+
     // Allocate DPUs and load binary
-    DPU_ASSERT(dpu_alloc(NR_DPUS, NULL, &dpu_set));
+#if !WITH_ALLOC_OVERHEAD
+    DPU_ASSERT(dpu_alloc(NR_DPUS, DPU_ALLOC_PROFILE, &dpu_set));
+    timer.time[0] = 0; // alloc
+#endif
+#if !WITH_LOAD_OVERHEAD
     DPU_ASSERT(dpu_load(dpu_set, DPU_BINARY, NULL));
     DPU_ASSERT(dpu_get_nr_dpus(dpu_set, &nr_of_dpus));
-    printf("Allocated %d DPU(s)\n", nr_of_dpus);
+    DPU_ASSERT(dpu_get_nr_ranks(dpu_set, &nr_of_ranks));
+    assert(nr_of_dpus == NR_DPUS);
+    timer.time[1] = 0; // load
+#endif
+#if !WITH_FREE_OVERHEAD
+    timer.time[6] = 0; // free
+#endif
 
     unsigned int i = 0;
     unsigned int input_size; // Size of input image
     unsigned int dpu_s = p.dpu_s;
     if(p.exp == 0)
-        input_size = p.input_size * nr_of_dpus; // Size of input image
+        input_size = p.input_size * NR_DPUS; // Size of input image
     else if(p.exp == 1)
         input_size = p.input_size; // Size of input image
     else
@@ -107,20 +134,20 @@ int main(int argc, char **argv) {
 
     const unsigned int input_size_8bytes = 
         ((input_size * sizeof(T)) % 8) != 0 ? roundup(input_size, 8) : input_size; // Input size per DPU (max.), 8-byte aligned
-    const unsigned int input_size_dpu = divceil(input_size, nr_of_dpus); // Input size per DPU (max.)
+    const unsigned int input_size_dpu = divceil(input_size, NR_DPUS); // Input size per DPU (max.)
     const unsigned int input_size_dpu_8bytes = 
         ((input_size_dpu * sizeof(T)) % 8) != 0 ? roundup(input_size_dpu, 8) : input_size_dpu; // Input size per DPU (max.), 8-byte aligned
 
     // Input/output allocation
-    A = malloc(input_size_dpu_8bytes * nr_of_dpus * sizeof(T));
+    A = malloc(input_size_dpu_8bytes * NR_DPUS * sizeof(T));
     T *bufferA = A;
     histo_host = malloc(p.bins * sizeof(unsigned int));
-    histo = malloc(nr_of_dpus * p.bins * sizeof(unsigned int));
+    histo = malloc(NR_DPUS * p.bins * sizeof(unsigned int));
 
     // Create an input file with arbitrary data
     read_input(A, p);
     if(p.exp == 0){
-        for(unsigned int j = 1; j < nr_of_dpus; j++){
+        for(unsigned int j = 1; j < NR_DPUS; j++){
             memcpy(&A[j * input_size_dpu_8bytes], &A[0], input_size_dpu_8bytes * sizeof(T));
         }
     }
@@ -129,40 +156,69 @@ int main(int argc, char **argv) {
             memcpy(&A[j * p.input_size], &A[0], p.input_size * sizeof(T));
     }
 
-    // Timer declaration
-    Timer timer;
-
-    printf("NR_TASKLETS\t%d\tBL\t%d\tinput_size\t%u\n", NR_TASKLETS, BL, input_size);
-
     // Loop over main kernel
     for(int rep = 0; rep < p.n_warmup + p.n_reps; rep++) {
         memset(histo_host, 0, p.bins * sizeof(unsigned int));
-        memset(histo, 0, nr_of_dpus * p.bins * sizeof(unsigned int));
+        memset(histo, 0, NR_DPUS * p.bins * sizeof(unsigned int));
 
-        // Compute output on CPU (performance comparison and verification purposes)
-        if(rep >= p.n_warmup)
+#if WITH_ALLOC_OVERHEAD
+        if(rep >= p.n_warmup) {
             start(&timer, 0, 0);
-        histogram_host(histo_host, A, p.bins, p.input_size, 1, nr_of_dpus);
-        if(rep >= p.n_warmup)
+        }
+        DPU_ASSERT(dpu_alloc(NR_DPUS, DPU_ALLOC_PROFILE, &dpu_set));
+        if(rep >= p.n_warmup) {
             stop(&timer, 0);
-
-        printf("Load input data\n");
-        if(rep >= p.n_warmup)
+        }
+#endif
+#if WITH_DPUINFO
+        printf("DPUs:");
+        DPU_FOREACH (dpu_set, dpu) {
+            int rank = dpu_get_rank_id(dpu_get_rank(dpu_from_set(dpu))) & DPU_TARGET_MASK;
+            int slice = dpu_get_slice_id(dpu_from_set(dpu));
+            int member = dpu_get_member_id(dpu_from_set(dpu));
+            printf(" %d(%d.%d)", rank, slice, member);
+        }
+        printf("\n");
+#endif
+#if WITH_LOAD_OVERHEAD
+        if(rep >= p.n_warmup) {
             start(&timer, 1, 0);
+        }
+        DPU_ASSERT(dpu_load(dpu_set, DPU_BINARY, NULL));
+        if(rep >= p.n_warmup) {
+            stop(&timer, 1);
+        }
+        DPU_ASSERT(dpu_get_nr_dpus(dpu_set, &nr_of_dpus));
+        DPU_ASSERT(dpu_get_nr_ranks(dpu_set, &nr_of_ranks));
+        assert(nr_of_dpus == NR_DPUS);
+#endif
+
+        // Compute output on CPU (performance comparison and verification purposes)
+        if(rep >= p.n_warmup) {
+            start(&timer, 2, 0);
+        }
+        histogram_host(histo_host, A, p.bins, p.input_size, 1, NR_DPUS);
+        if(rep >= p.n_warmup) {
+            stop(&timer, 2);
+        }
+
+        if(rep >= p.n_warmup) {
+            start(&timer, 3, 0);
+        }
         // Input arguments
         unsigned int kernel = 0;
         i = 0;
 	    dpu_arguments_t input_arguments[NR_DPUS];
-	    for(i=0; i<nr_of_dpus-1; i++) {
+	    for(i=0; i<NR_DPUS-1; i++) {
 	        input_arguments[i].size=input_size_dpu_8bytes * sizeof(T); 
 	        input_arguments[i].transfer_size=input_size_dpu_8bytes * sizeof(T); 
 	        input_arguments[i].bins=p.bins;
 	        input_arguments[i].kernel=kernel;
 	    }
-	    input_arguments[nr_of_dpus-1].size=(input_size_8bytes - input_size_dpu_8bytes * (NR_DPUS-1)) * sizeof(T); 
-	    input_arguments[nr_of_dpus-1].transfer_size=input_size_dpu_8bytes * sizeof(T); 
-	    input_arguments[nr_of_dpus-1].bins=p.bins;
-	    input_arguments[nr_of_dpus-1].kernel=kernel;
+	    input_arguments[NR_DPUS-1].size=(input_size_8bytes - input_size_dpu_8bytes * (NR_DPUS-1)) * sizeof(T); 
+	    input_arguments[NR_DPUS-1].transfer_size=input_size_dpu_8bytes * sizeof(T); 
+	    input_arguments[NR_DPUS-1].bins=p.bins;
+	    input_arguments[NR_DPUS-1].kernel=kernel;
 
         // Copy input arrays
         i = 0;
@@ -174,13 +230,13 @@ int main(int argc, char **argv) {
             DPU_ASSERT(dpu_prepare_xfer(dpu, bufferA + input_size_dpu_8bytes * i));
         }
         DPU_ASSERT(dpu_push_xfer(dpu_set, DPU_XFER_TO_DPU, DPU_MRAM_HEAP_POINTER_NAME, 0, input_size_dpu_8bytes * sizeof(T), DPU_XFER_DEFAULT));
-        if(rep >= p.n_warmup)
-            stop(&timer, 1);
+        if(rep >= p.n_warmup) {
+            stop(&timer, 3);
+        }
 
-        printf("Run program on DPU(s) \n");
         // Run DPU kernel
         if(rep >= p.n_warmup) {
-            start(&timer, 2, 0);
+            start(&timer, 4, 0);
             #if ENERGY
             DPU_ASSERT(dpu_probe_start(&probe));
             #endif
@@ -188,7 +244,7 @@ int main(int argc, char **argv) {
  
         DPU_ASSERT(dpu_launch(dpu_set, DPU_SYNCHRONOUS));
         if(rep >= p.n_warmup) {
-            stop(&timer, 2);
+            stop(&timer, 4);
             #if ENERGY
             DPU_ASSERT(dpu_probe_stop(&probe));
             #endif
@@ -206,10 +262,10 @@ int main(int argc, char **argv) {
         }
 #endif
 
-        printf("Retrieve results\n");
         i = 0;
-        if(rep >= p.n_warmup)
-            start(&timer, 3, 0);
+        if(rep >= p.n_warmup) {
+            start(&timer, 5, 0);
+        }
         // PARALLEL RETRIEVE TRANSFER
         DPU_FOREACH(dpu_set, dpu, i) {
             DPU_ASSERT(dpu_prepare_xfer(dpu, histo + p.bins * i));
@@ -217,40 +273,60 @@ int main(int argc, char **argv) {
         DPU_ASSERT(dpu_push_xfer(dpu_set, DPU_XFER_FROM_DPU, DPU_MRAM_HEAP_POINTER_NAME, input_size_dpu_8bytes * sizeof(T), p.bins * sizeof(unsigned int), DPU_XFER_DEFAULT));
 
         // Final histogram merging
-        for(i = 1; i < nr_of_dpus; i++){
+        for(i = 1; i < NR_DPUS; i++){
             for(unsigned int j = 0; j < p.bins; j++){
                 histo[j] += histo[j + i * p.bins];
             }
         }
-        if(rep >= p.n_warmup)
-            stop(&timer, 3);
+        if(rep >= p.n_warmup) {
+            stop(&timer, 5);
+        }
+
+#if WITH_ALLOC_OVERHEAD
+#if WITH_FREE_OVERHEAD
+        if(rep >= p.n_warmup) {
+            start(&timer, 6, 0);
+        }
+#endif
+        DPU_ASSERT(dpu_free(dpu_set));
+#if WITH_FREE_OVERHEAD
+        if(rep >= p.n_warmup) {
+            stop(&timer, 6);
+        }
+#endif
+#endif
 
         if (rep >= p.n_warmup) {
-            printf("[::] HST-S NMC | n_dpus=%d n_tasklets=%d e_type=%s n_elements=%u n_bins=%d "
-                "| throughput_cpu_MBps=%f throughput_pim_MBps=%f throughput_MBps=%f",
-                nr_of_dpus, NR_TASKLETS, XSTR(T), input_size, p.bins,
-                input_size * sizeof(T) / timer.time[0],
+            printf("[::] HST-S UPMEM | n_dpus=%d n_ranks=%d n_tasklets=%d e_type=%s n_elements=%d n_bins=%d b_sdk_singlethreaded=%d ",
+                nr_of_dpus, nr_of_ranks, NR_TASKLETS, XSTR(T), input_size, p.bins, SDK_SINGLETHREADED);
+            printf("| latency_alloc_us=%f latency_load_us=%f latency_cpu_us=%f latency_write_us=%f latency_kernel_us=%f latency_read_us=%f latency_free_us=%f",
+                timer.time[0],
+                timer.time[1],
+                timer.time[2],
+                timer.time[3],
+                timer.time[4],
+                timer.time[5],
+                timer.time[6]);
+            printf(" throughput_cpu_MBps=%f throughput_upmem_kernel_MBps=%f throughput_upmem_total_MBps=%f",
                 input_size * sizeof(T) / timer.time[2],
-                input_size * sizeof(T) / (timer.time[1] + timer.time[2] + timer.time[3]));
-            printf(" throughput_cpu_MOpps=%f throughput_pim_MOpps=%f throughput_MOpps=%f",
-                input_size / timer.time[0],
+                input_size * sizeof(T) / (timer.time[4]),
+                input_size * sizeof(T) / (timer.time[0] + timer.time[1] + timer.time[3] + timer.time[4] + timer.time[5] + timer.time[6]));
+            printf(" throughput_upmem_wxr_MBps=%f throughput_upmem_lwxr_MBps=%f throughput_upmem_alwxr_MBps=%f",
+                input_size * sizeof(T) / (timer.time[3] + timer.time[4] + timer.time[5]),
+                input_size * sizeof(T) / (timer.time[1] + timer.time[3] + timer.time[4] + timer.time[5]),
+                input_size * sizeof(T) / (timer.time[0] + timer.time[1] + timer.time[3] + timer.time[4] + timer.time[5]));
+            printf(" throughput_cpu_MOpps=%f throughput_upmem_kernel_MOpps=%f throughput_upmem_total_MOpps=%f",
                 input_size / timer.time[2],
-                input_size / (timer.time[1] + timer.time[2] + timer.time[3]));
-            printall(&timer, 3);
+                input_size / (timer.time[4]),
+                input_size / (timer.time[0] + timer.time[1] + timer.time[3] + timer.time[4] + timer.time[5] + timer.time[6]));
+            printf(" throughput_upmem_wxr_MOpps=%f throughput_upmem_lwxr_MOpps=%f throughput_upmem_alwxr_MOpps=%f\n",
+                input_size / (timer.time[3] + timer.time[4] + timer.time[5]),
+                input_size / (timer.time[1] + timer.time[3] + timer.time[4] + timer.time[5]),
+                input_size / (timer.time[0] + timer.time[1] + timer.time[3] + timer.time[4] + timer.time[5]));
         }
 
     }
 
-    // Print timing results
-    printf("CPU ");
-    print(&timer, 0, p.n_reps);
-    printf("CPU-DPU ");
-    print(&timer, 1, p.n_reps);
-    printf("DPU Kernel ");
-    print(&timer, 2, p.n_reps);
-    printf("DPU-CPU ");
-    print(&timer, 3, p.n_reps);
-
     #if ENERGY
     double energy;
     DPU_ASSERT(dpu_probe_get(&probe, DPU_ENERGY, DPU_AVERAGE, &energy));
@@ -279,10 +355,10 @@ int main(int argc, char **argv) {
         }
     else
         for (unsigned int j = 0; j < p.bins; j++) {
-            if(nr_of_dpus * histo_host[j] != histo[j]){ 
+            if(NR_DPUS * histo_host[j] != histo[j]){ 
                 status = false;
 #if PRINT
-                printf("%u - %u: %u -- %u\n", j, j, nr_of_dpus * histo_host[j], histo[j]);
+                printf("%u - %u: %u -- %u\n", j, j, NR_DPUS * histo_host[j], histo[j]);
 #endif
             }
         }