WRAM copy: report latency and throughput

4 files changed, 33 insertions, 20 deletions

diff --git a/Microbenchmarks/WRAM/dpu/task.c b/Microbenchmarks/WRAM/dpu/task.c
index 9ded3da..6e54b0b 100644
--- a/Microbenchmarks/WRAM/dpu/task.c
+++ b/Microbenchmarks/WRAM/dpu/task.c
@@ -63,6 +63,7 @@ int main_kernel1() {
 
     dpu_results_t *result = &DPU_RESULTS[tasklet_id];
     result->cycles = 0;
+    result->count = 0;
 
     const uint32_t A_SIZE = (BLOCK_SIZE >> DIV) << 2;
     // Address of the current processing block in MRAM
@@ -81,7 +82,7 @@ int main_kernel1() {
         // Load cache with current MRAM block
         mram_read((__mram_ptr void const*)(mram_base_addr_A + A_byte_index), cache_A, A_SIZE);
         mram_read((__mram_ptr void const*)(mram_base_addr_B + byte_index), cache_B, BLOCK_SIZE);
-        mram_read((__mram_ptr void const*)(mram_base_addr_C + byte_index), cache_C, BLOCK_SIZE); // Clean cache_C
+        //mram_read((__mram_ptr void const*)(mram_base_addr_C + byte_index), cache_C, BLOCK_SIZE); // Clean cache_C
 
 #ifdef WRAM
         // Barrier
@@ -102,6 +103,7 @@ int main_kernel1() {
         mram_write(cache_C, (__mram_ptr void*)(mram_base_addr_C + byte_index), BLOCK_SIZE);
 
         A_byte_index += A_SIZE * NR_TASKLETS;
+        result->count += 1;
     }
 
 #ifndef WRAM
diff --git a/Microbenchmarks/WRAM/host/app.c b/Microbenchmarks/WRAM/host/app.c
index 65193fc..9cebfd1 100644
--- a/Microbenchmarks/WRAM/host/app.c
+++ b/Microbenchmarks/WRAM/host/app.c
@@ -38,7 +38,6 @@ static void read_input(unsigned int* A, T* B, unsigned int nr_elements, unsigned
 static void read_input(unsigned int* A, T* B, unsigned int nr_elements) {
 #endif
     srand(0);
-    printf("nr_elements\t%u\t", nr_elements);
     for (unsigned int i = 0; i < nr_elements; i++) {
 #ifdef streaming
         A[i] = i % (BLOCK_SIZE >> DIV);
@@ -77,6 +76,7 @@ int main(int argc, char **argv) {
     struct Params p = input_params(argc, argv);
 
     struct dpu_set_t dpu_set, dpu;
+    uint32_t clocks_per_sec;
     uint32_t nr_of_dpus;
     uint32_t nr_of_ranks;
     
@@ -85,7 +85,6 @@ int main(int argc, char **argv) {
     DPU_ASSERT(dpu_load(dpu_set, DPU_BINARY, NULL));
     DPU_ASSERT(dpu_get_nr_dpus(dpu_set, &nr_of_dpus));
     DPU_ASSERT(dpu_get_nr_ranks(dpu_set, &nr_of_ranks));
-    printf("Allocated %d DPU(s)\n", nr_of_dpus);
 
     unsigned int i = 0;
     double cc = 0;
@@ -111,8 +110,6 @@ int main(int argc, char **argv) {
     // Timer declaration
     Timer timer;
 
-    printf("NR_TASKLETS\t%d\tBL\t%d\n", NR_TASKLETS, BL);
-
     // Loop over main kernel
     for(int rep = 0; rep < p.n_warmup + p.n_reps; rep++) {
 
@@ -123,7 +120,6 @@ int main(int argc, char **argv) {
         if(rep >= p.n_warmup)
             stop(&timer, 0);
 
-        printf("Load input data\n");
         if(rep >= p.n_warmup)
             start(&timer, 1, rep - p.n_warmup);
         // Input arguments
@@ -142,7 +138,6 @@ int main(int argc, char **argv) {
         if(rep >= p.n_warmup)
             stop(&timer, 1);
 
-        printf("Run program on DPU(s) \n");
         // Run DPU kernel
         if(rep >= p.n_warmup)
             start(&timer, 2, rep - p.n_warmup);
@@ -162,7 +157,6 @@ int main(int argc, char **argv) {
         }
 #endif
 
-        printf("Retrieve results\n");
         if(rep >= p.n_warmup)
             start(&timer, 3, rep - p.n_warmup);
         dpu_results_t results[nr_of_dpus];
@@ -173,15 +167,19 @@ int main(int argc, char **argv) {
 			
 #if PERF
             results[i].cycles = 0;
+            DPU_ASSERT(dpu_copy_from(dpu, "CLOCKS_PER_SEC", 0, &clocks_per_sec, sizeof(clocks_per_sec)));
             // Retrieve tasklet timings
             for (unsigned int each_tasklet = 0; each_tasklet < NR_TASKLETS; each_tasklet++) {
                 dpu_results_t result;
                 result.cycles = 0;
+                result.count = 0;
                 DPU_ASSERT(dpu_copy_from(dpu, "DPU_RESULTS", each_tasklet * sizeof(dpu_results_t), &result, sizeof(dpu_results_t)));
-                printf("[::] DMA UPMEM | n_dpus=%d n_ranks=%d n_tasklets=%d e_type=%s n_elements=%u e_mode=%s block_size_B=%d"
-                    " | dpu_cycles=%lu\n",
+                printf("[::] COPY UPMEM | n_dpus=%d n_ranks=%d n_tasklets=%d e_type=%s n_elements=%u e_mode=%s block_size_B=%d"
+                    " | latency_block_copy_us=%f throughput_dpu_copy_MBps=%f throughput_tasklet_copy_MBps=%f\n",
                     nr_of_dpus, nr_of_ranks, NR_TASKLETS, XSTR(T), input_size_dpu, transfer_mode, BLOCK_SIZE,
-                    result.cycles);
+                    ((double)result.cycles * 1e6 / clocks_per_sec) / result.count,
+                    input_size_dpu * sizeof(T) / ((double)result.cycles * 1e6 / clocks_per_sec),
+                    input_size_dpu * sizeof(T) / ((double)result.cycles * 1e6 * NR_TASKLETS / clocks_per_sec));
                 if (result.cycles > results[i].cycles)
                     results[i].cycles = result.cycles;
             }
@@ -210,7 +208,6 @@ int main(int argc, char **argv) {
 #endif
 
     }
-    printf("DPU cycles  = %g cc\n", cc / p.n_reps);
 
     // Check output
     bool status = true;
diff --git a/Microbenchmarks/WRAM/run-transfer.sh b/Microbenchmarks/WRAM/run-transfer.sh
index 7c4a179..c707fbe 100755
--- a/Microbenchmarks/WRAM/run-transfer.sh
+++ b/Microbenchmarks/WRAM/run-transfer.sh
@@ -1,5 +1,14 @@
 #!/bin/bash
 
+# The DPU application reads BL<<2 sized blocks of data from MRAM to WRAM via
+# DMA (mram_read). It then performs the specified type of copy operation
+# (i.e., streaming copy / strided copy / copy of random elements) within this
+# block and writes the result back from WRAM to MRAM via DMA (mram_write).
+# By default (MEM=WRAM), each tasklet reports the total number of cycles
+# spent in the WRAM-to-WRAM copy operation. With MEM=MRAM, each tasklet instead
+# reports the total number of cycles spent in MRAM processing: DMA read,
+# WRAM-to-WRAM copy, and DMA write.
+
 set -e
 
 (
@@ -8,19 +17,21 @@ echo "prim-benchmarks WRAM microbenchmark (dfatool edition)"
 echo "Started at $(date)"
 echo "Revision $(git describe --always)"
 
-for ndpu in 1 4 8 16; do
-	for ntask in 1 2 4 8 12 16 20; do
+for ndpu in 1 4 8 16 32 48 64 128 256; do
+	for ntask in 1 2 4 8 16; do
 		for bl in 4 5 6 7 8 9 10 11; do
 			for op in streaming strided random; do
 				if make -B NR_DPUS=$ndpu NR_TASKLETS=$ntask BL=$bl OP=$op; then
-					bin/host_code -w 0 -e 10 || true
+					bin/host_code -w 0 -e 50 || true
 				fi
 			done
 		done
 	done
 done
 echo "Completed at $(date)"
-) | tee "log-$(hostname)-wram-only.txt"
+) | tee "log-$(hostname)-wram.txt"
+rm -f "log-$(hostname)-wram.txt"
+xz -v -9 -M 800M "log-$(hostname)-wram.txt"
 
 (
 
@@ -28,16 +39,18 @@ echo "prim-benchmarks WRAM microbenchmark (dfatool edition)"
 echo "Started at $(date)"
 echo "Revision $(git describe --always)"
 
-for ndpu in 1 4 8 16; do
-	for ntask in 1 2 4 8 12 16 20; do
+for ndpu in 1 4 8 16 32 48 64 128 256; do
+	for ntask in 1 2 4 8 16; do
 		for bl in 4 5 6 7 8 9 10 11; do
 			for op in streaming strided random; do
 				if make -B NR_DPUS=$ndpu NR_TASKLETS=$ntask BL=$bl OP=$op MEM=MRAM; then
-					bin/host_code -w 0 -e 10 || true
+					bin/host_code -w 0 -e 50 || true
 				fi
 			done
 		done
 	done
 done
 echo "Completed at $(date)"
-) | tee "log-$(hostname)-wram-mram.txt"
+) | tee "log-$(hostname)-mram.txt"
+rm -f "log-$(hostname)-mram.txt"
+xz -v -9 -M 800M "log-$(hostname)-mramd.txt"
diff --git a/Microbenchmarks/WRAM/support/common.h b/Microbenchmarks/WRAM/support/common.h
index 42914ac..e8c2823 100755
--- a/Microbenchmarks/WRAM/support/common.h
+++ b/Microbenchmarks/WRAM/support/common.h
@@ -12,6 +12,7 @@ typedef struct {
 
 typedef struct {
     uint64_t cycles;
+    uint64_t count;
 } dpu_results_t;
 
 // Transfer size between MRAM and WRAM