From 24ec32ca18b47d5e2e9f5f4d5c3bc32f41fe4cbf Mon Sep 17 00:00:00 2001 From: Birte Kristina Friesel Date: Fri, 5 Jul 2024 09:58:11 +0200 Subject: BS NMC: log NUMA data --- BS/dimes-hetsim-hbm.sh | 16 ++++++++++++---- BS/dimes-hetsim-nmc.sh | 39 +++++++++++++++++++++++++++++++++++---- BS/host/app.c | 25 +++++++++++++++++++++---- 3 files changed, 68 insertions(+), 12 deletions(-) (limited to 'BS') diff --git a/BS/dimes-hetsim-hbm.sh b/BS/dimes-hetsim-hbm.sh index af1a124..b4a1d6d 100755 --- a/BS/dimes-hetsim-hbm.sh +++ b/BS/dimes-hetsim-hbm.sh @@ -28,34 +28,42 @@ export -f run_benchmark ( +echo "single-node execution, DPU ref (1/4)" >&2 + parallel -j1 --eta --joblog ${fn}.1.joblog --resume --header : \ run_benchmark i={i} nr_threads={nr_threads} ram={ram} cpu={cpu} \ input_size=${input_size_dpu} num_queries=${num_queries_dpu} \ - ::: i $(seq 1 20) \ + ::: i $(seq 1 5) \ ::: nr_threads 1 2 4 8 12 16 \ ::: cpu $(seq 0 7) \ ::: ram $(seq 0 15) +echo "single-node execution, HBM ref (2/4)" >&2 + parallel -j1 --eta --joblog ${fn}.2.joblog --resume --header : \ run_benchmark i={i} nr_threads={nr_threads} ram={ram} cpu={cpu} \ input_size=${input_size_hbm} num_queries=${num_queries_hbm} \ - ::: i $(seq 1 20) \ + ::: i $(seq 1 5) \ ::: nr_threads 1 2 4 8 12 16 \ ::: cpu $(seq 0 7) \ ::: ram $(seq 0 15) +echo "multi-node execution, DPU ref (3/4)" >&2 + parallel -j1 --eta --joblog ${fn}.3.joblog --resume --header : \ run_benchmark i={i} nr_threads={nr_threads} ram={ram} cpu={cpu} \ input_size=${input_size_dpu} num_queries=${num_queries_dpu} \ - ::: i $(seq 1 20) \ + ::: i $(seq 1 5) \ ::: nr_threads 32 48 64 96 128 \ ::: cpu -1 \ ::: ram $(seq 0 15) +echo "multi-node execution, HBM ref (4/4)" >&2 + parallel -j1 --eta --joblog ${fn}.4.joblog --resume --header : \ run_benchmark i={i} nr_threads={nr_threads} ram={ram} cpu={cpu} \ input_size=${input_size_hbm} num_queries=${num_queries_hbm} \ - ::: i $(seq 1 20) \ + ::: i $(seq 1 5) \ ::: nr_threads 32 48 64 96 128 \ ::: cpu -1 \ ::: ram $(seq 0 15) diff --git a/BS/dimes-hetsim-nmc.sh b/BS/dimes-hetsim-nmc.sh index ec27ceb..7a2b12a 100755 --- a/BS/dimes-hetsim-nmc.sh +++ b/BS/dimes-hetsim-nmc.sh @@ -13,6 +13,7 @@ num_queries_dpu=1048576 run_benchmark_nmc() { local "$@" + sudo limit_ranks_to_numa_node ${numa_rank} if make -B NR_DPUS=${nr_dpus} NR_TASKLETS=${nr_tasklets} WITH_ALLOC_OVERHEAD=1 WITH_LOAD_OVERHEAD=1 WITH_FREE_OVERHEAD=1 INPUT_SIZE=${input_size} PROBLEM_SIZE=${num_queries}; then bin/bs_host -w 0 -e 100 fi @@ -31,15 +32,37 @@ export -f run_benchmark_baseline ( +echo "NMC single-node upstream-ref (1/4)" >&2 + parallel -j1 --eta --joblog ${fn}.1.joblog --resume --header : \ - run_benchmark_nmc nr_dpus={nr_dpus} nr_tasklets=16 \ + run_benchmark_nmc nr_dpus={nr_dpus} nr_tasklets=16 numa_rank={numa_rank} \ num_queries=${num_queries_upstream} input_size=${input_size_upstream} \ - ::: nr_dpus 64 128 256 512 768 1024 1536 2048 2304 + ::: numa_rank 0 1 \ + ::: nr_dpus 64 128 256 512 768 1024 + +echo "NMC multi-node upstream-ref (2/4)" >&2 + +parallel -j1 --eta --joblog ${fn}.1.joblog --resume --header : \ + run_benchmark_nmc nr_dpus={nr_dpus} nr_tasklets=16 numa_rank={numa_rank} \ + num_queries=${num_queries_upstream} input_size=${input_size_upstream} \ + ::: numa_rank -1 \ + ::: nr_dpus 1536 2048 2304 + +echo "NMC single-node DPU-ref (3/4)" >&2 + +parallel -j1 --eta --joblog ${fn}.2.joblog --resume --header : \ + run_benchmark_nmc nr_dpus={nr_dpus} nr_tasklets=16 numa_rank={numa_rank} \ + num_queries=${num_queries_dpu} input_size=${input_size_dpu} \ + ::: numa_rank 0 1 \ + ::: nr_dpus 64 128 256 512 768 1024 + +echo "NMC multi-node DPU-ref (4/4)" >&2 parallel -j1 --eta --joblog ${fn}.2.joblog --resume --header : \ - run_benchmark_nmc nr_dpus={nr_dpus} nr_tasklets=16 \ + run_benchmark_nmc nr_dpus={nr_dpus} nr_tasklets=16 numa_rank={numa_rank} \ num_queries=${num_queries_dpu} input_size=${input_size_dpu} \ - ::: nr_dpus 64 128 256 512 768 1024 1536 2048 2304 + ::: numa_rank -1 \ + ::: nr_dpus 1536 2048 2304 ) > ${fn}.txt @@ -50,6 +73,8 @@ make -B NUMA=1 ( +echo "CPU single-node upstream-ref (1/4)" >&2 + parallel -j1 --eta --joblog ${fn}.1.joblog --resume --header : \ run_benchmark_baseline i={i} nr_threads={nr_threads} ram={ram} cpu={cpu} \ num_queries=${num_queries_upstream} input_size=${input_size_upstream} \ @@ -58,6 +83,8 @@ parallel -j1 --eta --joblog ${fn}.1.joblog --resume --header : \ ::: ram 0 1 \ ::: nr_threads 1 2 4 8 12 16 32 +echo "CPU single-node DPU-ref (2/4)" >&2 + parallel -j1 --eta --joblog ${fn}.2.joblog --resume --header : \ run_benchmark_baseline i={i} nr_threads={nr_threads} ram={ram} cpu={cpu} \ num_queries=${num_queries_dpu} input_size=${input_size_dpu} \ @@ -66,6 +93,8 @@ parallel -j1 --eta --joblog ${fn}.2.joblog --resume --header : \ ::: ram 0 1 \ ::: nr_threads 1 2 4 8 12 16 32 +echo "CPU multi-node upstream-ref (3/4)" >&2 + parallel -j1 --eta --joblog ${fn}.3.joblog --resume --header : \ run_benchmark_baseline i={i} nr_threads={nr_threads} ram={ram} cpu={cpu} \ num_queries=${num_queries_upstream} input_size=${input_size_upstream} \ @@ -74,6 +103,8 @@ parallel -j1 --eta --joblog ${fn}.3.joblog --resume --header : \ ::: ram 0 1 \ ::: nr_threads 48 64 +echo "CPU multi-node DPU-ref (4/4)" >&2 + parallel -j1 --eta --joblog ${fn}.4.joblog --resume --header : \ run_benchmark_baseline i={i} nr_threads={nr_threads} ram={ram} cpu={cpu} \ num_queries=${num_queries_dpu} input_size=${input_size_dpu} \ diff --git a/BS/host/app.c b/BS/host/app.c index 6d8db6a..85eb883 100644 --- a/BS/host/app.c +++ b/BS/host/app.c @@ -18,10 +18,8 @@ #include #endif -#if WITH_DPUINFO #include #include -#endif #define XSTR(x) STR(x) #define STR(x) #x @@ -89,6 +87,8 @@ int main(int argc, char **argv) { // Timer declaration Timer timer; + int numa_node_rank = -2; + // Allocate DPUs and load binary #if !WITH_ALLOC_OVERHEAD DPU_ASSERT(dpu_alloc(NR_DPUS, NULL, &dpu_set)); @@ -162,6 +162,23 @@ int main(int argc, char **argv) { assert(nr_of_dpus == NR_DPUS); #endif + // int prev_rank_id = -1; + int rank_id = -1; + DPU_FOREACH (dpu_set, dpu) { + rank_id = dpu_get_rank_id(dpu_get_rank(dpu_from_set(dpu))) & DPU_TARGET_MASK; + if ((numa_node_rank != -2) && numa_node_rank != dpu_get_rank_numa_node(dpu_get_rank(dpu_from_set(dpu)))) { + numa_node_rank = -1; + } else { + numa_node_rank = dpu_get_rank_numa_node(dpu_get_rank(dpu_from_set(dpu))); + } + /* + if (rank_id != prev_rank_id) { + printf("/dev/dpu_rank%d @ NUMA node %d\n", rank_id, numa_node_rank); + prev_rank_id = rank_id; + } + */ + } + // Compute host solution if(rep >= p.n_warmup) { start(&timer, 2, 0); @@ -283,8 +300,8 @@ int main(int argc, char **argv) { if (rep >= p.n_warmup) { printf("[::] BS-UPMEM | n_dpus=%d n_ranks=%d n_tasklets=%d e_type=%s block_size_B=%d n_elements=%lu", NR_DPUS, nr_of_ranks, NR_TASKLETS, XSTR(DTYPE), BLOCK_SIZE, input_size); - printf(" with_alloc_overhead=%d b_with_load_overhead=%d b_with_free_overhead=%d ", - WITH_ALLOC_OVERHEAD, WITH_LOAD_OVERHEAD, WITH_FREE_OVERHEAD); + printf(" with_alloc_overhead=%d b_with_load_overhead=%d b_with_free_overhead=%d numa_node_rank=%d ", + WITH_ALLOC_OVERHEAD, WITH_LOAD_OVERHEAD, WITH_FREE_OVERHEAD, numa_node_rank); printf("| latency_alloc_us=%f latency_load_us=%f latency_cpu_us=%f latency_write_us=%f latency_kernel_us=%f latency_read_us=%f latency_free_us=%f", timer.time[0], timer.time[1], -- cgit v1.2.3