From 569709f82e52078f14cf05961565a58aee7893be Mon Sep 17 00:00:00 2001 From: Birte Kristina Friesel Date: Tue, 23 Jul 2024 16:58:27 +0200 Subject: GEMV: specify memcpy cpu node --- GEMV/baselines/cpu/gemv_openmp.c | 24 ++++++++++++++-- GEMV/dimes-hetsim-hbm.sh | 47 ++++++++++++++++++++----------- GEMV/dimes-hetsim-nmc.sh | 61 ++++++++++++++++++++++++---------------- 3 files changed, 87 insertions(+), 45 deletions(-) (limited to 'GEMV') diff --git a/GEMV/baselines/cpu/gemv_openmp.c b/GEMV/baselines/cpu/gemv_openmp.c index 1a3eed1..21e24cb 100644 --- a/GEMV/baselines/cpu/gemv_openmp.c +++ b/GEMV/baselines/cpu/gemv_openmp.c @@ -23,6 +23,7 @@ int numa_node_cpu = -1; #if NUMA_MEMCPY struct bitmask* bitmask_cpu; +int numa_node_cpu_memcpy = -1; int numa_node_local = -1; int numa_node_in_is_local = 0; #endif @@ -54,6 +55,7 @@ int main(int argc, char *argv[]) numa_node_cpu = atoi(argv[3]); #if NUMA_MEMCPY bitmask_cpu = numa_parse_nodestring(argv[4]); + numa_node_cpu_memcpy = atoi(argv[5]); #endif // NUMA_MEMCPY #else (void) argv; @@ -142,7 +144,7 @@ int main(int argc, char *argv[]) #endif Timer timer; - for (int i = 0; i < 40; i++) { + for (int i = 0; i < 20; i++) { #pragma omp parallel { @@ -172,6 +174,15 @@ int main(int argc, char *argv[]) return 1; } + if (!numa_node_in_is_local) { + if (numa_node_cpu_memcpy != -1) { + if (numa_run_on_node(numa_node_cpu_memcpy) == -1) { + perror("numa_run_on_node"); + numa_node_cpu_memcpy = -1; + } + } + } + start(&timer, 2, 0); if (!numa_node_in_is_local) { //for (size_t i=0; i < rows; i++ ) { @@ -185,6 +196,13 @@ int main(int argc, char *argv[]) } stop(&timer, 2); + if (numa_node_cpu != -1) { + if (numa_run_on_node(numa_node_cpu) == -1) { + perror("numa_run_on_node"); + numa_node_cpu = -1; + } + } + mp_pages[0] = A_local; if (move_pages(0, 1, mp_pages, NULL, mp_status, 0) == -1) { perror("move_pages(A_local)"); @@ -218,10 +236,10 @@ int main(int argc, char *argv[]) #if NUMA_MEMCPY printf("[::] GEMV-CPU-MEMCPY | n_threads=%d e_type=%s n_elements=%ld" - " numa_node_in=%d numa_node_out=%d numa_node_cpu=%d numa_node_local=%d numa_distance_in_cpu=%d numa_distance_cpu_out=%d" + " numa_node_in=%d numa_node_out=%d numa_node_cpu=%d numa_node_local=%d numa_node_cpu_memcpy=%d numa_distance_in_cpu=%d numa_distance_cpu_out=%d" " | throughput_MBps=%f throughput_MOpps=%f", nr_threads, XSTR(T), rows * cols, - numa_node_in, numa_node_out, numa_node_cpu, numa_node_local, numa_distance(numa_node_in, numa_node_cpu), numa_distance(numa_node_cpu, numa_node_out), + numa_node_in, numa_node_out, numa_node_cpu, numa_node_local, numa_node_cpu_memcpy, numa_distance(numa_node_in, numa_node_cpu), numa_distance(numa_node_cpu, numa_node_out), rows * cols * sizeof(T) / timer.time[0], rows * cols / timer.time[0]); printf(" latency_kernel_us=%f latency_alloc_us=%f latency_memcpy_us=%f latency_free_us=%f latency_total_us=%f\n", diff --git a/GEMV/dimes-hetsim-hbm.sh b/GEMV/dimes-hetsim-hbm.sh index ad6895e..bbba9e0 100755 --- a/GEMV/dimes-hetsim-hbm.sh +++ b/GEMV/dimes-hetsim-hbm.sh @@ -1,10 +1,9 @@ #!/bin/bash cd baselines/cpu -make -B NUMA=1 mkdir -p log/$(hostname) -fn=log/$(hostname)/$(date +%Y%m%d) +fn=log/$(hostname)/dimes-hetsim-hbm # upstream DPU version uses -m 163840 -n 4096 # → (163840 * 4096 + 4096) * uint32 ≈ 2.5 GiB @@ -14,7 +13,7 @@ fn=log/$(hostname)/$(date +%Y%m%d) run_benchmark() { local "$@" - OMP_NUM_THREADS=$nr_threads ./gemv $ram $ram $cpu + OMP_NUM_THREADS=$nr_threads ./gemv $ram_in $ram_out $cpu $ram_local $cpu_memcpy return $? } @@ -22,22 +21,36 @@ export -f run_benchmark ( -echo "CPU single-node (1/2)" >&2 +make -B NUMA=1 NUMA_MEMCPY=1 -parallel -j1 --eta --joblog ${fn}.1.joblog --header : \ - run_benchmark nr_threads={nr_threads} ram={ram} cpu={cpu} \ - ::: cpu $(seq 0 7) \ - ::: ram $(seq 0 15) \ - ::: nr_threads 1 2 4 8 12 16 +echo "CPU single-node operation with setup cost, memcpy node == input node, cpu node == output node (1/3)" >&2 -echo "CPU multi-node (2/2)" >&2 +parallel -j1 --eta --joblog ${fn}.1.joblog --resume --header : \ + run_benchmark nr_threads={nr_threads} ram_in={ram_in} ram_out={ram_out} ram_local={ram_local} cpu={cpu} cpu_memcpy={cpu_memcpy} \ + ::: nr_threads 1 2 4 8 12 16 \ + ::: ram_in $(seq 0 15) \ + :::+ cpu_memcpy $(seq 0 7) $(seq 0 7) \ + ::: ram_local $(seq 0 15) \ + :::+ cpu $(seq 0 7) $(seq 0 7) \ + :::+ ram_out $(seq 0 15) -parallel -j1 --eta --joblog ${fn}.2.joblog --header : \ - run_benchmark nr_threads={nr_threads} ram={ram} cpu={cpu} \ - ::: cpu -1 \ - ::: ram $(seq 0 15) \ - ::: nr_threads 32 48 64 96 128 +make -B NUMA=1 -) > ${fn}.txt +echo "single-node execution, cpu/out on same node (2/3)" >&2 + +parallel -j1 --eta --joblog ${fn}.2.joblog --resume --header : \ + run_benchmark nr_threads={nr_threads} ram_in={ram_in} ram_out={ram_out} cpu={cpu} \ + ::: nr_threads 1 2 4 8 12 16 \ + ::: ram_in $(seq 0 15) \ + ::: cpu $(seq 0 7) $(seq 0 7) \ + :::+ ram_out $(seq 0 15) + +echo "multi-node execution (3/3)" >&2 +parallel -j1 --eta --joblog ${fn}.3.joblog --resume --header : \ + run_benchmark nr_threads={nr_threads} ram_in={ram_in} ram_out={ram_out} cpu={cpu} \ + ::: nr_threads 32 48 64 96 128 \ + ::: cpu -1 \ + ::: ram_in $(seq 0 15) \ + ::: ram_out $(seq 0 15) -xz -f -v -9 -M 800M ${fn}.txt +) >> ${fn}.txt diff --git a/GEMV/dimes-hetsim-nmc.sh b/GEMV/dimes-hetsim-nmc.sh index ed01cdc..416665d 100755 --- a/GEMV/dimes-hetsim-nmc.sh +++ b/GEMV/dimes-hetsim-nmc.sh @@ -1,7 +1,7 @@ #!/bin/bash mkdir -p log/$(hostname) baselines/cpu/log/$(hostname) -fn=log/$(hostname)/$(date +%Y%m%d) +fn=log/$(hostname)/dimes-hetsim-nmc # upstream DPU version uses -m 163840 -n 4096 # → (163840 * 4096 + 4096) * uint32 ≈ 2.5 GiB @@ -10,23 +10,22 @@ fn=log/$(hostname)/$(date +%Y%m%d) # → ≈ 1.25 GiB # Note: Upstream uses int32_t in DPU version and double in baseline. -# Here, we use int32_t for both. +# Here, we use (163840 * 4096 + 4096 ) * int32_t for both. run_benchmark_nmc() { local "$@" + set -e sudo limit_ranks_to_numa_node ${numa_rank} - if make -B NR_DPUS=${nr_dpus} NR_TASKLETS=${nr_tasklets} WITH_ALLOC_OVERHEAD=1 WITH_LOAD_OVERHEAD=1 WITH_FREE_OVERHEAD=1; then - bin/gemv_host -w 0 -e 100 -m 163840 -n 4096 - fi - return $? + make -B NR_DPUS=${nr_dpus} NR_TASKLETS=${nr_tasklets} WITH_ALLOC_OVERHEAD=1 WITH_LOAD_OVERHEAD=1 WITH_FREE_OVERHEAD=1 + bin/gemv_host -w 0 -e 100 -m 163840 -n 4096 } export -f run_benchmark_nmc run_benchmark_baseline() { local "$@" - OMP_NUM_THREADS=$nr_threads ./gemv $ram $ram $cpu - return $? + set -e + OMP_NUM_THREADS=$nr_threads ./gemv $ram_in $ram_out $cpu $ram_local $cpu_memcpy } export -f run_benchmark_baseline @@ -35,43 +34,55 @@ export -f run_benchmark_baseline echo "NMC single-node (1/2)" >&2 -parallel -j1 --eta --joblog ${fn}.1.joblog --header : \ +parallel -j1 --eta --joblog ${fn}.1.joblog --resume --header : \ run_benchmark_nmc nr_dpus={nr_dpus} nr_tasklets=16 numa_rank={numa_rank} \ ::: numa_rank 0 1 \ ::: nr_dpus 64 128 256 512 768 1024 echo "NMC multi-node (2/2)" >&2 -parallel -j1 --eta --joblog ${fn}.2.joblog --header : \ +parallel -j1 --eta --joblog ${fn}.2.joblog --resume --header : \ run_benchmark_nmc nr_dpus={nr_dpus} nr_tasklets=16 numa_rank={numa_rank} \ ::: numa_rank -1 \ ::: nr_dpus 1536 2048 2304 -) > ${fn}.txt - -xz -f -v -9 -M 800M ${fn}.txt +) >> ${fn}.txt cd baselines/cpu -make -B NUMA=1 TYPE=int32_t ( -echo "CPU single-node (1/2)" >&2 +make -B NUMA=1 TYPE=int32_t NUMA_MEMCPY=1 + +echo "CPU single-node operation with setup cost, memcpy/in and cpu/out on same node (1/3)" >&2 + +parallel -j1 --eta --joblog ${fn}.1.joblog --resume --header : \ + run_benchmark_baseline nr_threads={nr_threads} ram_in={ram_in} ram_out={ram_out} cpu={cpu} ram_local={ram_local} cpu_memcpy={cpu_memcpy} \ + ::: ram_in 0 1 \ + :::+ cpu_memcpy 0 1 \ + ::: cpu 0 1 \ + :::+ ram_out 0 1 \ + :::+ ram_local 0 1 \ + ::: nr_threads 1 2 4 8 12 16 -parallel -j1 --eta --joblog ${fn}.1.joblog --header : \ - run_benchmark_baseline nr_threads={nr_threads} ram={ram} cpu={cpu} \ +make -B NUMA=1 TYPE=int32_t + +echo "CPU single-node (2/3)" >&2 + +parallel -j1 --eta --joblog ${fn}.2.joblog --resume --header : \ + run_benchmark_baseline nr_threads={nr_threads} ram_in={ram_in} ram_out={ram_out} cpu={cpu} \ ::: cpu 0 1 \ - ::: ram 0 1 \ + ::: ram_in 0 1 \ + ::: ram_out 0 1 \ ::: nr_threads 1 2 4 8 12 16 -echo "CPU multi-node (2/2)" >&2 +echo "CPU multi-node (3/3)" >&2 -parallel -j1 --eta --joblog ${fn}.2.joblog --header : \ - run_benchmark_baseline nr_threads={nr_threads} ram={ram} cpu={cpu} \ +parallel -j1 --eta --joblog ${fn}.3.joblog --resume --header : \ + run_benchmark_baseline nr_threads={nr_threads} ram_in={ram_in} ram_out={ram_out} cpu={cpu} \ ::: cpu -1 \ - ::: ram 0 1 \ + ::: ram_in 0 1 \ + ::: ram_out 0 1 \ ::: nr_threads 24 32 -) > ${fn}.txt - -xz -f -v -9 -M 800M ${fn}.txt +) >> ${fn}.txt -- cgit v1.2.3