From 9612a7d5e3b685f9d1d9af559999999ce2b3880c Mon Sep 17 00:00:00 2001 From: Birte Kristina Friesel Date: Wed, 17 Jul 2024 15:00:09 +0200 Subject: VA memcpy: HBM support --- VA/baselines/cpu/app_baseline.c | 23 +++++++++++++++------- VA/dimes-hetsim-hbm.sh | 42 +++++++++++++++++++++++++++++++++-------- VA/dimes-hetsim-nmc.sh | 23 ++++++++++++---------- 3 files changed, 63 insertions(+), 25 deletions(-) diff --git a/VA/baselines/cpu/app_baseline.c b/VA/baselines/cpu/app_baseline.c index 84f1486..f5981ac 100644 --- a/VA/baselines/cpu/app_baseline.c +++ b/VA/baselines/cpu/app_baseline.c @@ -39,6 +39,7 @@ static T *B; static T *C; #if NUMA_MEMCPY +int numa_node_local = -1; int numa_node_in_is_local = 0; static T *A_local; static T *B_local; @@ -109,7 +110,7 @@ struct Params input_params(int argc, char **argv) { #endif int opt; - while((opt = getopt(argc, argv, "hi:w:e:x:t:a:b:c:")) >= 0) { + while((opt = getopt(argc, argv, "hi:w:e:x:t:a:b:c:C:")) >= 0) { switch(opt) { case 'h': usage(); @@ -123,11 +124,9 @@ struct Params input_params(int argc, char **argv) { #if NUMA case 'a': p.bitmask_in = numa_parse_nodestring(optarg); break; case 'b': p.bitmask_out = numa_parse_nodestring(optarg); break; -#if NUMA_MEMCPY - case 'c': p.numa_node_cpu = atoi(optarg); - p.bitmask_cpu = numa_parse_nodestring(optarg); break; -#else case 'c': p.numa_node_cpu = atoi(optarg); break; +#if NUMA_MEMCPY + case 'C': p.bitmask_cpu = numa_parse_nodestring(optarg); break; #endif // NUMA_MEMCPY #endif // NUMA default: @@ -254,6 +253,16 @@ int main(int argc, char **argv) { A_local = A; B_local = B; } + mp_pages[0] = A_local; + if (move_pages(0, 1, mp_pages, NULL, mp_status, 0) == -1) { + perror("move_pages(A_local)"); + } + else if (mp_status[0] < 0) { + printf("move_pages error: %d", mp_status[0]); + } + else { + numa_node_local = mp_status[0]; + } stop(&timer, 2); #endif @@ -278,10 +287,10 @@ int main(int argc, char **argv) { if (rep >= p.n_warmup) { #if NUMA_MEMCPY printf("[::] VA-CPU-MEMCPY | n_threads=%d e_type=%s n_elements=%d" - " numa_node_in=%d numa_node_out=%d numa_node_cpu=%d numa_distance_in_cpu=%d numa_distance_cpu_out=%d" + " numa_node_in=%d numa_node_local=%d numa_node_out=%d numa_node_cpu=%d numa_distance_in_cpu=%d numa_distance_cpu_out=%d" " | throughput_MBps=%f", nr_threads, XSTR(T), input_size, - numa_node_in, numa_node_out, numa_node_cpu, numa_distance(numa_node_in, numa_node_cpu), numa_distance(numa_node_cpu, numa_node_out), + numa_node_in, numa_node_local, numa_node_out, numa_node_cpu, numa_distance(numa_node_in, numa_node_cpu), numa_distance(numa_node_cpu, numa_node_out), input_size * 3 * sizeof(T) / timer.time[0]); printf(" throughput_MOpps=%f", input_size / timer.time[0]); diff --git a/VA/dimes-hetsim-hbm.sh b/VA/dimes-hetsim-hbm.sh index a7c9ef4..7c0df73 100755 --- a/VA/dimes-hetsim-hbm.sh +++ b/VA/dimes-hetsim-hbm.sh @@ -1,7 +1,6 @@ #!/bin/sh cd baselines/cpu -make -B NUMA=1 mkdir -p log/$(hostname) fn=log/$(hostname)/dimes-hetsim-hbm @@ -10,22 +9,49 @@ fn=log/$(hostname)/dimes-hetsim-hbm ( -echo "single-node execution (1/2)" >&2 +make -B NUMA=1 NUMA_MEMCPY=1 + +echo "CPU single-node operation with setup cost, cpu/out on same node (1/4)" >&2 parallel -j1 --eta --joblog ${fn}.1.joblog --resume --header : \ - ./va -i {input_size} -a {ram} -b {ram} -c {cpu} -t {nr_threads} -w 0 -e 5 \ + ./va -i {input_size} -a {ram_in} -b {ram_out} -c {cpu} -C {ram_local} -t {nr_threads} -w 0 -e 40 \ ::: nr_threads 1 2 4 8 12 16 \ - ::: cpu $(seq 0 7) \ - ::: ram $(seq 0 15) \ + ::: ram_in $(seq 0 15) \ + ::: cpu $(seq 0 7) $(seq 0 7) \ + :::+ ram_local $(seq 0 15) \ + :::+ ram_out $(seq 0 15) \ ::: input_size 167772160 -echo "multi-node execution (2/2)" >&2 +make -B NUMA=1 + +echo "single-node execution, cpu/out on same node (2/4)" >&2 parallel -j1 --eta --joblog ${fn}.2.joblog --resume --header : \ - ./va -i {input_size} -a {ram} -b {ram} -c {cpu} -t {nr_threads} -w 0 -e 40 \ + ./va -i {input_size} -a {ram_in} -b {ram_out} -c {cpu} -t {nr_threads} -w 0 -e 5 \ + ::: nr_threads 1 2 4 8 12 16 \ + ::: ram_in $(seq 0 15) \ + ::: cpu $(seq 0 7) $(seq 0 7) \ + :::+ ram_out $(seq 0 15) \ + ::: input_size 167772160 + +echo "single-node execution, in/out on same node (3/4)" >&2 + +parallel -j1 --eta --joblog ${fn}.3.joblog --resume --header : \ + ./va -i {input_size} -a {ram_in} -b {ram_out} -c {cpu} -t {nr_threads} -w 0 -e 5 \ + ::: nr_threads 1 2 4 8 12 16 \ + ::: cpu $(seq 0 7) \ + ::: ram_in $(seq 0 15) \ + :::+ ram_out $(seq 0 15) \ + ::: input_size 167772160 + +echo "multi-node execution (4/4)" >&2 + +parallel -j1 --eta --joblog ${fn}.4.joblog --resume --header : \ + ./va -i {input_size} -a {ram_in} -b {ram_out} -c {cpu} -t {nr_threads} -w 0 -e 40 \ ::: nr_threads 32 48 64 96 128 \ ::: cpu -1 \ - ::: ram $(seq 0 15) \ + ::: ram_in $(seq 0 15) \ + ::: ram_out $(seq 0 15) \ ::: input_size 167772160 ) >> ${fn}.txt diff --git a/VA/dimes-hetsim-nmc.sh b/VA/dimes-hetsim-nmc.sh index fa9e975..1e33c09 100755 --- a/VA/dimes-hetsim-nmc.sh +++ b/VA/dimes-hetsim-nmc.sh @@ -40,20 +40,22 @@ make -B NUMA=1 ( -echo "CPU single-node operation (1/2)" >&2 +echo "CPU single-node operation (1/4)" >&2 parallel -j1 --eta --joblog ${fn}.1.joblog --resume --header : \ - ./va -i {input_size} -a {ram} -b {ram} -c {cpu} -t {nr_threads} -w 0 -e 40 \ - ::: ram 0 1 \ - ::: cpu 0 1 \ + ./va -i {input_size} -a {ram_in} -b {ram_out} -c {cpu} -t {nr_threads} -w 0 -e 40 \ + ::: ram_in 0 1 \ + ::: cpu 0 1 \ + :::+ ram_out 0 1 \ ::: nr_threads 1 2 4 8 12 16 \ ::: input_size 167772160 -echo "CPU multi-node operation (1/2)" >&2 +echo "CPU multi-node operation (2/4)" >&2 parallel -j1 --eta --joblog ${fn}.2.joblog --resume --header : \ - ./va -i {input_size} -a {ram} -b {ram} -c {cpu} -t {nr_threads} -w 0 -e 40 \ - ::: ram 0 1 \ + ./va -i {input_size} -a {ram_in} -b {ram_out} -c {cpu} -t {nr_threads} -w 0 -e 40 \ + ::: ram_in 0 1 \ + ::: ram_out 0 1 \ ::: cpu -1 \ ::: nr_threads 24 32 \ ::: input_size 167772160 @@ -66,10 +68,11 @@ make -B NUMA=1 NUMA_MEMCPY=1 echo "CPU single-node operation with setup cost" >&2 -parallel -j1 --eta --joblog ${fn}.1.joblog --resume --header : \ - ./va -i {input_size} -a {ram} -b {cpu} -c {cpu} -t {nr_threads} -w 0 -e 40 \ +parallel -j1 --eta --joblog ${fn}.3.joblog --resume --header : \ + ./va -i {input_size} -a {ram} -b {ram_out} -c {cpu} -C {cpu} -t {nr_threads} -w 0 -e 40 \ ::: ram 0 1 \ - ::: cpu 0 1 \ + ::: cpu 0 1 \ + :::+ ram_out 0 1 \ ::: nr_threads 1 2 4 8 12 16 \ ::: input_size 167772160 -- cgit v1.2.3