https://www.bsc.es/supportkc/assets/js/657f89df.07ef1113.js

1"use strict";(self.webpackChunksupportkc_new=self.webpackChunksupportkc_new||[]).push([[4308],{3905:(e,t,n)=>{n.d(t,{Zo:()=>c,kt:()=>k});var a=n(67294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function o(e,t){if(null==e)return{};var n,a,r=function(e,t){if(null==e)return{};var n,a,r={},i=Object.keys(e);for(a=0;a<i.length;a++)n=i[a],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a<i.length;a++)n=i[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var p=a.createContext({}),s=function(e){var t=a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},c=function(e){var t=s(e.components);return a.createElement(p.Provider,{value:t},e.children)},u="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},d=a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,p=e.parentName,c=o(e,["components","mdxType","originalType","parentName"]),u=s(n),d=r,k=u["".concat(p,".").concat(d)]||u[d]||m[d]||i;return n?a.createElement(k,l(l({ref:t},c),{},{components:n})):a.createElement(k,l({ref:t},c))}));function k(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,l=new Array(i);l[0]=d;var o={};for(var p in t)hasOwnProperty.call(t,p)&&(o[p]=t[p]);o.originalType=e,o[u]="string"==typeof e?e:r,l[1]=o;for(var s=2;s<i;s++)l[s]=n[s];return a.createElement.apply(null,l)}return a.createElement.apply(null,n)}d.displayName="MDXCreateElement"},52904:(e,t,n)=>{n.r(t),n.d(t,{Card:()=>u,assets:()=>s,contentTitle:()=>o,default:()=>k,frontMatter:()=>l,metadata:()=>p,toc:()=>c});var a=n(87462),r=(n(67294),n(3905)),i=n(86010);const l={sidebar_position:2,sidebar_label:"ACC applications"},o="Accelerated Partition Applications",p={unversionedId:"MareNostrum5/Marenostrum5-Applications/ACC/ACC",id:"MareNostrum5/Marenostrum5-Applications/ACC/ACC",title:"Accelerated Partition Applications",description:"This section covers GPU-accelerated applications running on the ACC. These workloads utilize CUDA, OpenACC to exploit GPU parallelism for higher computational efficiency. The execution process, GPU optimizations, and performance scaling will be examined for each application.",source:"@site/docs/MareNostrum5/Marenostrum5-Applications/ACC/ACC.md",sourceDirName:"MareNostrum5/Marenostrum5-Applications/ACC",slug:"/MareNostrum5/Marenostrum5-Applications/ACC/",permalink:"/supportkc/docs/MareNostrum5/Marenostrum5-Applications/ACC/",draft:!1,tags:[],version:"current",sidebarPosition:2,frontMatter:{sidebar_position:2,sidebar_label:"ACC applications"},sidebar:"tutorialSidebar",previous:{title:"VASP",permalink:"/supportkc/docs/MareNostrum5/Marenostrum5-Applications/GPP/VASP"},next:{title:"CP2K",permalink:"/supportkc/docs/MareNostrum5/Marenostrum5-Applications/ACC/CP2K"}},s={},c=[{value:"<code>wrapper.sh</code>: Resource-Aware MPI Wrapper Script",id:"wrappersh-resource-aware-mpi-wrapper-script",level:2},{value:"\ud83e\udde0 Purpose",id:"-purpose",level:3},{value:"\u2699\ufe0f How It Works",id:"\ufe0f-how-it-works",level:3},{value:"\u25b6\ufe0f Example Usage",id:"\ufe0f-example-usage",level:3},{value:"\ud83d\udcdd Full Script",id:"-full-script",level:3},{value:"\ud83d\udca1 Notes",id:"-notes",level:3},{value:"\u2705 Summary",id:"-summary",level:3}],u=e=>{let{children:t,title:n,to:a}=e;return(0,r.kt)("div",{className:(0,i.Z)("col col--6")},(0,r.kt)("a",{className:"card margin-bottom--lg padding--lg cardContainer_OZJu cardContainerLink_18OG",href:a},(0,r.kt)("div",{className:"text--center padding-horiz--md"},(0,r.kt)("h2",null,n),(0,r.kt)("div",{class:"text--truncate cardDescription_5VZh",title:t},t))))},m={toc:c,Card:u},d="wrapper";

1function k(e){let{components:t,...n}=e;return(0,r.kt)(d,(0,a.Z)({},m,n,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"accelerated-partition-applications"},"Accelerated Partition Applications"),(0,r.kt)("p",null,"This section covers GPU-accelerated applications running on the ACC. These workloads utilize CUDA, OpenACC to exploit GPU parallelism for higher computational efficiency. The execution process, GPU optimizations, and performance scaling will be examined for each application."),(0,r.kt)("p",null,"For each dataset, we tested different combinations of ",(0,r.kt)("strong",{parentName:"p"},"MPI ranks")," and ",(0,r.kt)("strong",{parentName:"p"},"OpenMP threads")," on a single ",(0,r.kt)("strong",{parentName:"p"},"ACC node")," (featuring 80 CPU cores and 4x NVIDIA Hopper H100) to identify the optimal configuration. After finding the best configuration, we conducted strong scaling tests using multiple nodes."),(0,r.kt)("p",null,"The following applications have been benchmarked and analyzed on the GPP to showcase their performance and scalability on ACC nodes:"),(0,r.kt)("section",{className:"features"},(0,r.kt)("div",{className:"container"},(0,r.kt)("div",{className:"row"},(0,r.kt)(u,{to:"/supportkc/docs/MareNostrum5/Marenostrum5-Applications/ACC/CP2K",title:"CP2K",mdxType:"Card"}),(0,r.kt)(u,{to:"/supportkc/docs/MareNostrum5/Marenostrum5-Applications/ACC/GROMACS",title:"GROMACS",mdxType:"Card"}),(0,r.kt)(u,{to:"/supportkc/docs/MareNostrum5/Marenostrum5-Applications/ACC/NAMD",title:"NAMD",mdxType:"Card"}),(0,r.kt)(u,{to:"/supportkc/docs/MareNostrum5/Marenostrum5-Applications/ACC/QuantumESPRESSO",title:"QuantumESPRESSO",mdxType:"Card"}),(0,r.kt)(u,{to:"/supportkc/docs/MareNostrum5/Marenostrum5-Applications/ACC/VASP",title:"VASP",mdxType:"Card"})))),(0,r.kt)("p",null,"For each application, you will find its optimal configuration, strong scaling behavior, and a sample job script tailored for the ACC environment."),(0,r.kt)("p",null,"The CPU core binding is enabled when running on the ACC partition, using wrapper.sh along with application-specific flags."),(0,r.kt)("h2",{id:"wrappersh-resource-aware-mpi-wrapper-script"},(0,r.kt)("inlineCode",{parentName:"h2"},"wrapper.sh"),": Resource-Aware MPI Wrapper Script"),(0,r.kt)("p",null,"This script is used to ",(0,r.kt)("strong",{parentName:"p"},"assign each MPI process on a node")," to a specific ",(0,r.kt)("strong",{parentName:"p"},"GPU, CPU core range, and InfiniBand (IB) device"),". It ensures optimal use of compute and communication resources by avoiding contention and promoting NUMA locality. The wrapper is particularly useful in our setup, where we run 4 MPI processes per node, each using 1 GPU."),(0,r.kt)("hr",null),(0,r.kt)("h3",{id:"-purpose"},"\ud83e\udde0 Purpose"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Bind each MPI rank to a specific ",(0,r.kt)("strong",{parentName:"li"},"GPU")," using ",(0,r.kt)("inlineCode",{parentName:"li"},"CUDA_VISIBLE_DEVICES")),(0,r.kt)("li",{parentName:"ul"},"Bind each MPI rank to a ",(0,r.kt)("strong",{parentName:"li"},"CPU core range")," using ",(0,r.kt)("inlineCode",{parentName:"li"},"numactl")),(0,r.kt)("li",{parentName:"ul"},"Assign a specific ",(0,r.kt)("strong",{parentName:"li"},"UCX-compatible network interface")," (e.g., InfiniBand device) using ",(0,r.kt)("inlineCode",{parentName:"li"},"UCX_NET_DEVICES"))),(0,r.kt)("p",null,"This is especially useful on multi-GPU, multi-NIC nodes where you want fine-grained control over hardware assignment."),(0,r.kt)("hr",null),(0,r.kt)("h3",{id:"\ufe0f-how-it-works"},"\u2699\ufe0f How It Works"),(0,r.kt)("ol",null,(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"The script determines the ",(0,r.kt)("strong",{parentName:"p"},"local MPI rank")," using:"),(0,r.kt)("pre",{parentName:"li"},(0,r.kt)("code",{parentName:"pre",className:"language-bash"},"lrank=${OMPI_COMM_WORLD_LOCAL_RANK:-${SLURM_LOCALID}}\n")),(0,r.kt)("p",{parentName:"li"},"This uses the ",(0,r.kt)("inlineCode",{parentName:"p"},"OMPI_COMM_WORLD_LOCAL_RANK")," variable if available (OpenMPI), or ",(0,r.kt)("inlineCode",{parentName:"p"},"SLURM_LOCALID")," if running under SLURM.")),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"It calculates the rank module 4 to classify the process for resource allocation:"),(0,r.kt)("pre",{parentName:"li"},(0,r.kt)("code",{parentName:"pre",className:"language-bash"},"lrank2=$(( lrank % 4 ))\n"))),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Based on the value of ",(0,r.kt)("inlineCode",{parentName:"p"},"lrank2"),", the script selects a GPU, CPU core range, and IB interface:"),(0,r.kt)("table",{parentName:"li"},(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Local Rank (",(0,r.kt)("inlineCode",{parentName:"th"},"lrank2"),")"),(0,r.kt)("th",{parentName:"tr",align:null},"GPU (CUDA_VISIBLE_DEVICES)"),(0,r.kt)("th",{parentName:"tr",align:null},"CPU Core Range (",(0,r.kt)("inlineCode",{parentName:"th"},"mycores"),")"),(0,r.kt)("th",{parentName:"tr",align:null},"UCX Network Device (",(0,r.kt)("inlineCode",{parentName:"th"},"UCX_NET_DEVICES"),")"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"0"),(0,r.kt)("td",{parentName:"tr",align:null},"0"),(0,r.kt)("td",{parentName:"tr",align:null},"0\u201319"),(0,r.kt)("td",{parentName:"tr",align:null},"mlx5_0:1")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"1"),(0,r.kt)("td",{parentName:"tr",align:null},"1"),(0,r.kt)("td",{parentName:"tr",align:null},"20\u201339"),(0,r.kt)("td",{parentName:"tr",align:null},"mlx5_1:1")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"2"),(0,r.kt)("td",{parentName:"tr",align:null},"2"),(0,r.kt)("td",{parentName:"tr",align:null},"40\u201359"),(0,r.kt)("td",{parentName:"tr",align:null},"mlx5_4:1")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"3"),(0,r.kt)("td",{parentName:"tr",align:null},"3"),(0,r.kt)("td",{parentName:"tr",align:null},"60\u201379"),(0,r.kt)("td",{parentName:"tr",align:null},"mlx5_5:1"))))),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"These settings are exported in the environment:"),(0,r.kt)("pre",{parentName:"li"},(0,r.kt)("code",{parentName:"pre",className:"language-bash"},'export CUDA_VISIBLE_DEVICES=...\nexport mycores="..."\nexport UCX_NET_DEVICES=...\n'))),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"The application is launched with CPU core binding:"),(0,r.kt)("pre",{parentName:"li"},(0,r.kt)("code",{parentName:"pre",className:"language-bash"},"numactl --physcpubind=$mycores $@\n"))),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"If the rank exceeds the expected range (more than 3), the script exits:"),(0,r.kt)("pre",{parentName:"li"},(0,r.kt)("code",{parentName:"pre",className:"language-bash"},'echo "ERROR: MPI local rank larger than expected! -> $lrank"\nexit 1\n')))),(0,r.kt)("hr",null),(0,r.kt)("h3",{id:"\ufe0f-example-usage"},"\u25b6\ufe0f Example Usage"),(0,r.kt)("p",null,"With SLURM:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-bash"},"srun -n 4 ./wrapper.sh ./my_gpu_app --input config.in\n")),(0,r.kt)("p",null,"With ",(0,r.kt)("inlineCode",{parentName:"p"},"mpirun"),":"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-bash"},"mpirun -np 4 ./wrapper.sh ./my_gpu_app --input config.in\n")),(0,r.kt)("hr",null),(0,r.kt)("h3",{id:"-full-script"},"\ud83d\udcdd Full Script"),(0,r.kt)("p",null,"Below is the complete content of the ",(0,r.kt)("inlineCode",{parentName:"p"},"wrapper.sh")," script:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-bash"},'#!/bin/bash\n\nlrank=${OMPI_COMM_WORLD_LOCAL_RANK:-${SLURM_LOCALID}}

1\nlrank2=$(( lrank % 4 ))\n\ncase $lrank2 in\n    0)\n        export CUDA_VISIBLE_DEVICES=0\n        export mycores="0-19"\n        export UCX_NET_DEVICES=mlx5_0:1\n        ;;\n    1)\n        export CUDA_VISIBLE_DEVICES=1\n        export mycores="20-39"\n        export UCX_NET_DEVICES=mlx5_1:1\n        ;;\n    2)\n        export CUDA_VISIBLE_DEVICES=2\n        export mycores="40-59"\n        export UCX_NET_DEVICES=mlx5_4:1\n        ;;\n    3)\n        export CUDA_VISIBLE_DEVICES=3\n        export mycores="60-79"\n        export UCX_NET_DEVICES=mlx5_5:1\n        ;;\n    *)\n        echo "ERROR: MPI local rank larger than expected! -> $lrank"\n        exit 1\nesac\n\necho "numactl --physcpubind=$mycores $@ using CUDA $CUDA_VISIBLE_DEVICES"\n\nnumactl --physcpubind=$mycores $@\n')),(0,r.kt)("hr",null),(0,r.kt)("h3",{id:"-notes"},"\ud83d\udca1 Notes"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Make sure the values for ",(0,r.kt)("inlineCode",{parentName:"li"},"UCX_NET_DEVICES")," correspond to actual devices (check with ",(0,r.kt)("inlineCode",{parentName:"li"},"ibv_devinfo"),")."),(0,r.kt)("li",{parentName:"ul"},"This script currently supports up to 4 MPI processes per node. To support more, extend the ",(0,r.kt)("inlineCode",{parentName:"li"},"case")," block accordingly.")),(0,r.kt)("hr",null),(0,r.kt)("h3",{id:"-summary"},"\u2705 Summary"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"wrapper.sh")," is a simple but powerful utility for resource-aware execution of MPI+GPU jobs."),(0,r.kt)("li",{parentName:"ul"},"Helps prevent resource contention and improves performance on multi-GPU, multi-core systems."),(0,r.kt)("li",{parentName:"ul"},"Works in SLURM and OpenMPI environments.")))}k.isMDXComponent=!0}}]);