################################ ## Cluster Configuration File ## ################################ [cluster Slurm] FormLayout = selectionpanel Category = Schedulers Autoscale = $Autoscale [[node defaults]] UsePublicNetwork = $UsePublicNetwork Credentials = $Credentials SubnetId = $SubnetId Region = $Region KeyPairLocation = ~/.ssh/cyclecloud.pem Azure.Identities = $ManagedIdentity Tags = $NodeTags # Lustre mounts require termination notifications to unmount EnableTerminateNotification = ${NFSType == "lustre" || NFSSchedType == "lustre" || AdditionalNFSType == "lustre" || EnableTerminateNotification} TerminateNotificationTimeout = 10m [[[configuration]]] slurm.version = $configuration_slurm_version slurm.insiders = false slurm.user.uid = 11100 slurm.user.gid = 11100 munge.user.uid = 11101 munge.user.gid = 11101 slurm.slurmrestd.user.uid = 11102 slurm.slurmrestd.user.gid = 11102 slurm.enable_healthchecks = true slurm.accounting.enabled = $configuration_slurm_accounting_enabled slurm.accounting.url = $configuration_slurm_accounting_url slurm.accounting.user = $configuration_slurm_accounting_user slurm.accounting.password = $configuration_slurm_accounting_password slurm.accounting.certificate_url = $configuration_slurm_accounting_certificate_url slurm.accounting.storageloc = $configuration_slurm_accounting_storageloc slurm.additional.config = $additional_slurm_config slurm.ha_enabled = $configuration_slurm_ha_enabled slurm.launch_parameters = $configuration_slurm_launch_parameters # Disable ip-XXXXXXXX hostname generation cyclecloud.hosts.standalone_dns.enabled = ${NodeNameIsHostname==false} cyclecloud.hosts.simple_vpc_dns.enabled = ${NodeNameIsHostname==false} # For fast spin-up after Deallocate, force an immediate re-converge on boot cyclecloud.converge_on_boot = true # Disable normal NFS exports and mounts cyclecloud.mounts.sched.disabled = true cyclecloud.mounts.shared.disabled = true cyclecloud.exports.sched.disabled = true cyclecloud.exports.shared.disabled = true cyclecloud.exports.sched.samba.enabled = false cyclecloud.exports.shared.samba.enabled = false cyclecloud.exports.defaults.samba.enabled = false cshared.server.legacy_links_disabled = true # Monitoring cyclecloud.monitoring.enabled = $configuration_monitoring_enabled cyclecloud.monitoring.identity_client_id = $configuration_identity_client_id cyclecloud.monitoring.ingestion_endpoint = $configuration_ingestion_endpoint # May be used to identify the ID in cluster-init scripts cluster.identities.default = $ManagedIdentity [[[cluster-init cyclecloud/healthagent:default]]] [[[cluster-init cyclecloud/monitoring:default]]] [[[cluster-init cyclecloud/slurm:default:4.0.5]]] [[[volume boot]]] Size = ${ifThenElse(BootDiskSize > 0, BootDiskSize, undefined)} SSD = True [[[configuration cyclecloud.mounts.nfs_shared]]] type = $NFSType mountpoint = /shared export_path = ${ifThenElse(NFSType == "lustre", strcat("tcp:/lustrefs", NFSSharedExportPath), NFSSharedExportPath)} address = $NFSAddress options = $NFSSharedMountOptions [[[configuration cyclecloud.mounts.nfs_sched]]] type = $NFSSchedType mountpoint = /sched export_path = ${ifThenElse(NFSSchedType == "lustre", strcat("tcp:/lustrefs", NFSSchedExportPath), NFSSchedExportPath)} address = ${ifThenElse(UseBuiltinSched && !configuration_slurm_ha_enabled, undefined, NFSSchedAddress)} options = $NFSSchedMountOptions [[[configuration cyclecloud.mounts.additional_nfs]]] disabled = ${AdditionalNFS isnt true} type = $AdditionalNFSType address = $AdditionalNFSAddress mountpoint = $AdditionalNFSMountPoint export_path = ${ifThenElse(AdditionalNFSType == "lustre", strcat("tcp:/lustrefs", AdditionalNFSExportPath), AdditionalNFSExportPath)} options = $AdditionalNFSMountOptions [[node scheduler]] MachineType = $SchedulerMachineType ImageName = $SchedulerImageName IsReturnProxy = $ReturnProxy AdditionalClusterInitSpecs = $SchedulerClusterInitSpecs ComputerName = ${toLower(regexps("([^a-zA-Z0-9-])", ifThenElse(SchedulerHostName=="Cluster Prefix", StrJoin("-", ClusterName, "scheduler"), ifThenElse(Size(Trim(SchedulerHostName)) == 0 || SchedulerHostName == "Generated", undefined, SchedulerHostName)), "-"))} # indented version, for clarity. # ${toLower( # regexps("([^a-zA-Z0-9-])", # ifThenElse(SchedulerHostName=="Cluster Prefix", # StrJoin("-", ClusterName, "scheduler"), # ifThenElse(Size(Trim(SchedulerHostName)) == 0 || SchedulerHostName == "Generated", # undefined, # SchedulerHostName)), # "-"))} Zone = ${ifThenElse(configuration_slurm_ha_enabled, SchedulerZone, undefined)} [[[configuration]]] slurm.role = scheduler # Disable NFS mount of built-in /sched since it is a local volume mount: cyclecloud.mounts.builtinsched cyclecloud.mounts.nfs_sched.disabled = ${UseBuiltinSched && !configuration_slurm_ha_enabled} cyclecloud.mounts.nfs_shared.disabled = ${UseBuiltinShared && !configuration_slurm_ha_enabled} slurm.secondary_scheduler_name = ${ifThenElse(configuration_slurm_ha_enabled, "scheduler-ha-1", undefined)} [[[cluster-init cyclecloud/healthagent:default]]] [[[cluster-init cyclecloud/monitoring:default]]] [[[cluster-init cyclecloud/slurm:scheduler:4.0.5]]] [[[network-interface eth0]]] AssociatePublicIpAddress = $UsePublicNetwork [[[volume sched]]] Size = $SchedFilesystemSize SSD = True Mount = builtinsched Persistent = True Disabled = ${!UseBuiltinSched || configuration_slurm_ha_enabled} [[[volume shared]]] Size = $FilesystemSize SSD = True Mount = builtinshared Persistent = True Disabled = ${!UseBuiltinShared || configuration_slurm_ha_enabled} [[[configuration cyclecloud.mounts.builtinsched]]] disabled = ${!UseBuiltinSched || configuration_slurm_ha_enabled} mountpoint = /sched fs_type = xfs [[[configuration cyclecloud.mounts.builtinshared]]] disabled = ${!UseBuiltinShared || configuration_slurm_ha_enabled} mountpoint = /shared fs_type = xfs [[[configuration cyclecloud.exports.builtinsched]]] disabled = ${!UseBuiltinSched || configuration_slurm_ha_enabled} export_path = /sched options = no_root_squash samba.enabled = false type = nfs [[[configuration cyclecloud.exports.builtinshared]]] disabled = ${!UseBuiltinShared || configuration_slurm_ha_enabled} export_path = /shared samba.enabled = false type = nfs [[nodearray scheduler-ha]] Extends = scheduler IsReturnProxy = false InitialCount = $configuration_slurm_ha_enabled Zone = $SchedulerHAZone # Do not inherit property from node that is not used in nodearray # The equivalent is ComputerNamePrefix for nodearray, however Cluster-init will handle renaming of all hosts in a VMSS ComputerName := undefined [[[configuration]]] autoscale.enabled = false slurm.node_prefix = ${ifThenElse(NodeNamePrefix=="Cluster Prefix", StrJoin("-", ClusterName, ""), NodeNamePrefix)} slurm.use_nodename_as_hostname = $NodeNameIsHostname slurm.is_primary_scheduler = false [[nodearray login]] InitialCount = $NumberLoginNodes MaxCount = $MaxLoginNodeCount MachineType = $loginMachineType ImageName = $LoginImageName AdditionalClusterInitSpecs = $LoginClusterInitSpecs [[[cluster-init cyclecloud/healthagent:default]]] [[[cluster-init cyclecloud/monitoring:default]]] [[[cluster-init cyclecloud/slurm:login:4.0.5]]] [[[configuration]]] slurm.role = login autoscale.enabled = false slurm.node_prefix = ${ifThenElse(NodeNamePrefix=="Cluster Prefix", StrJoin("-", ClusterName, ""), NodeNamePrefix)} slurm.use_nodename_as_hostname = $NodeNameIsHostname [[node nodearraybase]] Abstract = true [[[configuration]]] slurm.role = execute slurm.autoscale = true slurm.node_prefix = ${ifThenElse(NodeNamePrefix=="Cluster Prefix", StrJoin("-", ClusterName, ""), NodeNamePrefix)} slurm.use_nodename_as_hostname = $NodeNameIsHostname [[[cluster-init cyclecloud/healthagent:default]]] [[[cluster-init cyclecloud/monitoring:default]]] [[[cluster-init cyclecloud/slurm:execute:4.0.5]]] [[[network-interface eth0]]] AssociatePublicIpAddress = $ExecuteNodesPublic [[nodearray hpc]] Extends = nodearraybase MachineType = $HPCMachineType ImageName = $HPCImageName MaxCount = $MaxHPCExecuteNodeCount Zone = ${ifThenElse(DefineNodesAvailabilityZone, HPCAvailabilityZone, undefined)} Azure.MaxScalesetSize = $HPCMaxScalesetSize AdditionalClusterInitSpecs = $HPCClusterInitSpecs EnableNodeHealthChecks = $EnableNodeHealthChecks [[[configuration]]] slurm.default_partition = true slurm.hpc = true slurm.partition = hpc [[nodearray htc]] Extends = nodearraybase MachineType = $HTCMachineType ImageName = $HTCImageName MaxCount = $MaxHTCExecuteNodeCount Zone = ${ifThenElse(DefineNodesAvailabilityZone, HTCAvailabilityZone, undefined)} Interruptible = $HTCUseLowPrio MaxPrice = $HTCSpotMaxPrice AdditionalClusterInitSpecs = $HTCClusterInitSpecs [[[configuration]]] slurm.hpc = false slurm.partition = htc # set pcpu = false for all hyperthreaded VMs slurm.use_pcpu = false [[nodearray gpu]] Extends = nodearraybase MachineType = $GPUMachineType ImageName = $GPUImageName MaxCount = $MaxGPUExecuteNodeCount Zone = ${ifThenElse(DefineNodesAvailabilityZone, GPUAvailabilityZone, undefined)} Azure.MaxScalesetSize = $HPCMaxScalesetSize EnableNodeHealthChecks = $EnableNodeHealthChecks Interruptible = $GPUUseLowPrio MaxPrice = $GPUSpotMaxPrice AdditionalClusterInitSpecs = $GPUClusterInitSpecs [[[configuration]]] slurm.default_partition = true slurm.hpc = true slurm.partition = gpu #Parameter to enable or disable IMEX service on a per-job basis #IMEX Support is enabled by default for GB200/GB300 but can be disabled by setting param to False #slurm.imex.enabled=True [[nodearray dynamic]] Extends = nodearraybase MachineType = $DynamicMachineType ImageName = $DynamicImageName MaxCoreCount = $MaxDynamicExecuteCoreCount Interruptible = $DynamicUseLowPrio MaxPrice = $DynamicSpotMaxPrice AdditionalClusterInitSpecs = $DynamicClusterInitSpecs [[[configuration]]] slurm.hpc = false # Slurm only allows a single feature to be defined in a Nodeset. If multiple features are defined here, only first value will be used for the nodeset. slurm.dynamic_feature := "dyn" # If this option is used, slurmd is started with this configuration for dynamic nodes. #slurm.dynamic_config := "-Z --conf \"Feature=dyn\"" # set pcpu = false for all hyperthreaded VMs slurm.use_pcpu = false slurm.autoscale = $EnableDynamicPartition [parameters About] Order = 1 [[parameters About Slurm]] [[[parameter slurm]]] HideLabel = true Config.Plugin = pico.widget.HtmlTemplateWidget Config.Template = '''
Follow the instructions in theREADMEfor details on instructions on extending and configuring the Project for your environment. |
Slurm is the most widely used workload manager in HPC, as the scheduler of choice for six of the top ten systems in the TOP500 and with market penetration of more than 70%. Slurm is an advanced, open-source scheduler designed to satisfy the demanding needs of high-performance computing (HPC), high-throughput computing (HTC), and artificial intelligence (AI). |
Commercial Support provided by SchedMD |
Get more from your HPC investment! SchedMD, the company behind Slurm development, can answer your Slurm questions and explain our options for consultation, training, support, and migration. |
Slurm at a glance |
Slurm provides massive scalability and can easily manage performance requirements for small cluster, large cluster, and supercomputer needs. Slurm outperforms competitive schedulers with compute rates at: |
|
Slurm’s plug-in based architecture enables optimization and control in scheduling operations to meet organizational priorities. With first class resource management for GPUs, Slurm allows users to request GPU resources alongside CPUs. This flexibility ensures that jobs are executed quickly and efficiently, while maximizing resource utilization. |
Other Slurm features include: |
|
The directories /sched and /shared are network attached mounts and exist on all nodes of the cluster.
Options for providing these mounts:
[Builtin]: The scheduler node is an NFS server that provides the mountpoint to the other nodes of the cluster (not supported for HA configurations).
[External NFS]: A network attached storage such as Azure Netapp Files, HPC Cache, or another VM running an NFS server provides the mountpoint.
[Azure Managed Lustre]: An Azure Managed Lustre deployment provides the mountpoint.
Note: the cluster must be terminated for changes to filesystem mounts to take effect.
''' Conditions.Hidden := false [[parameters Scheduler Mount]] Order = 20 Label = File-system Mount for /sched [[[parameter About sched]]] HideLabel = true Config.Plugin = pico.widget.HtmlTemplateWidget Config.Template = '''Slurm's configuration is linked in from the /sched directory. It is managed by the scheduler node.
Uncheck the box below to disable the built-in NFS export of the /sched directory and use an external file-system.
Warning: switching an active cluster over to NFS or Lustre from Builtin will delete the shared disk.
" Conditions.Hidden := UseBuiltinSched || configuration_slurm_ha_enabled [[[parameter NFSSchedType]]] Label = FS Type ParameterType = StringList Config.Label = Type of shared filesystem to use for this cluster Config.Plugin = pico.form.Dropdown Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} DefaultValue = nfs Conditions.Hidden := UseBuiltinSched && !configuration_slurm_ha_enabled [[[parameter NFSSchedAddress]]] Label = IP Address Description = The IP address or hostname of the NFS server or Lustre FS. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. Config.ParameterType = String Conditions.Hidden := UseBuiltinSched && !configuration_slurm_ha_enabled [[[parameter NFSSchedExportPath]]] Label = Export Path Description = The path exported by the file system DefaultValue = /sched Conditions.Hidden := UseBuiltinSched && !configuration_slurm_ha_enabled [[[parameter NFSSchedMountOptions]]] Label = Mount Options Description = NFS Client Mount Options Conditions.Hidden := UseBuiltinSched && !configuration_slurm_ha_enabled [[[parameter SchedFilesystemSize]]] Label = Size (GB) Description = The filesystem size (cannot be changed after initial start) DefaultValue = 30 Config.Plugin = pico.form.NumberTextBox Config.MinValue = 10 Config.MaxValue = 10240 Config.IntegerOnly = true Conditions.Excluded := !UseBuiltinSched || configuration_slurm_ha_enabled [[parameters Default NFS Share]] Order = 30 Label = File-system Mount for /shared [[[parameter About shared]]] HideLabel = true Config.Plugin = pico.widget.HtmlTemplateWidget Config.Template = '''Users' home directories reside within the /shared mountpoint with the base homedir /shared/home.
Uncheck the box below to disable the built-in NFS export of the /shared directory and use an external file-system.
Warning: switching an active cluster over to NFS or Lustre from Builtin will delete the shared disk.
" Conditions.Hidden := UseBuiltinShared || configuration_slurm_ha_enabled [[[parameter NFSType]]] Label = FS Type ParameterType = StringList Config.Label = Type of shared filesystem to use for this cluster Config.Plugin = pico.form.Dropdown Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} DefaultValue = nfs Conditions.Hidden := UseBuiltinShared && !configuration_slurm_ha_enabled [[[parameter NFSAddress]]] Label = IP Address Description = The IP address or hostname of the NFS server or Lustre FS. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. Config.ParameterType = String Conditions.Hidden := UseBuiltinShared && !configuration_slurm_ha_enabled [[[parameter NFSSharedExportPath]]] Label = Export Path Description = The path exported by the file system DefaultValue = /shared Conditions.Hidden := UseBuiltinShared && !configuration_slurm_ha_enabled [[[parameter NFSSharedMountOptions]]] Label = Mount Options Description = NFS Client Mount Options Conditions.Hidden := UseBuiltinShared && !configuration_slurm_ha_enabled [[[parameter FilesystemSize]]] Label = Size (GB) Description = The filesystem size (cannot be changed after initial start) DefaultValue = 100 Config.Plugin = pico.form.NumberTextBox Config.MinValue = 10 Config.MaxValue = 10240 Config.IntegerOnly = true Conditions.Excluded := !UseBuiltinShared || configuration_slurm_ha_enabled [[parameters Additional NFS Mount]] Order = 40 Label = Additional Filesystem Mount [[[parameter Additional Shared FS Mount Readme]]] HideLabel = true Config.Plugin = pico.widget.HtmlTemplateWidget Config.Template := "Mount another shared file-system endpoint on the cluster nodes.
" Order = 20 [[[parameter AdditionalNFS]]] HideLabel = true DefaultValue = false Widget.Plugin = pico.form.BooleanCheckBox Widget.Label = Add Shared Filesystem mount [[[parameter AdditionalNFSType]]] Label = FS Type ParameterType = StringList Config.Label = Shared filesystem type of the additional mount Config.Plugin = pico.form.Dropdown Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} DefaultValue = nfs Conditions.Excluded := AdditionalNFS isnt true [[[parameter AdditionalNFSAddress]]] Label = IP Address Description = The IP address or hostname of the additional mount. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. Config.ParameterType = String Conditions.Excluded := AdditionalNFS isnt true [[[parameter AdditionalNFSMountPoint]]] Label = Mount Point Description = The path at which to mount the Filesystem DefaultValue = /data Conditions.Excluded := AdditionalNFS isnt true [[[parameter AdditionalNFSExportPath]]] Label = Export Path Description = The path exported by the file system DefaultValue = /data Conditions.Excluded := AdditionalNFS isnt true [[[parameter AdditionalNFSMountOptions]]] Label = Mount Options Description = Filesystem Client Mount Options Conditions.Excluded := AdditionalNFS isnt true [parameters Advanced Settings] Order = 20 [[parameters Azure Settings]] Order = 10 [[[parameter Credentials]]] Description = The credentials for the cloud provider ParameterType = Cloud.Credentials [[[parameter ManagedIdentity]]] Label = Managed Id Description = Optionally assign an Azure user assigned managed identity to all nodes to access Azure resources using assigned roles. ParameterType = Azure.ManagedIdentity DefaultValue = =undefined [[[parameter BootDiskSize]]] Description = Optional: Size of the OS/boot disk in GB for all nodes in the cluster (leave at 0 to use Image size) ParameterType = Integer Config.Plugin = pico.form.NumberTextBox Config.MinValue = 0 Config.MaxValue = 32,000 Config.IntegerOnly = true Config.Increment = 64 DefaultValue = 0 [[[parameter EnableDynamicPartition]]] Label = Enable Dynamic Partition DefaultValue = true Widget.Plugin = pico.form.BooleanCheckBox Widget.Label = Create a partition for the dynamic nodearray [[[parameter NodeTags]]] Label = VM Tags Description = Tags applied to all nodes ParameterType = Record DefaultValue := [] Config.MultiSelect = false [[parameters Slurm Settings ]] Order = 5 [[[parameter slurm_version_warning]]] HideLabel = true Config.Plugin = pico.widget.HtmlTemplateWidget Config.Template := "| Note: For SLES HPC, we can only install the version supported by SLES HPC's zypper repos. At the time of this release, that is 23.02.7 |
| Note: Checking this box will create persistent databases and tables in SQL DB provided. Deleting this cluster will not automatically delete those databases. User is responsible for periodically purging/archiving their slurm databases to maintain costs. |
Monitoring CycleCloud clusters with Prometheus and Grafana. Follow this README for details on setting up Managed Monitoring Infrastructure. |