# An unique identifier for the head node and workers of this cluster. cluster_name: vsan-fs # The maximum number of workers nodes to launch in addition to the head # node. max_workers: 2 # The autoscaler will scale up the cluster faster with higher upscaling speed. # E.g., if the task requires adding more nodes then autoscaler will gradually # scale up the cluster in chunks of upscaling_speed*currently_running_nodes. # This number should be > 0. upscaling_speed: 1.0 # This executes all commands on all nodes in the docker container, # and opens all the necessary ports to support the Ray cluster. # Empty string means disabled. docker: image: "rayproject/ray-ml:latest" # image: rayproject/ray:latest # use this one if you don't need ML dependencies, it's faster to pull container_name: "ray_container" # If true, pulls latest version of image. Otherwise, `docker run` will only pull the image # if no cached version is present. pull_before_run: True run_options: # Extra options to pass into "docker run" - --ulimit nofile=65536:65536 - -v /mnt/shared_storage/experiment_results:/mnt/shared_storage/experiment_results # If a node is idle for this many minutes, it will be removed. idle_timeout_minutes: 5 # Cloud-provider specific configuration. provider: type: vsphere # Credentials configured here will take precedence over credentials set in the # environment variables. vsphere_config: # credentials: # user: vc_username # password: vc_password # server: vc_address # The frozen VM related configurations. If "library_item" is unset, then either an existing frozen VM should be # specified by "name" of a resource pool name of Frozen VMs on every ESXi host should be specified by # "resource_pool". If "library_item" is set, then "name" must be set to indicate the name or the name prefix of # the frozen VM, and "resource_pool" can be set to indicate that a set of frozen VMs should be created on each # ESXi host. frozen_vm: # The name of the frozen VM, or the prefix for a set of frozen VMs. Can only be unset when # "frozen_vm.resource_pool" is set and pointing to an existing resource pool of Frozen VMs. name: frozen-vm # The library item of the OVF template of the frozen VM. If set, the frozen VM or a set of frozen VMs will # be deployed from an OVF template specified by library item. library_item: # The resource pool name of the frozen VMs, can point to an existing resource pool of frozen VMs. # Otherwise, "frozen_vm.library_item" must be specified and a set of frozen VMs will be deployed # on each ESXi host. The frozen VMs will be named as "{frozen_vm.name}-{the vm's ip address}" resource_pool: # The vSphere cluster name, only makes sense when "frozen_vm.library_item" is set and # "frozen_vm.resource_pool" is unset. Indicates to deploy a single frozen VM on the vSphere cluster # from OVF template. cluster: # The target vSphere datastore name for storing the vmdk of the frozen VM to be deployed from OVF template. # Will take effect only when "frozen_vm.library_item" is set. If "frozen_vm.resource_pool" is also set, # this datastore must be a shared datastore among the ESXi hosts. datastore: # How Ray will authenticate with newly launched nodes. auth: ssh_user: ray # By default Ray creates a new private keypair, but you can also use your own. # If you do so, make sure to also set "KeyName" in the head and worker node # configurations below. # ssh_private_key: /path/to/your/key.pem # Tell the autoscaler the allowed node types and the resources they provide. # The key is the name of the node type, which is just for debugging purposes. # The node config specifies the launch config and physical instance type. available_node_types: ray.head.default: # The node type's CPU and Memory resources are by default the same as the frozen VM. # You can override the resources here. Adding GPU to the head node is not recommended. # resources: { "CPU": 2, "Memory": 4096} resources: {} node_config: # The resource pool where the head node should live, if unset, will be # the frozen VM's resource pool. resource_pool: # The datastore to store the vmdk of the head node vm, if unset, will be # the frozen VM's datastore. datastore: worker: # The minimum number of nodes of this type to launch. # This number should be >= 0. min_workers: 1 # The node type's CPU and Memory resources are by default the same as the frozen VM. # You can override the resources here. For GPU, currently only Nvidia GPU is supported. If no ESXi host can # fulfill the requirement, the Ray node creation will fail. The number of created nodes may not meet the desired # minimum number. The vSphere node provider will not distinguish the GPU type. It will just count the quantity: # mount the first k random available Nvidia GPU to the VM, if the user set {"GPU": k}. # resources: {"CPU": 2, "Memory": 4096, "GPU": 1} resources: {} node_config: # The resource pool where the worker node should live, if unset, will be # the frozen VM's resource pool. resource_pool: # The datastore to store the vmdk(s) of the worker node vm(s), if unset, will be # the frozen VM's datastore. datastore: # Specify the node type of the head node (as configured above). head_node_type: ray.head.default # Files or directories to copy to the head and worker nodes. The format is a # dictionary from REMOTE_PATH: LOCAL_PATH, e.g. file_mounts: { # "/path1/on/remote/machine": "/path1/on/local/machine", # "/path2/on/remote/machine": "/path2/on/local/machine", } # Files or directories to copy from the head node to the worker nodes. The format is a # list of paths. The same path on the head node will be copied to the worker node. # This behavior is a subset of the file_mounts behavior. In the vast majority of cases # you should just use file_mounts. Only use this if you know what you're doing! cluster_synced_files: [] # Whether changes to directories in file_mounts or cluster_synced_files in the head node # should sync to the worker node continuously file_mounts_sync_continuously: False # Patterns for files to exclude when running rsync up or rsync down rsync_exclude: [] # Pattern files to use for filtering out files when running rsync up or rsync down. The file is searched for # in the source directory and recursively through all subdirectories. For example, if .gitignore is provided # as a value, the behavior will match git's behavior for finding and using .gitignore files. rsync_filter: [] # List of commands that will be run before `setup_commands`. If docker is # enabled, these commands will run outside the container and before docker # is setup. initialization_commands: - sudo mkdir -p /mnt/shared_storage/experiment_results - sudo mount : /mnt/shared_storage/experiment_results # List of shell commands to run to set up nodes. setup_commands: [] # Custom commands that will be run on the head node after common setup. head_setup_commands: - pip install 'git+https://github.com/vmware/vsphere-automation-sdk-python.git' # Custom commands that will be run on worker nodes after common setup. worker_setup_commands: [] # Command to start ray on the head node. You don't need to change this. head_start_ray_commands: - ray stop - ulimit -n 65536; ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml --dashboard-host=0.0.0.0 # Command to start ray on worker nodes. You don't need to change this. worker_start_ray_commands: - ray stop - ulimit -n 65536; ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076