# Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This Makefile includes all the variables and targets for running the MLPerf Inference power submission, including
# setting up the power monitors and collecting the power logs.


include $(CURDIR)/Makefile.const

# nsys and nvprof locked clock frequency
GPUCLK?=1000

# Set the power-dev directory
POWER_DEV_DIR = $(BUILD_DIR)/power-dev
POWER_DEV_URL = https://github.com/mlcommons/power-dev.git
POWER_DEV_HASH = f5ee305a867b24905fea1f04f1b68819d392a731
POWER_CLIENT_SCRIPT = $(BUILD_DIR)/power-dev/ptd_client_server/client.py
POWER_NTP_URL = ib-01.dc4-in.nvidia.com

POWER_SERVER_USERNAME = lab
POWER_SERVER_IP ?= $(shell $(PYTHON3_CMD) -m power.power_server_ips $(HOST_HOSTNAME_SHORT) --use_win=$(USE_WIN_PTD))
POWER_SERVER_USER_IP = $(POWER_SERVER_USERNAME)@$(POWER_SERVER_IP)

# Credentials for windows is different
ifeq ($(USE_WIN_PTD), 1)
    POWER_SERVER_PASSWORD = labuser
    POWER_SERVER_SCRIPT_DIR = C:/power-dev/ptd_client_server
    POWER_SERVER_LOG_DIR = C:/ptd-logs
    POWER_SERVER_CONFIG = power/server-$(HOST_HOSTNAME_SHORT)-win.cfg
    POWER_SERVER_PORT = 4950
    POWER_PTD_PORT = 8888
else
    POWER_SERVER_PASSWORD = labuser-mlpinf
    POWER_SERVER_SCRIPT_DIR = /home/lab/mlperf-power/power-dev/ptd_client_server
    POWER_SERVER_CONFIG_DIR = /home/lab/mlperf-power/power_server_configs
    POWER_SERVER_LOG_DIR = /home/lab/mlperf-power/power_logs/ptd-logs
    POWER_SERVER_CONFIG = power/server-$(HOST_HOSTNAME_SHORT).cfg
    POWER_SERVER_PORT = $(shell grep -oP '^listen:.*\b(\d+)\b' power/server-$(HOST_HOSTNAME_SHORT).cfg | awk '{print $$NF}')
    POWER_PTD_PORT = $(shell grep -oP '^networkPort: \K\d+' power/server-$(HOST_HOSTNAME_SHORT).cfg)
endif

POWER_SSH = sshpass -p $(POWER_SERVER_PASSWORD) ssh -o "StrictHostKeyChecking no"
POWER_SCP = sshpass -p $(POWER_SERVER_PASSWORD) scp -o "StrictHostKeyChecking no"

# Set log directories for power logs
POWER_LOGS_TEMP_DIR = $(BUILD_DIR)/power_logs_temp


# Clone power-dev repo.
.PHONY: clone_power_dev
clone_power_dev:
	@if [ ! -d $(POWER_DEV_DIR) ]; then \
		echo "Cloning Official Power-Dev repo" \
			&& git clone $(POWER_DEV_URL) $(POWER_DEV_DIR); \
	fi
	@echo "Updating Power-Dev repo" \
		&& cd $(POWER_DEV_DIR) \
		&& git fetch \
		&& git checkout $(POWER_DEV_HASH)


.PHONY: lock_clocks
lock_clocks:
ifneq ($(IS_SOC), 1)
	@echo "Locking clocks to $(GPUCLK)"
	@sudo nvidia-smi -lgc $(GPUCLK),$(GPUCLK)
else
endif


.PHONY: free_clocks
free_clocks:
ifneq ($(IS_SOC), 1)
	@echo "Freeing clocks"
	@sudo nvidia-smi -rgc
else
endif

# Launch power server on the server system
# NOTE: Only 1 process is allowed to run through windows power server.
# Concurrent runs only allowed on Linux power servers. The server and PTD will listen on
# unique ports.
.PHONY: power_prologue
power_prologue:
	@mkdir -p $(POWER_LOGS_TEMP_DIR)
	@mkdir -p $(POWER_LOGS_DIR)
	@rm -rf $(POWER_LOGS_TEMP_DIR)
	@echo "Checking if someone else is using the Power Server machine..."
	@$(MAKE) -f Makefile.power power_check_server_not_running
	@echo "Copying Server config to the Power Server..."
	$(POWER_SCP) $(POWER_SERVER_CONFIG) $(POWER_SERVER_USER_IP):$(POWER_SERVER_CONFIG_DIR)/server-$(HOST_HOSTNAME_SHORT).cfg
	@echo "Running Server script on the Power Server..."
ifeq ($(USE_WIN_PTD), 1)
	$(POWER_SSH) $(POWER_SERVER_USER_IP) \
		powershell -Command "Start-Process C:/Program\` Files/Python37/python -PassThru -Wait \
		-ArgumentList '$(POWER_SERVER_SCRIPT_DIR)/server.py -c $(POWER_SERVER_CONFIG_DIR)/server-$(HOST_HOSTNAME_SHORT).cfg' \
		-WorkingDirectory $(POWER_SERVER_SCRIPT_DIR) -WindowStyle Hidden \
		-RedirectStandardOutput $(POWER_SERVER_LOG_DIR)/server_stdout_$(HOST_HOSTNAME_SHORT).log \
		-RedirectStandardError $(POWER_SERVER_LOG_DIR)/server_stderr_$(HOST_HOSTNAME_SHORT).log" &
else
	$(POWER_SSH) $(POWER_SERVER_USER_IP) nohup \
		'python3 $(POWER_SERVER_SCRIPT_DIR)/server.py -c $(POWER_SERVER_CONFIG_DIR)/server-$(HOST_HOSTNAME_SHORT).cfg \
		1>$(POWER_SERVER_LOG_DIR)/server_stdout_$(HOST_HOSTNAME_SHORT).log 2>$(POWER_SERVER_LOG_DIR)/server_stderr_$(HOST_HOSTNAME_SHORT).log &\
		echo "$$!" > $(POWER_SERVER_CONFIG_DIR)/server-$(HOST_HOSTNAME_SHORT).pid'
endif
	@echo "Checking if Power Server script was successfully started..."
	@for i in {1..3}; do $(MAKE) -f Makefile.power power_check_server_running && break; done
	@echo "Power Server is running now. Sleeping for 40 secs to wait for ptd self testing."
	@sleep 40
	@echo "Done sleeping. Power server should be listening to client now"


.PHONY: power_check_server_running
power_check_server_running:
	@echo "Checking if Power Server is currently running..."
ifeq ($(USE_WIN_PTD), 1)
	@$(POWER_SSH) $(POWER_SERVER_USER_IP) \
		powershell -Command "Get-Process python -ErrorAction SilentlyContinue"
else
	@echo "Checking if Power Server for $(HOST_HOSTNAME_SHORT) is currently running..."
	@$(POWER_SSH) $(POWER_SERVER_USER_IP) 'ps aux | grep $$(cat $(POWER_SERVER_CONFIG_DIR)/server-$(HOST_HOSTNAME_SHORT).pid) | grep server.py'
	@echo "Power Server for $(HOST_HOSTNAME_SHORT) is indeed running."
endif


.PHONY: power_check_server_not_running
power_check_server_not_running:
	@echo "Checking if Power Server is NOT currently running..."
	@echo "You can run \"POWER_SERVER_IP=$(POWER_SERVER_IP) make power_kill_server\" to kill the job running on the Power Server"
ifeq ($(USE_WIN_PTD), 1)
	@! $(POWER_SSH) $(POWER_SERVER_USER_IP) \
		powershell -Command "Get-Process python -ErrorAction SilentlyContinue"
else
	@echo "Checking if Power Server for $(HOST_HOSTNAME_SHORT) is currently NOT running..."
	@! $(POWER_SSH) $(POWER_SERVER_USER_IP) 'ps aux | grep $$(cat $(POWER_SERVER_CONFIG_DIR)/server-$(HOST_HOSTNAME_SHORT).pid) | grep server.py'
	@echo "Power Server for $(HOST_HOSTNAME_SHORT) is indeed NOT running."
endif


.PHONY: power_kill_server
power_kill_server:
	@echo "Killing server.py on Power Server..."
ifeq ($(USE_WIN_PTD), 1)
	@$(POWER_SSH) $(POWER_SERVER_USER_IP) \
		powershell -Command "taskkill /IM python.exe /F ; taskkill /IM ptd-windows-x86.exe /F" || true
else
	@echo "Killing Power Server and PTD for $(HOST_HOSTNAME_SHORT)"
	$(POWER_SSH) $(POWER_SERVER_USER_IP) 'kill $$(cat $(POWER_SERVER_CONFIG_DIR)/server-$(HOST_HOSTNAME_SHORT).pid)' || true
	$(POWER_SSH) $(POWER_SERVER_USER_IP) 'kill $$(ps aux | grep "ptd-linux-x86" | grep "$(POWER_PTD_PORT)" | grep -v grep | awk "{print $$2}")' || true
endif


.PHONY: power_epilogue
power_epilogue:
	@echo "Clean up unnecessary files..."
	@mv $(POWER_LOGS_DIR)/20*/* $(POWER_LOGS_DIR)/
	@rm -rf $(POWER_LOGS_DIR)/20* $(POWER_LOGS_DIR)/*/mlperf_log*
	@mv $(POWER_LOGS_DIR)/ranging_tmp/* $(POWER_LOGS_DIR)/ranging/
	@mv $(POWER_LOGS_DIR)/testing_tmp/* $(POWER_LOGS_DIR)/run_1/
	# Both our framework and submission checker expects spl.txt to be in the same directory as mlperf logs
	# The mlperf logs will always be 3 subdirectories deep within run_1 (system_name/benchmark/scenario/mlperf_*)
	@find $(POWER_LOGS_DIR)/ranging -maxdepth 3 -mindepth 3 -type d | xargs -I {} cp ${POWER_LOGS_DIR}/ranging/spl.txt {}
	@find $(POWER_LOGS_DIR)/run_1 -maxdepth 3 -mindepth 3 -type d | xargs -I {} cp ${POWER_LOGS_DIR}/run_1/spl.txt {}
	@rm -rf $(POWER_LOGS_DIR)/ranging_tmp $(POWER_LOGS_DIR)/testing_tmp $(POWER_LOGS_TEMP_DIR)
	@set -o pipefail && LOG_DIR=$(POWER_LOGS_DIR)/run_1 $(PYTHON3_CMD) -m scripts.print_harness_result $(RUN_ARGS) 2>&1 | tee -a $(POWER_LOGS_DIR)/stdout.txt
	@echo "Power logs are located in $(POWER_LOGS_DIR)."


.PHONY: power_warn_exclusive_access
power_warn_exclusive_access:
	@echo "Changing MaxQ/MaxP state on a multi-GPU node will FAIL unless you have full access to the system. For SLURM, you need to have grabbed all the GPUs on this node."


.PHONY: power_warn_reboot_machine
power_warn_reboot_machine:
	@echo "It is recommended to reboot the machine after resetting it from MaxQ state to MaxP state!"


.PHONY: bios_warn_reboot_machine
bios_warn_reboot_machine:
	@echo "Please reboot the machine for BIOS changes to take place."


.PHONY: power_set_maxq_state
power_set_maxq_state:
ifeq ($(HOSTNAME_SHORT), ro-dvt-053-80gb)
	@$(MAKE) -f Makefile.power power_warn_exclusive_access
	@echo "Setting MaxQ power state on NVIDIA DGX Station A100 machine"
	NVSMI_RESET_DISABLE=0 $(SHELL) power/set_nvlink.sh 0
	sudo nvidia-smi -pl 250
else ifeq ($(HOSTNAME_SHORT), altra-g242-p31-01)
	@$(MAKE) -f Makefile.power power_warn_exclusive_access
	@echo "Setting MaxQ power state on A100-PCIe_aarch64x4 machine"
	source power/set_fan_speed.sh && MACHINE_TYPE=GIGABYTE set_fan_auto
	NVSMI_RESET_DISABLE=0 $(SHELL) power/set_nvlink.sh 0
	sudo nvidia-smi -pl 200
else ifeq ($(HOSTNAME_SHORT), ipp1-1469)
	@$(MAKE) -f Makefile.power power_warn_exclusive_access
	@echo "Setting MaxQ power state on A100-PCIex8 machine"
	source power/set_fan_speed.sh && MACHINE_TYPE=GIGABYTE set_fan_auto
	NVSMI_RESET_DISABLE=0 $(SHELL) power/set_nvlink.sh 0
	sudo nvidia-smi -pl 200
else ifeq ($(HOSTNAME_SHORT), ipp1-1470)
	@$(MAKE) -f Makefile.power power_warn_exclusive_access
	@echo "Setting MaxQ power state on H100-PCIex8 machine"
	source power/set_fan_speed.sh && MACHINE_TYPE=GIGABYTE set_fan_auto
	sudo nvidia-smi -pl 300 # TODO: Find the right power limit for this one.
else ifeq ($(HOSTNAME_SHORT), viking-cr-194)
	@$(MAKE) -f Makefile.power power_warn_exclusive_access
	@echo "Setting MaxQ power state on H100-SXMx8 machine"
	#sudo cpupower --cpu all frequency-set --governor powersave
else ifeq ($(HOSTNAME_SHORT), sjc1-luna-02)
	@$(MAKE) -f Makefile.power power_warn_exclusive_access
	@echo "Setting MaxQ power state on NVIDIA DGX A100 machine"
	source power/set_fan_speed.sh && MACHINE_TYPE=LUNA set_fan_manual_speed 30 && sleep 10
	$(SHELL) power/check_switch.sh && sleep 10
	$(SHELL) power/set_nvswitch.sh 0 && sleep 10
	NVSMI_RESET_DISABLE=0 $(SHELL) power/set_nvlink.sh 0 && sleep 10
	sudo nvidia-smi -pm 1
	sudo nvidia-smi -pl 275
else ifeq ($(HOSTNAME_SHORT), computelab-310)
	@echo "Setting MaxQ power state on NVIDIA AGX Xavier"
	sudo /usr/sbin/nvpmodel -f power/nvpmodel-computelab-310.conf -m 8
	sudo /usr/sbin/nvpmodel -d cool
	#These commands require password. Please manually run them.
	#sudo bash -c 'echo "14100000.pcie" > /sys/bus/platform/drivers/tegra-pcie-dw/unbind'
else ifeq ($(HOSTNAME_SHORT), computelab-501)
	@echo "Setting MaxQ power state on NVIDIA Xavier NX"
	sudo /usr/sbin/nvpmodel -f power/nvpmodel-computelab-501.conf -m 9
	sudo /usr/sbin/nvpmodel -d cool
	#These commands require password. Please manually run them.
	#sudo bash -c "echo 5000 > /sys/devices/c250000.i2c/i2c-7/7-0040/iio:device0/crit_current_limit_0"
	#sudo bash -c "echo 5000 > /sys/devices/c250000.i2c/i2c-7/7-0040/iio:device0/warn_current_limit_0"
else ifeq ($(HOSTNAME_SHORT), computelab-502)
	@echo "Setting MaxQ power state on NVIDIA Xavier NX"
	sudo /usr/sbin/nvpmodel -f power/nvpmodel-computelab-502.conf -m 9
	sudo /usr/sbin/nvpmodel -d cool
	#These commands require password. Please manually run them.
	#sudo bash -c "echo 5000 > /sys/devices/c250000.i2c/i2c-7/7-0040/iio:device0/crit_current_limit_0"
	#sudo bash -c "echo 5000 > /sys/devices/c250000.i2c/i2c-7/7-0040/iio:device0/warn_current_limit_0"
else ifeq ($(HOSTNAME_SHORT), orin-agx-01)
	@echo "NVIDIA Orin MaxQ state is set before each benchmark run. Do nothing here."
else ifeq ($(HOSTNAME_SHORT), orin-agx-02)
	@echo "NVIDIA Orin MaxQ state is set before each benchmark run. Do nothing here."
else ifeq ($(HOSTNAME_SHORT), orin-agx-03)
	@echo "NVIDIA Orin MaxQ state is set before each benchmark run. Do nothing here."
else ifeq ($(HOSTNAME_SHORT), orin-agx-04)
	@echo "NVIDIA Orin MaxQ state is set before each benchmark run. Do nothing here."
else ifeq ($(HOSTNAME_SHORT), a1u2n2g-0275-01-jetson)
	@echo "NVIDIA Orin MaxQ state is set before each benchmark run. Resetting the cpusets."
	@echo "0-11" | sudo -n tee  /sys/fs/cgroup/cpuset/docker/cpuset.cpus || true
	@sudo /usr/sbin/nvpmodel -f /etc/nvpmodel.conf  -m 0
else ifeq ($(HOSTNAME_SHORT), a1u2n2g-0277-02-jetson)
	@echo "NVIDIA Orin MaxQ state is set before each benchmark run. Resetting the cpusets."
	@echo "0-11" | sudo -n tee  /sys/fs/cgroup/cpuset/docker/cpuset.cpus || true
	@sudo /usr/sbin/nvpmodel -f /etc/nvpmodel.conf  -m 0
else ifeq ($(HOSTNAME_SHORT), a1u2n2g-0277-01-jetson)
	@echo "NVIDIA Orin NX MaxQ state is set before each benchmark run. Resetting the cpusets."
	@echo "0-7" | sudo -n tee  /sys/fs/cgroup/cpuset/docker/cpuset.cpus || true
	@sudo /usr/sbin/nvpmodel -f /etc/nvpmodel.conf  -m 0
else
	@echo "$(HOSTNAME_SHORT) does not have MaxQ submission."
endif


.PHONY: power_set_max_fan_speed_luna
power_set_max_fan_speed_luna:
	@echo "Setting fan speed to MAX on NVIDIA DGX A100 machine"
	source power/set_fan_speed.sh && MACHINE_TYPE=LUNA set_fan_manual_speed 100 && sleep 10


.PHONY: power_set_max_fan_speed_gigabyte
power_set_max_fan_speed_gigabyte:
	@echo "Setting fan speed to MAX on Gigabyte Z54 machine"
	source power/set_fan_speed.sh && MACHINE_TYPE=GIGABYTE set_fan_manual_speed 0xFF


.PHONY: power_set_max_fan_speed
power_set_max_fan_speed:
	@$(MAKE) -f Makefile.power power_warn_exclusive_access
ifeq ($(HOSTNAME_SHORT), ipp1-1468)
	@$(MAKE) -f Makefile.power power_set_max_fan_speed_gigabyte
else ifeq ($(HOSTNAME_SHORT), ipp1-1469)
	@$(MAKE) -f Makefile.power power_set_max_fan_speed_gigabyte
else ifeq ($(HOSTNAME_SHORT), ipp1-1470)
	@$(MAKE) -f Makefile.power power_set_max_fan_speed_gigabyte
else ifeq ($(HOSTNAME_SHORT), altra-g242-p31-01)
	@$(MAKE) -f Makefile.power power_set_max_fan_speed_gigabyte
else ifeq ($(HOSTNAME_SHORT), cl1-2591)
	@$(MAKE) -f Makefile.power power_set_max_fan_speed_gigabyte
else ifeq ($(HOSTNAME_SHORT), sjc1-luna-01)
	@$(MAKE) -f Makefile.power power_set_max_fan_speed_luna
else ifeq ($(HOSTNAME_SHORT), sjc1-luna-02)
	@$(MAKE) -f Makefile.power power_set_max_fan_speed_luna
else ifeq ($(HOSTNAME_SHORT), luna-prod-48)
	@$(MAKE) -f Makefile.power power_set_max_fan_speed_luna
else ifeq ($(HOSTNAME_SHORT), orin-agx-01)
	@sudo jetson_clocks --fan
else ifeq ($(HOSTNAME_SHORT), orin-agx-02)
	@sudo jetson_clocks --fan
else ifeq ($(HOSTNAME_SHORT), orin-agx-03)
	@sudo jetson_clocks --fan
else
	@echo "Setting fan speed is not supported on this machine"
endif


.PHONY: power_reset_maxq_state
power_reset_maxq_state:
ifeq ($(HOSTNAME_SHORT), computelab-ro-prod-01)
	@$(MAKE) -f Makefile.power power_warn_exclusive_access
	@echo "Resetting power state on NVIDIA DGX Station A100 machine"
	NVSMI_RESET_DISABLE=0 $(SHELL) power/set_nvlink.sh 1
	sudo nvidia-smi -pl 275
	@$(MAKE) -f Makefile.power power_warn_reboot_machine
else ifeq ($(HOSTNAME_SHORT), ro-dvt-060-80gb)
	@$(MAKE) -f Makefile.power power_warn_exclusive_access
	@echo "Resetting power state on NVIDIA DGX Station A100 machine"
	NVSMI_RESET_DISABLE=0 $(SHELL) power/set_nvlink.sh 1
	sudo nvidia-smi -pl 275
	@$(MAKE) -f Makefile.power power_warn_reboot_machine
else ifeq ($(HOSTNAME_SHORT), ro-dvt-053-80gb)
	@$(MAKE) -f Makefile.power power_warn_exclusive_access
	@echo "Resetting power state on NVIDIA DGX Station A100 machine"
	NVSMI_RESET_DISABLE=0 $(SHELL) power/set_nvlink.sh 1
	sudo nvidia-smi -pl 275
	@$(MAKE) -f Makefile.power power_warn_reboot_machine
else ifeq ($(HOSTNAME_SHORT), altra-g242-p31-01)
	@$(MAKE) -f Makefile.power power_warn_exclusive_access
	@echo "Resetting power state on A100-PCIe_aarch64x4 machine"
	NVSMI_RESET_DISABLE=0 $(SHELL) power/set_nvlink.sh 1
	@$(MAKE) -f Makefile.power power_set_max_fan_speed
	sudo nvidia-smi -pl 300
	@$(MAKE) -f Makefile.power power_warn_reboot_machine
else ifeq ($(HOSTNAME_SHORT), ipp1-1469)
	@$(MAKE) -f Makefile.power power_warn_exclusive_access
	@echo "Resetting power state on A100-PCIex8 machine"
	NVSMI_RESET_DISABLE=0 $(SHELL) power/set_nvlink.sh 1
	@$(MAKE) -f Makefile.power power_set_max_fan_speed
	sudo nvidia-smi -pl 300
	@$(MAKE) -f Makefile.power power_warn_reboot_machine
else ifeq ($(HOSTNAME_SHORT), ipp1-1470)
	@$(MAKE) -f Makefile.power power_warn_exclusive_access
	@echo "Resetting power state on H100-PCIex8 machine"
	NVSMI_RESET_DISABLE=0 $(SHELL) power/set_nvlink.sh 1
	@$(MAKE) -f Makefile.power power_set_max_fan_speed
	sudo nvidia-smi -pl 350
	@$(MAKE) -f Makefile.power power_warn_reboot_machine
else ifeq ($(HOSTNAME_SHORT), sjc1-luna-02)
	@$(MAKE) -f Makefile.power power_warn_exclusive_access
	@echo "Resetting power state on NVIDIA DGX A100 machine"
	@$(MAKE) -f Makefile.power power_set_max_fan_speed
	NVSMI_RESET_DISABLE=0 $(SHELL) power/set_nvlink.sh 1 && sleep 10
	$(SHELL) power/set_nvswitch.sh 1 && sleep 10
	sudo nvidia-smi -pm 1
	sudo nvidia-smi -pl 400
	# sudo systemctl start nvidia-fabricmanager
	# sudo systemctl start dcgm
	# sudo systemctl start nvsm
	# sudo systemctl start nvidia-persistenced.service
	@$(MAKE) -f Makefile.power power_warn_reboot_machine
else ifeq ($(HOSTNAME_SHORT), computelab-310)
	@echo "Resetting power state on NVIDIA AGX Xavier"
	@sudo /usr/sbin/nvpmodel -f power/nvpmodel-computelab-310.conf -m 0
	@sudo /usr/sbin/nvpmodel -d cool
else ifeq ($(HOSTNAME_SHORT), computelab-501)
	@echo "Resetting power state on NVIDIA Xavier NX"
	@sudo /usr/sbin/nvpmodel -f power/nvpmodel-computelab-501.conf -m 6
	@sudo /usr/sbin/nvpmodel -d cool
	#These commands require password. Please manually run them.
	#sudo bash -c "echo 5000 > /sys/devices/c250000.i2c/i2c-7/7-0040/iio:device0/crit_current_limit_0"
	#sudo bash -c "echo 5000 > /sys/devices/c250000.i2c/i2c-7/7-0040/iio:device0/warn_current_limit_0"
else ifeq ($(HOSTNAME_SHORT), computelab-502)
	@echo "Resetting power state on NVIDIA Xavier NX"
	@sudo /usr/sbin/nvpmodel -f power/nvpmodel-computelab-502.conf -m 6
	@sudo /usr/sbin/nvpmodel -d cool
	#These commands require password. Please manually run them.
	#sudo bash -c "echo 5000 > /sys/devices/c250000.i2c/i2c-7/7-0040/iio:device0/crit_current_limit_0"
	#sudo bash -c "echo 5000 > /sys/devices/c250000.i2c/i2c-7/7-0040/iio:device0/warn_current_limit_0"
else ifeq ($(HOSTNAME_SHORT), ipp1-2468-jetson)
	@echo "ipp1-2468-jetson does not support MaxP run"
else ifeq ($(HOSTNAME_SHORT), ipp1-2469-jetson)
	@echo "ipp1-2469-jetson does not support MaxP run"
else
	@echo "$(HOSTNAME_SHORT) does not have MaxQ submission."
endif


BIOS_SETTING_HOSTS := computelab-ro-prod-01 ipp1-1469 ipp1-1470 sjc1-luna-02 ro-dvt-060-80gb ro-dvt-053-80gb
.PHONY: power_set_maxq_bios
power_set_maxq_bios: bios_warn_reboot_machine
ifeq ($(HOSTNAME_SHORT), $(filter $(HOSTNAME_SHORT), $(BIOS_SETTING_HOSTS)))
	@echo "Setting BIOS state on $(HOSTNAME_SHORT)"
	source power/bios_setting.sh && set_4_ccd
else
	@echo "Setting MaxQ BIOS state is not supported on this machine."
endif


.PHONY: power_reset_maxq_bios
power_reset_maxq_bios: bios_warn_reboot_machine
ifeq ($(HOSTNAME_SHORT), $(filter $(HOSTNAME_SHORT), $(BIOS_SETTING_HOSTS)))
	@echo "Resetting BIOS state on $(HOSTNAME_SHORT)"
	source power/bios_setting.sh && set_all_ccd
else
	@echo "Resetting MaxQ BIOS state is not supported on this machine."
endif

# Make target to copy PTD and power_dev to Windows power server.
# Not needed if running Linux power director.
.PHONY: power_server_setup_win
power_server_setup_win:
ifeq ($(USE_WIN_PTD), 1)
	scp -o "StrictHostKeyChecking no" power/internal/add_user.ps1 $(UNAME)@$(POWER_SERVER_IP):C:/add_user.ps1
	ssh -o "StrictHostKeyChecking no" $(UNAME)@$(POWER_SERVER_IP) powershell -file C:/add_user.ps1
	$(POWER_SCP) -r $(MLPERF_SCRATCH_PATH)/mlperf_power/PTD $(POWER_SERVER_USER_IP):C:/PTD
	$(POWER_SCP) -r $(MLPERF_SCRATCH_PATH)/mlperf_power/power-dev $(POWER_SERVER_USER_IP):C:/power-dev
	$(POWER_SCP) $(MLPERF_SCRATCH_PATH)/mlperf_power/python-3.7.9-amd64.exe $(POWER_SERVER_USER_IP):C:/python-3.7.9-amd64.exe
	$(POWER_SCP) power/internal/install_power_dependencies.ps1 $(POWER_SERVER_USER_IP):C:/install_power_dependencies.ps1
	$(POWER_SSH) $(POWER_SERVER_USER_IP) powershell -file C:/install_power_dependencies.ps1
else
	@echo "Please run with USE_WIN_PTD=1."
endif

.PHONY: power_server_update_power_dir_win
power_server_update_power_dir_win:
ifeq ($(USE_WIN_PTD), 1)
	$(POWER_SSH) $(POWER_SERVER_USER_IP) 'rd /s /q "C:/power-dev"'
	$(POWER_SSH) $(POWER_SERVER_USER_IP) 'rd /s /q "C:/PTD"'
	$(POWER_SCP) -r $(MLPERF_SCRATCH_PATH)/mlperf_power/power-dev $(POWER_SERVER_USER_IP):C:/power-dev
	$(POWER_SCP) -r $(MLPERF_SCRATCH_PATH)/mlperf_power/PTD $(POWER_SERVER_USER_IP):C:/PTD
else
	@echo "Please run with USE_WIN_PTD=1."
endif