# Copyright (c) Facebook, Inc. and its affiliates. # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. # # Based on the 'util/collect_env.py' script from PyTorch. # # # From PyTorch: # # Copyright (c) 2016- Facebook, Inc (Adam Paszke) # Copyright (c) 2014- Facebook, Inc (Soumith Chintala) # Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) # Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) # Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) # Copyright (c) 2011-2013 NYU (Clement Farabet) # Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) # Copyright (c) 2006 Idiap Research Institute (Samy Bengio) # Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) # # From Caffe2: # # Copyright (c) 2016-present, Facebook Inc. All rights reserved. # # All contributions by Facebook: # Copyright (c) 2016 Facebook Inc. # # All contributions by Google: # Copyright (c) 2015 Google Inc. # All rights reserved. # # All contributions by Yangqing Jia: # Copyright (c) 2015 Yangqing Jia # All rights reserved. # # All contributions by Kakao Brain: # Copyright 2019-2020 Kakao Brain # # All contributions by Cruise LLC: # Copyright (c) 2022 Cruise LLC. # All rights reserved. # # All contributions from Caffe: # Copyright(c) 2013, 2014, 2015, the respective contributors # All rights reserved. # # All other contributions: # Copyright(c) 2015, 2016 the respective contributors # All rights reserved. # # Caffe2 uses a copyright model similar to Caffe: each contributor holds # copyright over their contributions to Caffe2. The project versioning records # all such contribution and copyright details. If a contributor wants to further # mark their specific copyright on a particular contribution, they should # indicate their copyright solely in the commit message of the change when it is # committed. # # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America # and IDIAP Research Institute nor the names of its contributors may be # used to endorse or promote products derived from this software without # specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # from __future__ import print_function # Unlike the rest of the PyTorch this file must be python2 compliant. # This script outputs relevant system environment info # Run it with `python collect_env.py`. import locale import os import re import subprocess import sys from collections import namedtuple try: import compiler_gym COMPILER_GYM_AVAILABLE = True except (ImportError, NameError, AttributeError, OSError): COMPILER_GYM_AVAILABLE = False # System Environment Information SystemEnv = namedtuple( "SystemEnv", [ "compiler_gym_version", "is_debug_build", "gcc_version", "clang_version", "cmake_version", "os", "libc_version", "python_version", "python_platform", "pip_version", # 'pip' or 'pip3' "pip_packages", "conda_packages", ], ) def run(command): """Returns (return-code, stdout, stderr)""" p = subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True ) raw_output, raw_err = p.communicate() rc = p.returncode if get_platform() == "win32": enc = "oem" else: enc = locale.getpreferredencoding() output = raw_output.decode(enc) err = raw_err.decode(enc) return rc, output.strip(), err.strip() def run_and_read_all(run_lambda, command): """Runs command using run_lambda; reads and returns entire output if rc is 0""" rc, out, _ = run_lambda(command) if rc != 0: return None return out def run_and_parse_first_match(run_lambda, command, regex): """Runs command using run_lambda, returns the first regex match if it exists""" rc, out, _ = run_lambda(command) if rc != 0: return None match = re.search(regex, out) if match is None: return None return match.group(1) def run_and_return_first_line(run_lambda, command): """Runs command using run_lambda and returns first line if output is not empty""" rc, out, _ = run_lambda(command) if rc != 0: return None return out.split("\n")[0] def get_conda_packages(run_lambda): conda = os.environ.get("CONDA_EXE", "conda") out = run_and_read_all(run_lambda, conda + " list") if out is None: return out # Comment starting at beginning of line comment_regex = re.compile(r"^#.*\n") return re.sub(comment_regex, "", out) def get_gcc_version(run_lambda): return run_and_parse_first_match(run_lambda, "gcc --version", r"gcc (.*)") def get_clang_version(run_lambda): return run_and_parse_first_match( run_lambda, "clang --version", r"clang version (.*)" ) def get_cmake_version(run_lambda): return run_and_parse_first_match(run_lambda, "cmake --version", r"cmake (.*)") def get_platform(): if sys.platform.startswith("linux"): return "linux" elif sys.platform.startswith("win32"): return "win32" elif sys.platform.startswith("cygwin"): return "cygwin" elif sys.platform.startswith("darwin"): return "darwin" else: return sys.platform def get_mac_version(run_lambda): return run_and_parse_first_match(run_lambda, "sw_vers -productVersion", r"(.*)") def get_windows_version(run_lambda): system_root = os.environ.get("SYSTEMROOT", "C:\\Windows") wmic_cmd = os.path.join(system_root, "System32", "Wbem", "wmic") findstr_cmd = os.path.join(system_root, "System32", "findstr") return run_and_read_all( run_lambda, "{} os get Caption | {} /v Caption".format(wmic_cmd, findstr_cmd) ) def get_lsb_version(run_lambda): return run_and_parse_first_match( run_lambda, "lsb_release -a", r"Description:\t(.*)" ) def check_release_file(run_lambda): return run_and_parse_first_match( run_lambda, "cat /etc/*-release", r'PRETTY_NAME="(.*)"' ) def get_os(run_lambda): from platform import machine platform = get_platform() if platform == "win32" or platform == "cygwin": return get_windows_version(run_lambda) if platform == "darwin": version = get_mac_version(run_lambda) if version is None: return None return "macOS {} ({})".format(version, machine()) if platform == "linux": # Ubuntu/Debian based desc = get_lsb_version(run_lambda) if desc is not None: return "{} ({})".format(desc, machine()) # Try reading /etc/*-release desc = check_release_file(run_lambda) if desc is not None: return "{} ({})".format(desc, machine()) return "{} ({})".format(platform, machine()) # Unknown platform return platform def get_python_platform(): import platform return platform.platform() def get_libc_version(): import platform if get_platform() != "linux": return "N/A" return "-".join(platform.libc_ver()) def indent(s): return " " + "\n ".join(s.split("\n")) def get_pip_packages(run_lambda): """Returns `pip list` output. Note: will also find conda-installed pytorch and numpy packages.""" # People generally have `pip` as `pip` or `pip3` # But here it is incoved as `python -mpip` def run_with_pip(pip): return run_and_read_all(run_lambda, pip + " list --format=freeze") pip_version = "pip3" if sys.version[0] == "3" else "pip" out = run_with_pip(sys.executable + " -mpip") return pip_version, out def get_cachingallocator_config(): ca_config = os.environ.get("PYTORCH_CUDA_ALLOC_CONF", "") return ca_config def get_env_info(): run_lambda = run pip_version, pip_list_output = get_pip_packages(run_lambda) if COMPILER_GYM_AVAILABLE: version_str = compiler_gym.__version__ # NOTE(cummins): CompilerGym does not yet have a debug string. debug_mode_str = "N/A" else: version_str = debug_mode_str = "N/A" sys_version = sys.version.replace("\n", " ") return SystemEnv( compiler_gym_version=version_str, is_debug_build=debug_mode_str, python_version="{} ({}-bit runtime)".format( sys_version, sys.maxsize.bit_length() + 1 ), python_platform=get_python_platform(), pip_version=pip_version, pip_packages=pip_list_output, conda_packages=get_conda_packages(run_lambda), os=get_os(run_lambda), libc_version=get_libc_version(), gcc_version=get_gcc_version(run_lambda), clang_version=get_clang_version(run_lambda), cmake_version=get_cmake_version(run_lambda), ) env_info_fmt = """ CompilerGym: {compiler_gym_version} Is debug build: {is_debug_build} Python version: {python_version} Python platform: {python_platform} OS: {os} GCC version: {gcc_version} Clang version: {clang_version} CMake version: {cmake_version} Libc version: {libc_version} Versions of all installed libraries: {pip_packages} {conda_packages} """.strip() def pretty_str(envinfo): def replace_nones(dct, replacement="Could not collect"): for key in dct.keys(): if dct[key] is not None: continue dct[key] = replacement return dct def replace_bools(dct, true="Yes", false="No"): for key in dct.keys(): if dct[key] is True: dct[key] = true elif dct[key] is False: dct[key] = false return dct def prepend(text, tag="[prepend]"): lines = text.split("\n") updated_lines = [tag + line for line in lines] return "\n".join(updated_lines) def replace_if_empty(text, replacement="No relevant packages"): if text is not None and len(text) == 0: return replacement return text def maybe_start_on_next_line(string): # If `string` is multiline, prepend a \n to it. if string is not None and len(string.split("\n")) > 1: return "\n{}\n".format(string) return string mutable_dict = envinfo._asdict() # Replace True with Yes, False with No mutable_dict = replace_bools(mutable_dict) # Replace all None objects with 'Could not collect' mutable_dict = replace_nones(mutable_dict) # If either of these are '', replace with 'No relevant packages' mutable_dict["pip_packages"] = replace_if_empty(mutable_dict["pip_packages"]) mutable_dict["conda_packages"] = replace_if_empty(mutable_dict["conda_packages"]) # Tag conda and pip packages with a prefix # If they were previously None, they'll show up as ie '[conda] Could not collect' if mutable_dict["pip_packages"]: mutable_dict["pip_packages"] = prepend( mutable_dict["pip_packages"], " [{}] ".format(envinfo.pip_version) ) if mutable_dict["conda_packages"]: mutable_dict["conda_packages"] = prepend( mutable_dict["conda_packages"], " [conda] " ) return env_info_fmt.format(**mutable_dict) def get_pretty_env_info(): return pretty_str(get_env_info()) def main(): print("Collecting environment information...") print() print(pretty_str(get_env_info())) if __name__ == "__main__": main()