#! /usr/bin/ruby if RUBY_VERSION < "2.5.0" STDERR.puts "autoproj requires Ruby >= 2.5.0" exit 1 elsif ENV["AUTOPROJ_CURRENT_ROOT"] && (ENV["AUTOPROJ_CURRENT_ROOT"] != Dir.pwd) STDERR.puts "it seems that you've already loaded an env.sh script in this console, open a new console and try again" exit 1 end # frozen_string_literal: true require "pathname" require "optparse" require "fileutils" require "yaml" require "English" module Autoproj module Ops # This class contains the functionality necessary to install autoproj in a # clean root # # It can be required standalone (i.e. does not depend on anything else than # ruby and the ruby standard library) class Install class UnexpectedBinstub < RuntimeError; end RUBYLIB_REINIT = <<~RUBY if defined?(Bundler) if Bundler.respond_to?(:with_unbundled_env) Bundler.with_unbundled_env do exec(Hash['RUBYLIB' => nil], $0, *ARGV) end else Bundler.with_clean_env do exec(Hash['RUBYLIB' => nil], $0, *ARGV) end end elsif ENV['RUBYLIB'] exec(Hash['RUBYLIB' => nil], $0, *ARGV) end RUBY WITHOUT_BUNDLER = <<~RUBY if defined?(Bundler) if Bundler.respond_to?(:with_unbundled_env) Bundler.with_unbundled_env do exec($0, *ARGV) end else Bundler.with_clean_env do exec($0, *ARGV) end end end RUBY # The created workspace's root directory attr_reader :root_dir # Content of the Gemfile generated to install autoproj itself attr_accessor :gemfile # The environment that is passed to the bundler installs attr_reader :env # The configuration hash attr_reader :config # A set of options that should be passed to autoproj when calling it # in a subprocess attr_reader :autoproj_options # The Ruby interpreter we use for this install attr_reader :ruby_executable # The URL of the source to be used to get gems attr_accessor :gem_source def initialize(root_dir) @root_dir = root_dir @gem_source = "https://rubygems.org" @gemfile = nil @skip_stage2 = false @autoproj_options = Array.new @env = Hash.new env["RUBYOPT"] = [] env["RUBYLIB"] = [] env["PATH"] = self.class.sanitize_env(ENV["PATH"] || "") env["BUNDLE_GEMFILE"] = [] load_config if config["ruby_executable"] != Gem.ruby raise "this autoproj installation was already bootstrapped using "\ "#{config['ruby_executable']}, but you are currently running "\ "under #{Gem.ruby}. Changing the ruby interpreter in a given "\ "workspace is not supported, you need to do a clean bootstrap" end @ruby_executable = config["ruby_executable"] @local = false @gems_install_path ||= default_gems_install_path @gems_install_path = File.expand_path(@gems_install_path) env["GEM_HOME"] = [gems_gem_home] env["GEM_PATH"] = [gems_gem_home] end def env_for_child(env = self.env) env.inject(Hash.new) do |h, (k, v)| h[k] = (v.join(File::PATH_SEPARATOR) if v && !v.empty?) h end end def apply_env(env) env.each do |k, v| if v ENV[k] = v else ENV.delete(k) end end end def self.sanitize_env(value) value.split(File::PATH_SEPARATOR) .find_all { |p| !in_workspace?(p) } end def self.in_workspace?(base_dir) path = Pathname.new(base_dir) until path.root? if path.join(".autoproj").exist? || path.join("autoproj").exist? return true end path = path.parent end false end # The path to the .autoproj configuration directory # # @return [String] def dot_autoproj File.join(root_dir, ".autoproj") end # The path to the gemfile used to install autoproj # # @return [String] def autoproj_gemfile_path File.join(dot_autoproj, "Gemfile") end # The path to the autoproj configuration file # # @return [String] def autoproj_config_path File.join(dot_autoproj, "config.yml") end # Whether the stage2 install should be called or not def skip_stage2? !!@skip_stage2 end # (see #skip_stage2?) attr_writer :skip_stage2 # Whether we can access the network while installing def local? !!@local end # (see #local?) attr_writer :local # The version and platform-specific suffix # # This is also the suffix used by bundler to install gems def self.gems_path_suffix return @gems_path_suffix if @gem_path_suffix parts = [Gem.ruby_engine] unless RbConfig::CONFIG["ruby_version"].empty? parts << RbConfig::CONFIG["ruby_version"] end @gems_path_suffix = File.join parts end # The path into which the workspace's gems should be installed # # They are installed in a versioned subdirectory of this path, e.g. # {#gem_path_suffix}. # # @return [String] attr_accessor :gems_install_path # The GEM_HOME under which the workspace's gems should be installed # # @return [String] def gems_gem_home File.join(gems_install_path, self.class.gems_path_suffix) end # Sets where the workspace's gems should be installed # # @param [String] path the absolute path that should be given to # bundler. The gems themselves will be installed in the # {.gems_path_suffix} subdirectory under this private def xdg_var(varname, default) if (env = ENV[varname]) && !env.empty? env else default end end # Get autoproj's default path for installing gems def default_gems_install_path xdg_default_gem_path = xdg_var("XDG_DATA_HOME", File.join(Dir.home, ".local", "share", "autoproj", "gems")) default_gem_path = File.join( Dir.home, ".autoproj", "gems" ) if File.directory?(xdg_default_gem_path) xdg_default_gem_path elsif File.directory?(default_gem_path) default_gem_path else xdg_default_gem_path end end # Whether autoproj should prefer OS-independent packages over their # OS-packaged equivalents (e.g. the thor gem vs. the ruby-thor # Debian package) def prefer_indep_over_os_packages? @prefer_indep_over_os_packages end # (see #prefer_index_over_os_packages?) def prefer_indep_over_os_packages=(flag) @prefer_indep_over_os_packages = !!flag end def self.guess_gem_program ruby_bin = RbConfig::CONFIG["RUBY_INSTALL_NAME"] ruby_bindir = RbConfig::CONFIG["bindir"] candidates = ["gem"] candidates.unshift "gem#{$1}" if ruby_bin =~ /^ruby(.+)$/ candidates.each do |gem_name| if File.file?(gem_full_path = File.join(ruby_bindir, gem_name)) return gem_full_path end end raise ArgumentError, "cannot find a gem program "\ "(tried #{candidates.sort.join(', ')} in #{ruby_bindir})" end # The content of the default {#gemfile} # # @param [String] autoproj_version a constraint on the autoproj version # that should be used # @return [String] def default_gemfile_contents(autoproj_version = ">= 2.16.0") ["source \"#{gem_source}\"", "ruby \"#{RUBY_VERSION}\" if respond_to?(:ruby)", "gem \"autoproj\", \"#{autoproj_version}\""].join("\n") end def add_seed_config(path) @config.merge!(YAML.safe_load(File.read(path), [Symbol])) end # Parse the provided command line options and returns the non-options def parse_options(args = ARGV) options = OptionParser.new do |opt| opt.on "--local", "do not access the network (may fail)" do @local = true end opt.on "--skip-stage2", "do not run the stage2 install" do @skip_stage2 = true end opt.on "--debug", "Run in debug mode" do @autoproj_options << "--debug" end opt.on "--gem-source=URL", String, "use this source for RubyGems "\ "instead of rubygems.org" do |url| @gem_source = url end opt.on "--gems-path=PATH", "install gems under this path instead "\ "of #{default_gems_install_path} (do not use with --public-gems)" do |path| @gems_install_path = path end opt.on "--public-gems", "install gems in the default gem location: #{default_gems_install_path}"\ " (do not use with --gems-path)" do @gems_install_path = default_gems_install_path end opt.on "--bundler-version=VERSION_CONSTRAINT", String, "use the provided "\ "string as a version constraint for bundler" do |version| @config["bundler_version"] = version end opt.on "--version=VERSION_CONSTRAINT", String, "use the provided "\ "string as a version constraint for autoproj" do |version| raise "cannot give both --version and --gemfile" if @gemfile @gemfile = default_gemfile_contents(version) end opt.on "--gemfile=PATH", String, "use the given Gemfile to install "\ "autoproj instead of the default" do |path| raise "cannot give both --version and --gemfile" if @gemfile @gemfile = File.read(path) end opt.on "--no-seed-config", "when reinstalling an existing autoproj workspace, do not "\ "use the config in .autoproj/ as seed" do @config.clear @config["bundler_version"] = Install.default_bundler_version end opt.on "--seed-config=PATH", String, "path to a seed file that "\ "should be used to initialize the configuration" do |path| add_seed_config(path) end opt.on "--prefer-os-independent-packages", "prefer OS-independent "\ "packages (such as a RubyGem) over their OS-packaged equivalent "\ "(e.g. the thor gem vs. the ruby-thor debian package)" do @prefer_indep_over_os_packages = true end opt.on "--[no-]color", "do not use colored output (enabled by "\ "default if the terminal supports it)" do |color| if color then @autoproj_options << "--color" else @autoproj_options << "--no-color" end end opt.on "--[no-]progress", "do not use progress output (enabled by "\ "default if the terminal supports it)" do |progress| if progress then @autoproj_options << "--progress" else @autoproj_options << "--no-progress" end end opt.on "--[no-]interactive", "if non-interactive, use default "\ "answer for questions" do |flag| if flag then @autoproj_options << "--interactive" else @autoproj_options << "--no-interactive" end end end args = options.parse(ARGV) @autoproj_options + args end def bundler_version @config["bundler_version"] end def find_bundler(gem_program, version: nil) bundler_path = File.join(gems_gem_home, "bin", "bundle") return unless File.exist?(bundler_path) setup_paths = if version find_versioned_bundler_setup(gem_program, version) else find_unversioned_bundler_setup(gem_program) end setup_paths.each do |setup_path| return bundler_path if setup_path.start_with?(gems_gem_home) end nil end def find_versioned_bundler_setup(gem_program, version) contents = IO.popen( [env_for_child, Gem.ruby, gem_program, "contents", "-v", version, "bundler"], &:readlines ) return [] unless $CHILD_STATUS.success? contents.grep(%r{bundler/setup.rb$}) end def find_unversioned_bundler_setup(gem_program) setup_paths = IO.popen( [env_for_child, Gem.ruby, gem_program, "which", "-a", "bundler/setup"], &:readlines ) return [] unless $CHILD_STATUS.success? setup_paths end def install_bundler(gem_program, version: nil, silent: false) local = ["--local"] if local? redirection = Hash.new redirection = Hash[out: :close] if silent version_args = [] version_args << "-v" << version if version # Shut up the bundler warning about 'bin' not being in PATH env = self.env env = env.merge( { "PATH" => env["PATH"] + [File.join(gems_gem_home, "bin")] } ) result = system( env_for_child(env), Gem.ruby, gem_program, "install", "--env-shebang", "--no-document", "--no-format-executable", "--clear-sources", "--source", gem_source, "--no-user-install", "--install-dir", gems_gem_home, *local, "--bindir=#{File.join(gems_gem_home, 'bin')}", "bundler", *version_args, **redirection ) unless result STDERR.puts "FATAL: failed to install bundler in #{gems_gem_home}" nil end if (bundler_path = find_bundler(gem_program, version: version)) bundler_path else STDERR.puts "gem install bundler returned successfully, but still "\ "cannot find bundler in #{gems_gem_home}" nil end end def install_autoproj(bundler, bundler_version: self.bundler_version) # Force bundler to update. If the user does not want this, let # him specify a Gemfile with tighter version constraints lockfile = File.join(dot_autoproj, "Gemfile.lock") FileUtils.rm lockfile if File.exist?(lockfile) run_bundler(bundler, "config", "set", "--local", "path", gems_install_path, bundler_version: bundler_version) run_bundler(bundler, "config", "set", "--local", "shebang", Gem.ruby, bundler_version: bundler_version) install_args = ["--gemfile=#{autoproj_gemfile_path}"] install_args << "--local" if local? run_bundler(bundler, "install", *install_args, bundler_version: bundler_version) shims_path = File.join(dot_autoproj, "bin") run_bundler(bundler, "binstubs", "--all", "--force", "--path", shims_path, bundler_version: bundler_version) self.class.rewrite_shims( shims_path, ruby_executable, root_dir, autoproj_gemfile_path, gems_gem_home ) end class BundlerFailed < RuntimeError; end def run_bundler(bundler, *args, bundler_version: self.bundler_version) clean_env = env_for_child.dup version_arg = [] version_arg << "_#{bundler_version}_" if bundler_version result = system( clean_env, Gem.ruby, bundler, *version_arg, *args, chdir: dot_autoproj ) unless result raise BundlerFailed, "FAILED: bundler #{args.join(', ')} in #{dot_autoproj}" end end EXCLUDED_FROM_SHIMS = %w[rake thor].freeze def self.rewrite_shims(shim_path, ruby_executable, root_dir, autoproj_gemfile_path, gems_gem_home) FileUtils.mkdir_p shim_path File.open(File.join(shim_path, "ruby"), "w") do |io| io.puts "#! /bin/sh" io.puts "exec #{ruby_executable} \"$@\"" end FileUtils.chmod 0755, File.join(shim_path, "ruby") Dir.glob(File.join(shim_path, "*")) do |bin_script| next unless File.file?(bin_script) bin_name = File.basename(bin_script) if EXCLUDED_FROM_SHIMS.include?(bin_name) FileUtils.rm_f bin_script next end next if bin_name == "ruby" bin_shim = File.join(shim_path, bin_name) bin_script_lines = File.readlines(bin_script) next if has_autoproj_preamble?(bin_script_lines) next unless ruby_script?(bin_script_lines) File.open(bin_shim, "w") do |io| if bin_name == "bundler" || bin_name == "bundle" io.puts shim_bundler(bin_script_lines, ruby_executable, autoproj_gemfile_path, gems_gem_home) else io.puts shim_script(bin_script_lines, ruby_executable, root_dir, autoproj_gemfile_path, gems_gem_home) end end FileUtils.chmod 0755, bin_shim end end def self.ruby_script?(script_lines) script_lines.first =~ /\#\s*!(.*ruby.*)/ end def self.new_style_bundler_binstub?(script_lines) script_lines.any? { |l| l =~ /This file was generated by Bundler/ } end def self.has_autoproj_preamble?(script_lines) script_lines.any? { |l| l =~ /Autoproj generated preamble/ } end def self.shim_bundler(script_lines, ruby_executable, autoproj_gemfile_path, gems_gem_home) return shim_bundler_old(ruby_executable, autoproj_gemfile_path, gems_gem_home) \ unless new_style_bundler_binstub?(script_lines) script_lines.insert(1, <<-RESTART_BUNDLER) # # This file was generated by Bundler. # # Autoproj generated preamble #{WITHOUT_BUNDLER} ENV['BUNDLE_GEMFILE'] ||= '#{autoproj_gemfile_path}' ENV['GEM_HOME'] = '#{gems_gem_home}' ENV.delete('GEM_PATH') Gem.paths = Hash['GEM_HOME' => '#{gems_gem_home}', 'GEM_PATH' => ''] RESTART_BUNDLER script_lines.join end def self.shim_bundler_old(ruby_executable, autoproj_gemfile_path, gems_gem_home) "#! #{ruby_executable} #{WITHOUT_BUNDLER} ENV['BUNDLE_GEMFILE'] ||= '#{autoproj_gemfile_path}' ENV['GEM_HOME'] = '#{gems_gem_home}' ENV.delete('GEM_PATH') Gem.paths = Hash['GEM_HOME' => '#{gems_gem_home}', 'GEM_PATH' => ''] load Gem.bin_path('bundler', 'bundler')" end def self.shim_script(script_lines, ruby_executable, root_dir, autoproj_gemfile_path, gems_gem_home) new_style = !script_lines.empty? && script_lines.any? do |l| l =~ /This file was generated by Bundler/ end load_line = script_lines.grep(/load Gem.bin_path/).first return shim_script_old(ruby_executable, root_dir, autoproj_gemfile_path, gems_gem_home, load_line) \ unless new_style script_lines.insert(1, <<-AUTOPROJ_PREAMBLE) # # This file was generated by Bundler. # # Autoproj generated preamble, v1 #{RUBYLIB_REINIT} ENV['BUNDLE_GEMFILE'] = '#{autoproj_gemfile_path}' ENV['AUTOPROJ_CURRENT_ROOT'] = '#{root_dir}' Gem.paths = Hash['GEM_HOME' => '#{gems_gem_home}', 'GEM_PATH' => ''] AUTOPROJ_PREAMBLE script_lines.join end def self.shim_script_old(ruby_executable, root_dir, autoproj_gemfile_path, gems_gem_home, load_line) "#! #{ruby_executable} #{RUBYLIB_REINIT} ENV['BUNDLE_GEMFILE'] = '#{autoproj_gemfile_path}' ENV['AUTOPROJ_CURRENT_ROOT'] = '#{root_dir}' require 'rubygems' Gem.paths = Hash['GEM_HOME' => '#{gems_gem_home}', 'GEM_PATH' => ''] require 'bundler/setup' #{load_line}" end def save_env_sh(*vars) env = Autobuild::Environment.new env.prepare vars.each do |kv| k, *v = kv.split("=") v = v.join("=") if v.empty? env.unset k else env.set k, *v.split(File::PATH_SEPARATOR) end end # Generate environment files right now, we can at least use bundler File.open(File.join(dot_autoproj, "env.sh"), "w") do |io| env.export_env_sh(io) end # And now the root envsh env = Autobuild::Environment.new env.source_before File.join(dot_autoproj, "env.sh") env.set("AUTOPROJ_CURRENT_ROOT", root_dir) File.open(File.join(root_dir, "env.sh"), "w") do |io| env.export_env_sh(io) end end def save_gemfile gemfile = if @gemfile @gemfile elsif File.file?(autoproj_gemfile_path) File.read(autoproj_gemfile_path) else default_gemfile_contents end gemfile += [ "", "config_path = File.join(__dir__, 'config.yml')", "if File.file?(config_path)", " require 'yaml'", " config = YAML.load(File.read(config_path)) || Hash.new", " (config['plugins'] || Hash.new).", " each do |plugin_name, (version, options)|", " gem plugin_name, version, **options", " end", "end" ].join("\n") FileUtils.mkdir_p File.dirname(autoproj_gemfile_path) File.open(autoproj_gemfile_path, "w") do |io| io.write gemfile end end ENV_BUNDLE_GEMFILE_RX = /^(\s*ENV\[['"]BUNDLE_GEMFILE['"]\]\s*)(?:\|\|)?=/ def find_in_clean_path(command, *additional_paths) clean_path = env_for_child["PATH"].split(File::PATH_SEPARATOR) + additional_paths clean_path.each do |p| full_path = File.join(p, command) return full_path if File.file?(full_path) end nil end # The path of the bin/ folder for installed gems def gem_bindir return @gem_bindir if @gem_bindir # Here, we're getting into the esotheric # # The problem is that e.g. Ubuntu and Debian install an # operating_system.rb file that sets proper OS defaults. Some # autoproj installs have it in their RUBYLIB but should not # because of limitations of autoproj 1.x. This leads to # Gem.bindir being *not* valid for subprocesses # # So, we're calling 'gem' as a subcommand to discovery the # actual bindir bindir = IO.popen( env_for_child, [Gem.ruby, "-e", 'puts "#{Gem.user_dir}/bin"'], # rubocop:disable Lint/InterpolationCheck &:read ) if bindir @gem_bindir = bindir.chomp else raise "FATAL: cannot run #{Gem.ruby} -e 'puts Gem.bindir'" end end def install(bundler_version: self.bundler_version) if ENV["BUNDLER_GEMFILE"] raise "cannot run autoproj_install or autoproj_bootstrap while "\ "under a 'bundler exec' subcommand or having loaded an "\ "env.sh. Open a new console and try again" end gem_program = self.class.guess_gem_program puts "Detected 'gem' to be #{gem_program}" env["GEM_HOME"] = [gems_gem_home] env["GEM_PATH"] = [gems_gem_home] if (bundler = find_bundler(gem_program, version: bundler_version)) puts "Detected bundler at #{bundler}" else puts "Installing bundler in #{gems_gem_home}" bundler = install_bundler(gem_program, version: bundler_version) exit(1) unless bundler end self.class.rewrite_shims( File.join(dot_autoproj, "bin"), ruby_executable, root_dir, autoproj_gemfile_path, gems_gem_home ) env["PATH"].unshift File.join(dot_autoproj, "bin") save_gemfile puts "Installing autoproj in #{gems_gem_home}" install_autoproj(bundler, bundler_version: bundler_version) end def load_config v1_config_path = File.join(root_dir, "autoproj", "config.yml") config = Hash.new if File.file?(v1_config_path) config.merge!(YAML.load(File.read(v1_config_path)) || Hash.new) end if File.file?(autoproj_config_path) config.merge!(YAML.load(File.read(autoproj_config_path)) || Hash.new) end ruby = RbConfig::CONFIG["RUBY_INSTALL_NAME"] ruby_bindir = RbConfig::CONFIG["bindir"] ruby_executable = File.join(ruby_bindir, ruby) if (current = config["ruby_executable"]) # When upgrading or reinstalling if current != ruby_executable raise "this workspace has already been initialized using "\ "#{current}, you cannot run autoproj install with "\ "#{ruby_executable}. If you know what you're doing, "\ "delete the ruby_executable line in config.yml and try again" end else config["ruby_executable"] = ruby_executable end @config = config @config["bundler_version"] ||= self.class.default_bundler_version %w[gems_install_path prefer_indep_over_os_packages].each do |flag| instance_variable_set "@#{flag}", config.fetch(flag, false) end end def self.default_bundler_version if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("2.6.0") "2.3.6" elsif Gem::Version.new(RUBY_VERSION) < Gem::Version.new("3.0.0") "2.4.22" end end def save_config config["gems_install_path"] = gems_install_path config["prefer_indep_over_os_packages"] = prefer_indep_over_os_packages? File.open(autoproj_config_path, "w") { |io| YAML.dump(config, io) } end def autoproj_path File.join(dot_autoproj, "bin", "autoproj") end def run_autoproj(*args) system env_for_child.merge("BUNDLE_GEMFILE" => autoproj_gemfile_path), Gem.ruby, autoproj_path, *args, *@autoproj_options end def v1_workspace? File.file?(File.join(root_dir, "autoproj", "config.yml")) && !File.directory?(File.join(root_dir, ".autoproj")) end def stage1 if v1_workspace? && File.file?(v1_envsh = File.join(root_dir, "env.sh")) FileUtils.cp v1_envsh, "env.sh-autoproj-v1" end FileUtils.mkdir_p dot_autoproj save_config install rescue Exception FileUtils.rm_rf dot_autoproj raise end def call_stage2 clean_env = env_for_child stage2_vars = clean_env.map { |k, v| "#{k}=#{v}" } puts "starting the newly installed autoproj for stage2 install" unless run_autoproj("install-stage2", root_dir, *stage2_vars, *@autoproj_options) raise "failed to execute autoproj install-stage2" end end def stage2(*vars) require "autobuild" puts "saving temporary env.sh and .autoproj/env.sh" save_env_sh(*vars) puts "running 'autoproj envsh' to generate a proper env.sh" unless system(Gem.ruby, autoproj_path, "envsh", *@autoproj_options) STDERR.puts "failed to run autoproj envsh on the newly installed "\ "autoproj (#{autoproj_path})" exit 1 end # This is really needed on an existing install to install the # gems that were present in the v1 layout puts "running 'autoproj osdeps' to re-install missing gems" unless system(Gem.ruby, autoproj_path, "osdeps", *@autoproj_options) STDERR.puts "failed to run autoproj osdeps on the newly installed "\ "autoproj (#{autoproj_path})" exit 1 end end end end end ENV.delete("BUNDLE_GEMFILE") ENV.delete("RUBYLIB") ops = Autoproj::Ops::Install.new(Dir.pwd) existing_config = File.join(Dir.pwd, ".autoproj", "config.yml") if File.file?(existing_config) puts "Found existing configuration, using it as seed" puts "use --no-seed-config to avoid this behavior" ops.add_seed_config(existing_config) end ops.parse_options(ARGV) ops.stage1 ops.call_stage2 unless ops.skip_stage2?