docker_builder.DockerFBCodeBuilder Class Reference

Inheritance diagram for docker_builder.DockerFBCodeBuilder:

Public Member Functions
def	setup (self)
def	step (self, name, actions)
def	run (self, shell_cmd)
def	workdir (self, dir)
def	comment (self, comment)
def	copy_local_repo (self, repo_dir, dest_name)
def	debian_ccache_setup_steps (self)
def	setup (self)
def	step (self, name, actions)
def	run (self, shell_cmd)
def	workdir (self, dir)
def	comment (self, comment)
def	copy_local_repo (self, repo_dir, dest_name)
def	debian_ccache_setup_steps (self)
def	setup (self)
def	step (self, name, actions)
def	run (self, shell_cmd)
def	workdir (self, dir)
def	comment (self, comment)
def	copy_local_repo (self, repo_dir, dest_name)
def	debian_ccache_setup_steps (self)
def	setup (self)
def	step (self, name, actions)
def	run (self, shell_cmd)
def	workdir (self, dir)
def	comment (self, comment)
def	copy_local_repo (self, repo_dir, dest_name)
def	debian_ccache_setup_steps (self)
Public Member Functions inherited from fbcode_builder.FBCodeBuilder
def	__init__ (self, kwargs)
def	__repr__ (self)
def	option (self, name, default=None)
def	has_option (self, name)
def	add_option (self, name, value)
def	render (self, steps)
def	build (self, steps)
def	setup (self)
def	diagnostics (self)
def	step (self, name, actions)
def	run (self, shell_cmd)
def	workdir (self, dir)
def	copy_local_repo (self, dir, dest_name)
def	debian_deps (self)
def	install_debian_deps (self)
def	debian_ccache_setup_steps (self)
def	github_project_workdir (self, project, path)
def	fb_github_project_workdir (self, project_and_path, github_org='facebook')
def	parallel_make (self, make_vars=None)
def	make_and_install (self, make_vars=None)
def	configure (self, name=None)
def	autoconf_install (self, name)
def	cmake_configure (self, name, cmake_path='..')
def	cmake_install (self, name, cmake_path='..')
def	fb_github_autoconf_install (self, project_and_path, github_org='facebook')
def	fb_github_cmake_install (self, project_and_path, cmake_path='..', github_org='facebook')
def	__init__ (self, kwargs)
def	__repr__ (self)
def	option (self, name, default=None)
def	has_option (self, name)
def	add_option (self, name, value)
def	render (self, steps)
def	build (self, steps)
def	setup (self)
def	diagnostics (self)
def	step (self, name, actions)
def	run (self, shell_cmd)
def	workdir (self, dir)
def	copy_local_repo (self, dir, dest_name)
def	debian_deps (self)
def	install_debian_deps (self)
def	debian_ccache_setup_steps (self)
def	github_project_workdir (self, project, path)
def	fb_github_project_workdir (self, project_and_path, github_org='facebook')
def	parallel_make (self, make_vars=None)
def	make_and_install (self, make_vars=None)
def	configure (self, name=None)
def	autoconf_install (self, name)
def	cmake_configure (self, name, cmake_path='..')
def	cmake_install (self, name, cmake_path='..')
def	fb_github_autoconf_install (self, project_and_path, github_org='facebook')
def	fb_github_cmake_install (self, project_and_path, cmake_path='..', github_org='facebook')
def	__init__ (self, kwargs)
def	__repr__ (self)
def	option (self, name, default=None)
def	has_option (self, name)
def	add_option (self, name, value)
def	render (self, steps)
def	build (self, steps)
def	setup (self)
def	diagnostics (self)
def	step (self, name, actions)
def	run (self, shell_cmd)
def	workdir (self, dir)
def	copy_local_repo (self, dir, dest_name)
def	debian_deps (self)
def	install_debian_deps (self)
def	debian_ccache_setup_steps (self)
def	github_project_workdir (self, project, path)
def	fb_github_project_workdir (self, project_and_path, github_org='facebook')
def	parallel_make (self, make_vars=None)
def	make_and_install (self, make_vars=None)
def	configure (self, name=None)
def	autoconf_install (self, name)
def	cmake_configure (self, name, cmake_path='..')
def	cmake_install (self, name, cmake_path='..')
def	fb_github_autoconf_install (self, project_and_path, github_org='facebook')
def	fb_github_cmake_install (self, project_and_path, cmake_path='..', github_org='facebook')
def	__init__ (self, kwargs)
def	__repr__ (self)
def	option (self, name, default=None)
def	has_option (self, name)
def	add_option (self, name, value)
def	render (self, steps)
def	build (self, steps)
def	setup (self)
def	diagnostics (self)
def	step (self, name, actions)
def	run (self, shell_cmd)
def	workdir (self, dir)
def	copy_local_repo (self, dir, dest_name)
def	debian_deps (self)
def	install_debian_deps (self)
def	debian_ccache_setup_steps (self)
def	github_project_workdir (self, project, path)
def	fb_github_project_workdir (self, project_and_path, github_org='facebook')
def	parallel_make (self, make_vars=None)
def	make_and_install (self, make_vars=None)
def	configure (self, name=None)
def	autoconf_install (self, name)
def	cmake_configure (self, name, cmake_path='..')
def	cmake_install (self, name, cmake_path='..')
def	fb_github_autoconf_install (self, project_and_path, github_org='facebook')
def	fb_github_cmake_install (self, project_and_path, cmake_path='..', github_org='facebook')

Private Member Functions
def	_user (self)
def	_change_user (self)
def	_render_impl (self, steps)
def	_user (self)
def	_change_user (self)
def	_render_impl (self, steps)
def	_user (self)
def	_change_user (self)
def	_render_impl (self, steps)
def	_user (self)
def	_change_user (self)
def	_render_impl (self, steps)

Additional Inherited Members
Public Attributes inherited from fbcode_builder.FBCodeBuilder
	options_used

Detailed Description

Definition at line 31 of file docker_builder.py.

Member Function Documentation

def docker_builder.DockerFBCodeBuilder._change_user (   self )

private

Definition at line 36 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._user(), and folly.format().

Referenced by docker_builder.DockerFBCodeBuilder._change_user(), docker_builder.DockerFBCodeBuilder.copy_local_repo(), docker_builder.DockerFBCodeBuilder.setup(), and docker_builder.DockerFBCodeBuilder.workdir().

    def _change_user(self):
        return ShellQuoted('USER {u}').format(u=self._user())

def docker_builder.DockerFBCodeBuilder._change_user (   self )

private

Definition at line 36 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._change_user(), docker_builder.DockerFBCodeBuilder._user(), and folly.format().

    def _change_user(self):
        return ShellQuoted('USER {u}').format(u=self._user())

def docker_builder.DockerFBCodeBuilder._change_user (   self )

private

Definition at line 36 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._change_user(), docker_builder.DockerFBCodeBuilder._user(), and folly.format().

    def _change_user(self):
        return ShellQuoted('USER {u}').format(u=self._user())

def docker_builder.DockerFBCodeBuilder._change_user (   self )

private

Definition at line 36 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._change_user(), docker_builder.DockerFBCodeBuilder._user(), and folly.format().

    def _change_user(self):
        return ShellQuoted('USER {u}').format(u=self._user())

def docker_builder.DockerFBCodeBuilder._render_impl (   self,   steps  )

private

Definition at line 99 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._render_impl(), shell_quoting.raw_shell(), utils.recursively_flatten_list(), and shell_quoting.shell_join().

Referenced by fbcode_builder.FBCodeBuilder.render().

    def _render_impl(self, steps):
        return raw_shell(shell_join('\n', recursively_flatten_list(steps)))

def docker_builder.DockerFBCodeBuilder._render_impl (   self,   steps  )

private

Definition at line 99 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._render_impl(), shell_quoting.raw_shell(), utils.recursively_flatten_list(), and shell_quoting.shell_join().

Referenced by fbcode_builder.FBCodeBuilder.render().

    def _render_impl(self, steps):
        return raw_shell(shell_join('\n', recursively_flatten_list(steps)))

def docker_builder.DockerFBCodeBuilder._render_impl (   self,   steps  )

private

Definition at line 99 of file docker_builder.py.

References shell_quoting.raw_shell(), utils.recursively_flatten_list(), and shell_quoting.shell_join().

Referenced by docker_builder.DockerFBCodeBuilder._render_impl(), and fbcode_builder.FBCodeBuilder.render().

    def _render_impl(self, steps):
        return raw_shell(shell_join('\n', recursively_flatten_list(steps)))

def docker_builder.DockerFBCodeBuilder._render_impl (   self,   steps  )

private

Definition at line 99 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._render_impl(), shell_quoting.raw_shell(), utils.recursively_flatten_list(), and shell_quoting.shell_join().

Referenced by fbcode_builder.FBCodeBuilder.render().

    def _render_impl(self, steps):
        return raw_shell(shell_join('\n', recursively_flatten_list(steps)))

def docker_builder.DockerFBCodeBuilder._user (   self )

private

Definition at line 33 of file docker_builder.py.

References fbcode_builder.FBCodeBuilder.option().

Referenced by docker_builder.DockerFBCodeBuilder._change_user(), docker_builder.DockerFBCodeBuilder._user(), docker_builder.DockerFBCodeBuilder.copy_local_repo(), and docker_builder.DockerFBCodeBuilder.workdir().

    def _user(self):
        return self.option('user', 'root')

def docker_builder.DockerFBCodeBuilder._user (   self )

private

Definition at line 33 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._user(), and fbcode_builder.FBCodeBuilder.option().

    def _user(self):
        return self.option('user', 'root')

def docker_builder.DockerFBCodeBuilder._user (   self )

private

Definition at line 33 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._user(), and fbcode_builder.FBCodeBuilder.option().

    def _user(self):
        return self.option('user', 'root')

def docker_builder.DockerFBCodeBuilder._user (   self )

private

Definition at line 33 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._user(), and fbcode_builder.FBCodeBuilder.option().

    def _user(self):
        return self.option('user', 'root')

def docker_builder.DockerFBCodeBuilder.comment	(		self,
			comment
	)

Definition at line 74 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder.comment(), and shell_quoting.shell_comment().

Referenced by fbcode_builder.FBCodeBuilder.diagnostics().

    def comment(self, comment):
        # This should not be a command since we don't want comment changes
        # to invalidate the Docker build cache.
        return shell_comment(comment)

def docker_builder.DockerFBCodeBuilder.comment	(		self,
			comment
	)

Definition at line 74 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder.comment(), and shell_quoting.shell_comment().

Referenced by fbcode_builder.FBCodeBuilder.diagnostics().

    def comment(self, comment):
        # This should not be a command since we don't want comment changes
        # to invalidate the Docker build cache.
        return shell_comment(comment)

def docker_builder.DockerFBCodeBuilder.comment	(		self,
			comment
	)

Definition at line 74 of file docker_builder.py.

References shell_quoting.shell_comment().

Referenced by docker_builder.DockerFBCodeBuilder.comment(), and fbcode_builder.FBCodeBuilder.diagnostics().

    def comment(self, comment):
        # This should not be a command since we don't want comment changes
        # to invalidate the Docker build cache.
        return shell_comment(comment)

def docker_builder.DockerFBCodeBuilder.comment	(		self,
			comment
	)

Definition at line 74 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder.comment(), and shell_quoting.shell_comment().

Referenced by fbcode_builder.FBCodeBuilder.diagnostics().

    def comment(self, comment):
        # This should not be a command since we don't want comment changes
        # to invalidate the Docker build cache.
        return shell_comment(comment)

def docker_builder.DockerFBCodeBuilder.copy_local_repo	(		self,
			repo_dir,
			dest_name
	)

Definition at line 79 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._change_user(), docker_builder.DockerFBCodeBuilder._user(), docker_builder.DockerFBCodeBuilder.copy_local_repo(), folly.format(), fbcode_builder.FBCodeBuilder.option(), and utils.run_command().

    def copy_local_repo(self, repo_dir, dest_name):
        fd, archive_path = tempfile.mkstemp(
            prefix='local_repo_{0}_'.format(dest_name),
            suffix='.tgz',
            dir=os.path.abspath(self.option('docker_context_dir')),
        )
        os.close(fd)
        run_command('tar', 'czf', archive_path, '.', cwd=repo_dir)
        return [
            ShellQuoted('ADD {archive} {dest_name}').format(
                archive=os.path.basename(archive_path), dest_name=dest_name
            ),
            # Docker permissions make very little sense... see also workdir()
            ShellQuoted('USER root'),
            ShellQuoted('RUN chown -R {u} {d}').format(
                d=dest_name, u=self._user()
            ),
            self._change_user(),
        ]

def docker_builder.DockerFBCodeBuilder.copy_local_repo	(		self,
			repo_dir,
			dest_name
	)

Definition at line 79 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._change_user(), docker_builder.DockerFBCodeBuilder._user(), folly.format(), fbcode_builder.FBCodeBuilder.option(), and utils.run_command().

Referenced by docker_builder.DockerFBCodeBuilder.copy_local_repo(), and fbcode_builder.FBCodeBuilder.github_project_workdir().

    def copy_local_repo(self, repo_dir, dest_name):
        fd, archive_path = tempfile.mkstemp(
            prefix='local_repo_{0}_'.format(dest_name),
            suffix='.tgz',
            dir=os.path.abspath(self.option('docker_context_dir')),
        )
        os.close(fd)
        run_command('tar', 'czf', archive_path, '.', cwd=repo_dir)
        return [
            ShellQuoted('ADD {archive} {dest_name}').format(
                archive=os.path.basename(archive_path), dest_name=dest_name
            ),
            # Docker permissions make very little sense... see also workdir()
            ShellQuoted('USER root'),
            ShellQuoted('RUN chown -R {u} {d}').format(
                d=dest_name, u=self._user()
            ),
            self._change_user(),
        ]

def docker_builder.DockerFBCodeBuilder.copy_local_repo	(		self,
			repo_dir,
			dest_name
	)

Definition at line 79 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._change_user(), docker_builder.DockerFBCodeBuilder._user(), docker_builder.DockerFBCodeBuilder.copy_local_repo(), folly.format(), fbcode_builder.FBCodeBuilder.option(), and utils.run_command().

    def copy_local_repo(self, repo_dir, dest_name):
        fd, archive_path = tempfile.mkstemp(
            prefix='local_repo_{0}_'.format(dest_name),
            suffix='.tgz',
            dir=os.path.abspath(self.option('docker_context_dir')),
        )
        os.close(fd)
        run_command('tar', 'czf', archive_path, '.', cwd=repo_dir)
        return [
            ShellQuoted('ADD {archive} {dest_name}').format(
                archive=os.path.basename(archive_path), dest_name=dest_name
            ),
            # Docker permissions make very little sense... see also workdir()
            ShellQuoted('USER root'),
            ShellQuoted('RUN chown -R {u} {d}').format(
                d=dest_name, u=self._user()
            ),
            self._change_user(),
        ]

def docker_builder.DockerFBCodeBuilder.copy_local_repo	(		self,
			repo_dir,
			dest_name
	)

Definition at line 79 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._change_user(), docker_builder.DockerFBCodeBuilder._user(), docker_builder.DockerFBCodeBuilder.copy_local_repo(), folly.format(), fbcode_builder.FBCodeBuilder.option(), and utils.run_command().

    def copy_local_repo(self, repo_dir, dest_name):
        fd, archive_path = tempfile.mkstemp(
            prefix='local_repo_{0}_'.format(dest_name),
            suffix='.tgz',
            dir=os.path.abspath(self.option('docker_context_dir')),
        )
        os.close(fd)
        run_command('tar', 'czf', archive_path, '.', cwd=repo_dir)
        return [
            ShellQuoted('ADD {archive} {dest_name}').format(
                archive=os.path.basename(archive_path), dest_name=dest_name
            ),
            # Docker permissions make very little sense... see also workdir()
            ShellQuoted('USER root'),
            ShellQuoted('RUN chown -R {u} {d}').format(
                d=dest_name, u=self._user()
            ),
            self._change_user(),
        ]

def docker_builder.DockerFBCodeBuilder.debian_ccache_setup_steps

(

self

)

Definition at line 102 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder.debian_ccache_setup_steps(), folly.format(), fbcode_builder.FBCodeBuilder.option(), and docker_builder.DockerFBCodeBuilder.run().

    def debian_ccache_setup_steps(self):
        source_ccache_tgz = self.option('ccache_tgz', '')
        if not source_ccache_tgz:
            logging.info('Docker ccache not enabled')
            return []

        dest_ccache_tgz = os.path.join(
            self.option('docker_context_dir'), 'ccache.tgz'
        )

        try:
            try:
                os.link(source_ccache_tgz, dest_ccache_tgz)
            except OSError:
                logging.exception(
                    'Hard-linking {s} to {d} failed, falling back to copy'
                    .format(s=source_ccache_tgz, d=dest_ccache_tgz)
                )
                shutil.copyfile(source_ccache_tgz, dest_ccache_tgz)
        except Exception:
            logging.exception(
                'Failed to copy or link {s} to {d}, aborting'
                .format(s=source_ccache_tgz, d=dest_ccache_tgz)
            )
            raise

        return [
            # Separate layer so that in development we avoid re-downloads.
            self.run(ShellQuoted('apt-get install -yq ccache')),
            ShellQuoted('ADD ccache.tgz /'),
            ShellQuoted(
                # Set CCACHE_DIR before the `ccache` invocations below.
                'ENV CCACHE_DIR=/ccache '
                # No clang support for now, so it's easiest to hardcode gcc.
                'CC="ccache gcc" CXX="ccache g++" '
                # Always log for ease of debugging. For real FB projects,
                # this log is several megabytes, so dumping it to stdout
                # would likely exceed the Travis log limit of 4MB.
                #
                # On a local machine, `docker cp` will get you the data.  To
                # get the data out from Travis, I would compress and dump
                # uuencoded bytes to the log -- for Bistro this was about
                # 600kb or 8000 lines:
                #
                #   apt-get install sharutils
                #   bzip2 -9 < /tmp/ccache.log | uuencode -m ccache.log.bz2
                'CCACHE_LOGFILE=/tmp/ccache.log'
            ),
            self.run(ShellQuoted(
                # Future: Skipping this part made this Docker step instant,
                # saving ~1min of build time.  It's unclear if it is the
                # chown or the du, but probably the chown -- since a large
                # part of the cost is incurred at image save time.
                #
                # ccache.tgz may be empty, or may have the wrong
                # permissions.
                'mkdir -p /ccache && time chown -R nobody /ccache && '
                'time du -sh /ccache && '
                # Reset stats so `docker_build_with_ccache.sh` can print
                # useful values at the end of the run.
                'echo === Prev run stats === && ccache -s && ccache -z && '
                # Record the current time to let travis_build.sh figure out
                # the number of bytes in the cache that are actually used --
                # this is crucial for tuning the maximum cache size.
                'date +%s > /FBCODE_BUILDER_CCACHE_START_TIME && '
                # The build running as `nobody` should be able to write here
                'chown nobody /tmp/ccache.log'
            )),
        ]

def docker_builder.DockerFBCodeBuilder.debian_ccache_setup_steps

(

self

)

Definition at line 102 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder.debian_ccache_setup_steps(), folly.format(), fbcode_builder.FBCodeBuilder.option(), and docker_builder.DockerFBCodeBuilder.run().

    def debian_ccache_setup_steps(self):
        source_ccache_tgz = self.option('ccache_tgz', '')
        if not source_ccache_tgz:
            logging.info('Docker ccache not enabled')
            return []

        dest_ccache_tgz = os.path.join(
            self.option('docker_context_dir'), 'ccache.tgz'
        )

        try:
            try:
                os.link(source_ccache_tgz, dest_ccache_tgz)
            except OSError:
                logging.exception(
                    'Hard-linking {s} to {d} failed, falling back to copy'
                    .format(s=source_ccache_tgz, d=dest_ccache_tgz)
                )
                shutil.copyfile(source_ccache_tgz, dest_ccache_tgz)
        except Exception:
            logging.exception(
                'Failed to copy or link {s} to {d}, aborting'
                .format(s=source_ccache_tgz, d=dest_ccache_tgz)
            )
            raise

        return [
            # Separate layer so that in development we avoid re-downloads.
            self.run(ShellQuoted('apt-get install -yq ccache')),
            ShellQuoted('ADD ccache.tgz /'),
            ShellQuoted(
                # Set CCACHE_DIR before the `ccache` invocations below.
                'ENV CCACHE_DIR=/ccache '
                # No clang support for now, so it's easiest to hardcode gcc.
                'CC="ccache gcc" CXX="ccache g++" '
                # Always log for ease of debugging. For real FB projects,
                # this log is several megabytes, so dumping it to stdout
                # would likely exceed the Travis log limit of 4MB.
                #
                # On a local machine, `docker cp` will get you the data.  To
                # get the data out from Travis, I would compress and dump
                # uuencoded bytes to the log -- for Bistro this was about
                # 600kb or 8000 lines:
                #
                #   apt-get install sharutils
                #   bzip2 -9 < /tmp/ccache.log | uuencode -m ccache.log.bz2
                'CCACHE_LOGFILE=/tmp/ccache.log'
            ),
            self.run(ShellQuoted(
                # Future: Skipping this part made this Docker step instant,
                # saving ~1min of build time.  It's unclear if it is the
                # chown or the du, but probably the chown -- since a large
                # part of the cost is incurred at image save time.
                #
                # ccache.tgz may be empty, or may have the wrong
                # permissions.
                'mkdir -p /ccache && time chown -R nobody /ccache && '
                'time du -sh /ccache && '
                # Reset stats so `docker_build_with_ccache.sh` can print
                # useful values at the end of the run.
                'echo === Prev run stats === && ccache -s && ccache -z && '
                # Record the current time to let travis_build.sh figure out
                # the number of bytes in the cache that are actually used --
                # this is crucial for tuning the maximum cache size.
                'date +%s > /FBCODE_BUILDER_CCACHE_START_TIME && '
                # The build running as `nobody` should be able to write here
                'chown nobody /tmp/ccache.log'
            )),
        ]

def docker_builder.DockerFBCodeBuilder.debian_ccache_setup_steps

(

self

)

Definition at line 102 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder.debian_ccache_setup_steps(), folly.format(), fbcode_builder.FBCodeBuilder.option(), and docker_builder.DockerFBCodeBuilder.run().

    def debian_ccache_setup_steps(self):
        source_ccache_tgz = self.option('ccache_tgz', '')
        if not source_ccache_tgz:
            logging.info('Docker ccache not enabled')
            return []

        dest_ccache_tgz = os.path.join(
            self.option('docker_context_dir'), 'ccache.tgz'
        )

        try:
            try:
                os.link(source_ccache_tgz, dest_ccache_tgz)
            except OSError:
                logging.exception(
                    'Hard-linking {s} to {d} failed, falling back to copy'
                    .format(s=source_ccache_tgz, d=dest_ccache_tgz)
                )
                shutil.copyfile(source_ccache_tgz, dest_ccache_tgz)
        except Exception:
            logging.exception(
                'Failed to copy or link {s} to {d}, aborting'
                .format(s=source_ccache_tgz, d=dest_ccache_tgz)
            )
            raise

        return [
            # Separate layer so that in development we avoid re-downloads.
            self.run(ShellQuoted('apt-get install -yq ccache')),
            ShellQuoted('ADD ccache.tgz /'),
            ShellQuoted(
                # Set CCACHE_DIR before the `ccache` invocations below.
                'ENV CCACHE_DIR=/ccache '
                # No clang support for now, so it's easiest to hardcode gcc.
                'CC="ccache gcc" CXX="ccache g++" '
                # Always log for ease of debugging. For real FB projects,
                # this log is several megabytes, so dumping it to stdout
                # would likely exceed the Travis log limit of 4MB.
                #
                # On a local machine, `docker cp` will get you the data.  To
                # get the data out from Travis, I would compress and dump
                # uuencoded bytes to the log -- for Bistro this was about
                # 600kb or 8000 lines:
                #
                #   apt-get install sharutils
                #   bzip2 -9 < /tmp/ccache.log | uuencode -m ccache.log.bz2
                'CCACHE_LOGFILE=/tmp/ccache.log'
            ),
            self.run(ShellQuoted(
                # Future: Skipping this part made this Docker step instant,
                # saving ~1min of build time.  It's unclear if it is the
                # chown or the du, but probably the chown -- since a large
                # part of the cost is incurred at image save time.
                #
                # ccache.tgz may be empty, or may have the wrong
                # permissions.
                'mkdir -p /ccache && time chown -R nobody /ccache && '
                'time du -sh /ccache && '
                # Reset stats so `docker_build_with_ccache.sh` can print
                # useful values at the end of the run.
                'echo === Prev run stats === && ccache -s && ccache -z && '
                # Record the current time to let travis_build.sh figure out
                # the number of bytes in the cache that are actually used --
                # this is crucial for tuning the maximum cache size.
                'date +%s > /FBCODE_BUILDER_CCACHE_START_TIME && '
                # The build running as `nobody` should be able to write here
                'chown nobody /tmp/ccache.log'
            )),
        ]

def docker_builder.DockerFBCodeBuilder.debian_ccache_setup_steps

(

self

)

Definition at line 102 of file docker_builder.py.

References folly.format(), fbcode_builder.FBCodeBuilder.option(), and docker_builder.DockerFBCodeBuilder.run().

Referenced by docker_builder.DockerFBCodeBuilder.debian_ccache_setup_steps(), and fbcode_builder.FBCodeBuilder.install_debian_deps().

    def debian_ccache_setup_steps(self):
        source_ccache_tgz = self.option('ccache_tgz', '')
        if not source_ccache_tgz:
            logging.info('Docker ccache not enabled')
            return []

        dest_ccache_tgz = os.path.join(
            self.option('docker_context_dir'), 'ccache.tgz'
        )

        try:
            try:
                os.link(source_ccache_tgz, dest_ccache_tgz)
            except OSError:
                logging.exception(
                    'Hard-linking {s} to {d} failed, falling back to copy'
                    .format(s=source_ccache_tgz, d=dest_ccache_tgz)
                )
                shutil.copyfile(source_ccache_tgz, dest_ccache_tgz)
        except Exception:
            logging.exception(
                'Failed to copy or link {s} to {d}, aborting'
                .format(s=source_ccache_tgz, d=dest_ccache_tgz)
            )
            raise

        return [
            # Separate layer so that in development we avoid re-downloads.
            self.run(ShellQuoted('apt-get install -yq ccache')),
            ShellQuoted('ADD ccache.tgz /'),
            ShellQuoted(
                # Set CCACHE_DIR before the `ccache` invocations below.
                'ENV CCACHE_DIR=/ccache '
                # No clang support for now, so it's easiest to hardcode gcc.
                'CC="ccache gcc" CXX="ccache g++" '
                # Always log for ease of debugging. For real FB projects,
                # this log is several megabytes, so dumping it to stdout
                # would likely exceed the Travis log limit of 4MB.
                #
                # On a local machine, `docker cp` will get you the data.  To
                # get the data out from Travis, I would compress and dump
                # uuencoded bytes to the log -- for Bistro this was about
                # 600kb or 8000 lines:
                #
                #   apt-get install sharutils
                #   bzip2 -9 < /tmp/ccache.log | uuencode -m ccache.log.bz2
                'CCACHE_LOGFILE=/tmp/ccache.log'
            ),
            self.run(ShellQuoted(
                # Future: Skipping this part made this Docker step instant,
                # saving ~1min of build time.  It's unclear if it is the
                # chown or the du, but probably the chown -- since a large
                # part of the cost is incurred at image save time.
                #
                # ccache.tgz may be empty, or may have the wrong
                # permissions.
                'mkdir -p /ccache && time chown -R nobody /ccache && '
                'time du -sh /ccache && '
                # Reset stats so `docker_build_with_ccache.sh` can print
                # useful values at the end of the run.
                'echo === Prev run stats === && ccache -s && ccache -z && '
                # Record the current time to let travis_build.sh figure out
                # the number of bytes in the cache that are actually used --
                # this is crucial for tuning the maximum cache size.
                'date +%s > /FBCODE_BUILDER_CCACHE_START_TIME && '
                # The build running as `nobody` should be able to write here
                'chown nobody /tmp/ccache.log'
            )),
        ]

def docker_builder.DockerFBCodeBuilder.run	(		self,
			shell_cmd
	)

Definition at line 57 of file docker_builder.py.

References folly.format().

Referenced by fbcode_builder.FBCodeBuilder.autoconf_install(), fbcode_builder.FBCodeBuilder.cmake_configure(), fbcode_builder.FBCodeBuilder.configure(), docker_builder.DockerFBCodeBuilder.debian_ccache_setup_steps(), fbcode_builder.FBCodeBuilder.diagnostics(), fbcode_builder.FBCodeBuilder.github_project_workdir(), fbcode_builder.FBCodeBuilder.install_debian_deps(), fbcode_builder.FBCodeBuilder.make_and_install(), fbcode_builder.FBCodeBuilder.parallel_make(), and docker_builder.DockerFBCodeBuilder.run().

    def run(self, shell_cmd):
        return ShellQuoted('RUN {cmd}').format(cmd=shell_cmd)

def docker_builder.DockerFBCodeBuilder.run	(		self,
			shell_cmd
	)

Definition at line 57 of file docker_builder.py.

References folly.format(), and docker_builder.DockerFBCodeBuilder.run().

    def run(self, shell_cmd):
        return ShellQuoted('RUN {cmd}').format(cmd=shell_cmd)

def docker_builder.DockerFBCodeBuilder.run	(		self,
			shell_cmd
	)

Definition at line 57 of file docker_builder.py.

References folly.format(), and docker_builder.DockerFBCodeBuilder.run().

    def run(self, shell_cmd):
        return ShellQuoted('RUN {cmd}').format(cmd=shell_cmd)

def docker_builder.DockerFBCodeBuilder.run	(		self,
			shell_cmd
	)

Definition at line 57 of file docker_builder.py.

References folly.format(), and docker_builder.DockerFBCodeBuilder.run().

    def run(self, shell_cmd):
        return ShellQuoted('RUN {cmd}').format(cmd=shell_cmd)

def docker_builder.DockerFBCodeBuilder.setup

(

self

)

Definition at line 39 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._change_user(), folly.format(), fbcode_builder.FBCodeBuilder.install_debian_deps(), fbcode_builder.FBCodeBuilder.option(), docker_builder.DockerFBCodeBuilder.setup(), and docker_builder.DockerFBCodeBuilder.step().

    def setup(self):
        # Please add RPM-based OSes here as appropriate.
        #
        # To allow exercising non-root installs -- we change users after the
        # system packages are installed.  TODO: For users not defined in the
        # image, we should probably `useradd`.
        return self.step('Setup', [
            # Docker's FROM does not understand shell quoting.
            ShellQuoted('FROM {}'.format(self.option('os_image'))),
            # /bin/sh syntax is a pain
            ShellQuoted('SHELL ["/bin/bash", "-c"]'),
        ] + self.install_debian_deps() + [self._change_user()])

def docker_builder.DockerFBCodeBuilder.setup

(

self

)

Definition at line 39 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._change_user(), folly.format(), fbcode_builder.FBCodeBuilder.install_debian_deps(), fbcode_builder.FBCodeBuilder.option(), docker_builder.DockerFBCodeBuilder.setup(), and docker_builder.DockerFBCodeBuilder.step().

    def setup(self):
        # Please add RPM-based OSes here as appropriate.
        #
        # To allow exercising non-root installs -- we change users after the
        # system packages are installed.  TODO: For users not defined in the
        # image, we should probably `useradd`.
        return self.step('Setup', [
            # Docker's FROM does not understand shell quoting.
            ShellQuoted('FROM {}'.format(self.option('os_image'))),
            # /bin/sh syntax is a pain
            ShellQuoted('SHELL ["/bin/bash", "-c"]'),
        ] + self.install_debian_deps() + [self._change_user()])

def docker_builder.DockerFBCodeBuilder.setup

(

self

)

Definition at line 39 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._change_user(), folly.format(), fbcode_builder.FBCodeBuilder.install_debian_deps(), fbcode_builder.FBCodeBuilder.option(), and docker_builder.DockerFBCodeBuilder.step().

Referenced by fbcode_builder.FBCodeBuilder.build(), and docker_builder.DockerFBCodeBuilder.setup().

    def setup(self):
        # Please add RPM-based OSes here as appropriate.
        #
        # To allow exercising non-root installs -- we change users after the
        # system packages are installed.  TODO: For users not defined in the
        # image, we should probably `useradd`.
        return self.step('Setup', [
            # Docker's FROM does not understand shell quoting.
            ShellQuoted('FROM {}'.format(self.option('os_image'))),
            # /bin/sh syntax is a pain
            ShellQuoted('SHELL ["/bin/bash", "-c"]'),
        ] + self.install_debian_deps() + [self._change_user()])

def docker_builder.DockerFBCodeBuilder.setup

(

self

)

Definition at line 39 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._change_user(), folly.format(), fbcode_builder.FBCodeBuilder.install_debian_deps(), fbcode_builder.FBCodeBuilder.option(), docker_builder.DockerFBCodeBuilder.setup(), and docker_builder.DockerFBCodeBuilder.step().

    def setup(self):
        # Please add RPM-based OSes here as appropriate.
        #
        # To allow exercising non-root installs -- we change users after the
        # system packages are installed.  TODO: For users not defined in the
        # image, we should probably `useradd`.
        return self.step('Setup', [
            # Docker's FROM does not understand shell quoting.
            ShellQuoted('FROM {}'.format(self.option('os_image'))),
            # /bin/sh syntax is a pain
            ShellQuoted('SHELL ["/bin/bash", "-c"]'),
        ] + self.install_debian_deps() + [self._change_user()])

def docker_builder.DockerFBCodeBuilder.step	(		self,
			name,
			actions
	)

Definition at line 52 of file docker_builder.py.

References folly.format(), and docker_builder.DockerFBCodeBuilder.step().

    def step(self, name, actions):
        assert '\n' not in name, 'Name {0} would span > 1 line'.format(name)
        b = ShellQuoted('')
        return [ShellQuoted('### {0} ###'.format(name)), b] + actions + [b]

def docker_builder.DockerFBCodeBuilder.step	(		self,
			name,
			actions
	)

Definition at line 52 of file docker_builder.py.

References folly.format(), and docker_builder.DockerFBCodeBuilder.step().

    def step(self, name, actions):
        assert '\n' not in name, 'Name {0} would span > 1 line'.format(name)
        b = ShellQuoted('')
        return [ShellQuoted('### {0} ###'.format(name)), b] + actions + [b]

def docker_builder.DockerFBCodeBuilder.step	(		self,
			name,
			actions
	)

Definition at line 52 of file docker_builder.py.

References folly.format(), and docker_builder.DockerFBCodeBuilder.step().

    def step(self, name, actions):
        assert '\n' not in name, 'Name {0} would span > 1 line'.format(name)
        b = ShellQuoted('')
        return [ShellQuoted('### {0} ###'.format(name)), b] + actions + [b]

def docker_builder.DockerFBCodeBuilder.step	(		self,
			name,
			actions
	)

Definition at line 52 of file docker_builder.py.

References folly.format().

Referenced by fbcode_builder.FBCodeBuilder.autoconf_install(), fbcode_builder.FBCodeBuilder.cmake_install(), fbcode_builder.FBCodeBuilder.diagnostics(), fbcode_builder.FBCodeBuilder.github_project_workdir(), fbcode_builder.FBCodeBuilder.install_debian_deps(), docker_builder.DockerFBCodeBuilder.setup(), and docker_builder.DockerFBCodeBuilder.step().

    def step(self, name, actions):
        assert '\n' not in name, 'Name {0} would span > 1 line'.format(name)
        b = ShellQuoted('')
        return [ShellQuoted('### {0} ###'.format(name)), b] + actions + [b]

def docker_builder.DockerFBCodeBuilder.workdir	(		self,
			dir
	)

Definition at line 60 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._change_user(), docker_builder.DockerFBCodeBuilder._user(), and folly.format().

Referenced by fbcode_builder.FBCodeBuilder.github_project_workdir(), and docker_builder.DockerFBCodeBuilder.workdir().

    def workdir(self, dir):
        return [
            # As late as Docker 1.12.5, this results in `build` being owned
            # by root:root -- the explicit `mkdir` works around the bug:
            #   USER nobody
            #   WORKDIR build
            ShellQuoted('USER root'),
            ShellQuoted('RUN mkdir -p {d} && chown {u} {d}').format(
                d=dir, u=self._user()
            ),
            self._change_user(),
            ShellQuoted('WORKDIR {dir}').format(dir=dir),
        ]

def docker_builder.DockerFBCodeBuilder.workdir	(		self,
			dir
	)

Definition at line 60 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._change_user(), docker_builder.DockerFBCodeBuilder._user(), folly.format(), and docker_builder.DockerFBCodeBuilder.workdir().

    def workdir(self, dir):
        return [
            # As late as Docker 1.12.5, this results in `build` being owned
            # by root:root -- the explicit `mkdir` works around the bug:
            #   USER nobody
            #   WORKDIR build
            ShellQuoted('USER root'),
            ShellQuoted('RUN mkdir -p {d} && chown {u} {d}').format(
                d=dir, u=self._user()
            ),
            self._change_user(),
            ShellQuoted('WORKDIR {dir}').format(dir=dir),
        ]

def docker_builder.DockerFBCodeBuilder.workdir	(		self,
			dir
	)

Definition at line 60 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._change_user(), docker_builder.DockerFBCodeBuilder._user(), folly.format(), and docker_builder.DockerFBCodeBuilder.workdir().

    def workdir(self, dir):
        return [
            # As late as Docker 1.12.5, this results in `build` being owned
            # by root:root -- the explicit `mkdir` works around the bug:
            #   USER nobody
            #   WORKDIR build
            ShellQuoted('USER root'),
            ShellQuoted('RUN mkdir -p {d} && chown {u} {d}').format(
                d=dir, u=self._user()
            ),
            self._change_user(),
            ShellQuoted('WORKDIR {dir}').format(dir=dir),
        ]

def docker_builder.DockerFBCodeBuilder.workdir	(		self,
			dir
	)

Definition at line 60 of file docker_builder.py.

References docker_builder.DockerFBCodeBuilder._change_user(), docker_builder.DockerFBCodeBuilder._user(), folly.format(), and docker_builder.DockerFBCodeBuilder.workdir().

    def workdir(self, dir):
        return [
            # As late as Docker 1.12.5, this results in `build` being owned
            # by root:root -- the explicit `mkdir` works around the bug:
            #   USER nobody
            #   WORKDIR build
            ShellQuoted('USER root'),
            ShellQuoted('RUN mkdir -p {d} && chown {u} {d}').format(
                d=dir, u=self._user()
            ),
            self._change_user(),
            ShellQuoted('WORKDIR {dir}').format(dir=dir),
        ]

---

The documentation for this class was generated from the following file:

• build/fbcode_builder/docker_builder.py

• docker_builder
• DockerFBCodeBuilder
• Generated by 1.8.11