proxygen
docker_builder.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 # Copyright (c) Facebook, Inc. and its affiliates.
3 from __future__ import absolute_import
4 from __future__ import division
5 from __future__ import print_function
6 from __future__ import unicode_literals
7 '''
8 
9 Extends FBCodeBuilder to produce Docker context directories.
10 
11 In order to get the largest iteration-time savings from Docker's build
12 caching, you will want to:
13  - Use fine-grained steps as appropriate (e.g. separate make & make install),
14  - Start your action sequence with the lowest-risk steps, and with the steps
15  that change the least often, and
16  - Put the steps that you are debugging towards the very end.
17 
18 '''
19 import logging
20 import os
21 import shutil
22 import tempfile
23 
24 from fbcode_builder import FBCodeBuilder
25 from shell_quoting import (
26  raw_shell, shell_comment, shell_join, ShellQuoted
27 )
28 from utils import recursively_flatten_list, run_command
29 
30 
31 class DockerFBCodeBuilder(FBCodeBuilder):
32 
33  def _user(self):
34  return self.option('user', 'root')
35 
36  def _change_user(self):
37  return ShellQuoted('USER {u}').format(u=self._user())
38 
39  def setup(self):
40  # Please add RPM-based OSes here as appropriate.
41  #
42  # To allow exercising non-root installs -- we change users after the
43  # system packages are installed. TODO: For users not defined in the
44  # image, we should probably `useradd`.
45  return self.step('Setup', [
46  # Docker's FROM does not understand shell quoting.
47  ShellQuoted('FROM {}'.format(self.option('os_image'))),
48  # /bin/sh syntax is a pain
49  ShellQuoted('SHELL ["/bin/bash", "-c"]'),
50  ] + self.install_debian_deps() + [self._change_user()])
51 
52  def step(self, name, actions):
53  assert '\n' not in name, 'Name {0} would span > 1 line'.format(name)
54  b = ShellQuoted('')
55  return [ShellQuoted('### {0} ###'.format(name)), b] + actions + [b]
56 
57  def run(self, shell_cmd):
58  return ShellQuoted('RUN {cmd}').format(cmd=shell_cmd)
59 
60  def workdir(self, dir):
61  return [
62  # As late as Docker 1.12.5, this results in `build` being owned
63  # by root:root -- the explicit `mkdir` works around the bug:
64  # USER nobody
65  # WORKDIR build
66  ShellQuoted('USER root'),
67  ShellQuoted('RUN mkdir -p {d} && chown {u} {d}').format(
68  d=dir, u=self._user()
69  ),
70  self._change_user(),
71  ShellQuoted('WORKDIR {dir}').format(dir=dir),
72  ]
73 
74  def comment(self, comment):
75  # This should not be a command since we don't want comment changes
76  # to invalidate the Docker build cache.
77  return shell_comment(comment)
78 
79  def copy_local_repo(self, repo_dir, dest_name):
80  fd, archive_path = tempfile.mkstemp(
81  prefix='local_repo_{0}_'.format(dest_name),
82  suffix='.tgz',
83  dir=os.path.abspath(self.option('docker_context_dir')),
84  )
85  os.close(fd)
86  run_command('tar', 'czf', archive_path, '.', cwd=repo_dir)
87  return [
88  ShellQuoted('ADD {archive} {dest_name}').format(
89  archive=os.path.basename(archive_path), dest_name=dest_name
90  ),
91  # Docker permissions make very little sense... see also workdir()
92  ShellQuoted('USER root'),
93  ShellQuoted('RUN chown -R {u} {d}').format(
94  d=dest_name, u=self._user()
95  ),
96  self._change_user(),
97  ]
98 
99  def _render_impl(self, steps):
100  return raw_shell(shell_join('\n', recursively_flatten_list(steps)))
101 
103  source_ccache_tgz = self.option('ccache_tgz', '')
104  if not source_ccache_tgz:
105  logging.info('Docker ccache not enabled')
106  return []
107 
108  dest_ccache_tgz = os.path.join(
109  self.option('docker_context_dir'), 'ccache.tgz'
110  )
111 
112  try:
113  try:
114  os.link(source_ccache_tgz, dest_ccache_tgz)
115  except OSError:
116  logging.exception(
117  'Hard-linking {s} to {d} failed, falling back to copy'
118  .format(s=source_ccache_tgz, d=dest_ccache_tgz)
119  )
120  shutil.copyfile(source_ccache_tgz, dest_ccache_tgz)
121  except Exception:
122  logging.exception(
123  'Failed to copy or link {s} to {d}, aborting'
124  .format(s=source_ccache_tgz, d=dest_ccache_tgz)
125  )
126  raise
127 
128  return [
129  # Separate layer so that in development we avoid re-downloads.
130  self.run(ShellQuoted('apt-get install -yq ccache')),
131  ShellQuoted('ADD ccache.tgz /'),
132  ShellQuoted(
133  # Set CCACHE_DIR before the `ccache` invocations below.
134  'ENV CCACHE_DIR=/ccache '
135  # No clang support for now, so it's easiest to hardcode gcc.
136  'CC="ccache gcc" CXX="ccache g++" '
137  # Always log for ease of debugging. For real FB projects,
138  # this log is several megabytes, so dumping it to stdout
139  # would likely exceed the Travis log limit of 4MB.
140  #
141  # On a local machine, `docker cp` will get you the data. To
142  # get the data out from Travis, I would compress and dump
143  # uuencoded bytes to the log -- for Bistro this was about
144  # 600kb or 8000 lines:
145  #
146  # apt-get install sharutils
147  # bzip2 -9 < /tmp/ccache.log | uuencode -m ccache.log.bz2
148  'CCACHE_LOGFILE=/tmp/ccache.log'
149  ),
150  self.run(ShellQuoted(
151  # Future: Skipping this part made this Docker step instant,
152  # saving ~1min of build time. It's unclear if it is the
153  # chown or the du, but probably the chown -- since a large
154  # part of the cost is incurred at image save time.
155  #
156  # ccache.tgz may be empty, or may have the wrong
157  # permissions.
158  'mkdir -p /ccache && time chown -R nobody /ccache && '
159  'time du -sh /ccache && '
160  # Reset stats so `docker_build_with_ccache.sh` can print
161  # useful values at the end of the run.
162  'echo === Prev run stats === && ccache -s && ccache -z && '
163  # Record the current time to let travis_build.sh figure out
164  # the number of bytes in the cache that are actually used --
165  # this is crucial for tuning the maximum cache size.
166  'date +%s > /FBCODE_BUILDER_CCACHE_START_TIME && '
167  # The build running as `nobody` should be able to write here
168  'chown nobody /tmp/ccache.log'
169  )),
170  ]
def step(self, name, actions)
def raw_shell(s)
def recursively_flatten_list(l)
Definition: utils.py:19
def shell_comment(c)
def run_command(cmd, kwargs)
Definition: utils.py:26
Formatter< false, Args... > format(StringPiece fmt, Args &&...args)
Definition: Format.h:271
def copy_local_repo(self, repo_dir, dest_name)
def option(self, name, default=None)
def shell_join(delim, it)