proxygen
fbcode_builder.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 # Copyright (c) Facebook, Inc. and its affiliates.
3 from __future__ import absolute_import
4 from __future__ import division
5 from __future__ import print_function
6 from __future__ import unicode_literals
7 '''
8 
9 This is a small DSL to describe builds of Facebook's open-source projects
10 that are published to Github from a single internal repo, including projects
11 that depend on folly, wangle, proxygen, fbthrift, etc.
12 
13 This file defines the interface of the DSL, and common utilieis, but you
14 will have to instantiate a specific builder, with specific options, in
15 order to get work done -- see e.g. make_docker_context.py.
16 
17 == Design notes ==
18 
19 Goals:
20 
21  - A simple declarative language for what needs to be checked out & built,
22  how, in what order.
23 
24  - The same specification should work for external continuous integration
25  builds (e.g. Travis + Docker) and for internal VM-based continuous
26  integration builds.
27 
28  - One should be able to build without root, and to install to a prefix.
29 
30 Non-goals:
31 
32  - General usefulness. The only point of this is to make it easier to build
33  and test Facebook's open-source services.
34 
35 Ideas for the future -- these may not be very good :)
36 
37  - Especially on Ubuntu 14.04 the current initial setup is inefficient:
38  we add PPAs after having installed a bunch of packages -- this prompts
39  reinstalls of large amounts of code. We also `apt-get update` a few
40  times.
41 
42  - A "shell script" builder. Like DockerFBCodeBuilder, but outputs a
43  shell script that runs outside of a container. Or maybe even
44  synchronously executes the shell commands, `make`-style.
45 
46  - A "Makefile" generator. That might make iterating on builds even quicker
47  than what you can currently get with Docker build caching.
48 
49  - Generate a rebuild script that can be run e.g. inside the built Docker
50  container by tagging certain steps with list-inheriting Python objects:
51  * do change directories
52  * do NOT `git clone` -- if we want to update code this should be a
53  separate script that e.g. runs rebase on top of specific targets
54  across all the repos.
55  * do NOT install software (most / all setup can be skipped)
56  * do NOT `autoreconf` or `configure`
57  * do `make` and `cmake`
58 
59  - If we get non-Debian OSes, part of ccache setup should be factored out.
60 '''
61 
62 import os
63 import re
64 
65 from shell_quoting import path_join, shell_join, ShellQuoted
66 
67 
69  base_dir = 'deps/github_hashes/' # trailing slash used in regex below
70  for dirname, _, files in os.walk(base_dir):
71  for filename in files:
72  path = os.path.join(dirname, filename)
73  with open(path) as f:
74  m_proj = re.match('^' + base_dir + '(.*)-rev\.txt$', path)
75  if m_proj is None:
76  raise RuntimeError('Not a hash file? {0}'.format(path))
77  m_hash = re.match('^Subproject commit ([0-9a-f]+)\n$', f.read())
78  if m_hash is None:
79  raise RuntimeError('No hash in {0}'.format(path))
80  yield m_proj.group(1), m_hash.group(1)
81 
82 
83 class FBCodeBuilder(object):
84 
85  def __init__(self, **kwargs):
86  self._options_do_not_access = kwargs # Use .option() instead.
87  # This raises upon detecting options that are specified but unused,
88  # because otherwise it is very easy to make a typo in option names.
89  self.options_used = set()
91 
92  def __repr__(self):
93  return '{0}({1})'.format(
94  self.__class__.__name__,
95  ', '.join(
96  '{0}={1}'.format(k, repr(v))
97  for k, v in self._options_do_not_access.items()
98  )
99  )
100 
101  def option(self, name, default=None):
102  value = self._options_do_not_access.get(name, default)
103  if value is None:
104  raise RuntimeError('Option {0} is required'.format(name))
105  self.options_used.add(name)
106  return value
107 
108  def has_option(self, name):
109  return name in self._options_do_not_access
110 
111  def add_option(self, name, value):
112  if name in self._options_do_not_access:
113  raise RuntimeError('Option {0} already set'.format(name))
114  self._options_do_not_access[name] = value
115 
116  #
117  # Abstract parts common to every installation flow
118  #
119 
120  def render(self, steps):
121  '''
122 
123  Converts nested actions to your builder's expected output format.
124  Typically takes the output of build().
125 
126  '''
127  res = self._render_impl(steps) # Implementation-dependent
128  # Now that the output is rendered, we expect all options to have
129  # been used.
130  unused_options = set(self._options_do_not_access)
131  unused_options -= self.options_used
132  if unused_options:
133  raise RuntimeError(
134  'Unused options: {0} -- please check if you made a typo '
135  'in any of them. Those that are truly not useful should '
136  'be not be set so that this typo detection can be useful.'
137  .format(unused_options)
138  )
139  return res
140 
141  def build(self, steps):
142  if not steps:
143  raise RuntimeError('Please ensure that the config you are passing '
144  'contains steps')
145  return [self.setup(), self.diagnostics()] + steps
146 
147  def setup(self):
148  'Your builder may want to install packages here.'
149  raise NotImplementedError
150 
151  def diagnostics(self):
152  'Log some system diagnostics before/after setup for ease of debugging'
153  # The builder's repr is not used in a command to avoid pointlessly
154  # invalidating Docker's build cache.
155  return self.step('Diagnostics', [
156  self.comment('Builder {0}'.format(repr(self))),
157  self.run(ShellQuoted('hostname')),
158  self.run(ShellQuoted('cat /etc/issue || echo no /etc/issue')),
159  self.run(ShellQuoted('g++ --version || echo g++ not installed')),
160  self.run(ShellQuoted('cmake --version || echo cmake not installed')),
161  ])
162 
163  def step(self, name, actions):
164  'A labeled collection of actions or other steps'
165  raise NotImplementedError
166 
167  def run(self, shell_cmd):
168  'Run this bash command'
169  raise NotImplementedError
170 
171  def workdir(self, dir):
172  'Create this directory if it does not exist, and change into it'
173  raise NotImplementedError
174 
175  def copy_local_repo(self, dir, dest_name):
176  '''
177  Copy the local repo at `dir` into this step's `workdir()`, analog of:
178  cp -r /path/to/folly folly
179  '''
180  raise NotImplementedError
181 
182  def debian_deps(self):
183  return [
184  'autoconf-archive',
185  'bison',
186  'build-essential',
187  'cmake',
188  'curl',
189  'flex',
190  'git',
191  'gperf',
192  'joe',
193  'libboost-all-dev',
194  'libcap-dev',
195  'libdouble-conversion-dev',
196  'libevent-dev',
197  'libgflags-dev',
198  'libgoogle-glog-dev',
199  'libkrb5-dev',
200  'libpcre3-dev',
201  'libpthread-stubs0-dev',
202  'libnuma-dev',
203  'libsasl2-dev',
204  'libsnappy-dev',
205  'libsqlite3-dev',
206  'libssl-dev',
207  'libtool',
208  'netcat-openbsd',
209  'pkg-config',
210  'sudo',
211  'unzip',
212  'wget',
213  ]
214 
215  #
216  # Specific build helpers
217  #
218 
220  actions = [
221  self.run(
222  ShellQuoted('apt-get update && apt-get install -yq {deps}').format(
223  deps=shell_join(' ', (
224  ShellQuoted(dep) for dep in self.debian_deps())))
225  ),
226  ]
227  gcc_version = self.option('gcc_version')
228 
229  # Make the selected GCC the default before building anything
230  actions.extend([
231  self.run(ShellQuoted('apt-get install -yq {c} {cpp}').format(
232  c=ShellQuoted('gcc-{v}').format(v=gcc_version),
233  cpp=ShellQuoted('g++-{v}').format(v=gcc_version),
234  )),
235  self.run(ShellQuoted(
236  'update-alternatives --install /usr/bin/gcc gcc {c} 40 '
237  '--slave /usr/bin/g++ g++ {cpp}'
238  ).format(
239  c=ShellQuoted('/usr/bin/gcc-{v}').format(v=gcc_version),
240  cpp=ShellQuoted('/usr/bin/g++-{v}').format(v=gcc_version),
241  )),
242  self.run(ShellQuoted('update-alternatives --config gcc')),
243  ])
244 
245  actions.extend(self.debian_ccache_setup_steps())
246 
247  return self.step('Install packages for Debian-based OS', actions)
248 
250  return [] # It's ok to ship a renderer without ccache support.
251 
252  def github_project_workdir(self, project, path):
253  # Only check out a non-default branch if requested. This especially
254  # makes sense when building from a local repo.
255  git_hash = self.option(
256  '{0}:git_hash'.format(project),
257  # Any repo that has a hash in deps/github_hashes defaults to
258  # that, with the goal of making builds maximally consistent.
259  self._github_hashes.get(project, '')
260  )
261  maybe_change_branch = [
262  self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)),
263  ] if git_hash else []
264 
265  base_dir = self.option('projects_dir')
266 
267  local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '')
268  return self.step('Check out {0}, workdir {1}'.format(project, path), [
269  self.workdir(base_dir),
270  self.run(
271  ShellQuoted('git clone https://github.com/{p}').format(p=project)
272  ) if not local_repo_dir else self.copy_local_repo(
273  local_repo_dir, os.path.basename(project)
274  ),
275  self.workdir(path_join(base_dir, os.path.basename(project), path)),
276  ] + maybe_change_branch)
277 
278  def fb_github_project_workdir(self, project_and_path, github_org='facebook'):
279  'This helper lets Facebook-internal CI special-cases FB projects'
280  project, path = project_and_path.split('/', 1)
281  return self.github_project_workdir(github_org + '/' + project, path)
282 
283  def _make_vars(self, make_vars):
284  return shell_join(' ', (
285  ShellQuoted('{k}={v}').format(k=k, v=v)
286  for k, v in ({} if make_vars is None else make_vars).items()
287  ))
288 
289  def parallel_make(self, make_vars=None):
290  return self.run(ShellQuoted('make -j {n} {vars}').format(
291  n=self.option('make_parallelism'),
292  vars=self._make_vars(make_vars),
293  ))
294 
295  def make_and_install(self, make_vars=None):
296  return [
297  self.parallel_make(make_vars),
298  self.run(ShellQuoted('make install {vars}').format(
299  vars=self._make_vars(make_vars),
300  )),
301  ]
302 
303  def configure(self, name=None):
304  autoconf_options = {}
305  if name is not None:
306  autoconf_options.update(
307  self.option('{0}:autoconf_options'.format(name), {})
308  )
309  return [
310  self.run(ShellQuoted(
311  'LDFLAGS="$LDFLAGS -L"{p}"/lib -Wl,-rpath="{p}"/lib" '
312  'CFLAGS="$CFLAGS -I"{p}"/include" '
313  'CPPFLAGS="$CPPFLAGS -I"{p}"/include" '
314  'PY_PREFIX={p} '
315  './configure --prefix={p} {args}'
316  ).format(
317  p=self.option('prefix'),
318  args=shell_join(' ', (
319  ShellQuoted('{k}={v}').format(k=k, v=v)
320  for k, v in autoconf_options.items()
321  )),
322  )),
323  ]
324 
325  def autoconf_install(self, name):
326  return self.step('Build and install {0}'.format(name), [
327  self.run(ShellQuoted('autoreconf -ivf')),
328  ] + self.configure() + self.make_and_install())
329 
330  def cmake_configure(self, name, cmake_path='..'):
331  cmake_defines = {
332  'BUILD_SHARED_LIBS': 'ON',
333  'CMAKE_INSTALL_PREFIX': self.option('prefix'),
334  }
335  cmake_defines.update(
336  self.option('{0}:cmake_defines'.format(name), {})
337  )
338  return [
339  self.run(ShellQuoted(
340  'CXXFLAGS="$CXXFLAGS -fPIC -isystem "{p}"/include" '
341  'CFLAGS="$CFLAGS -fPIC -isystem "{p}"/include" '
342  'cmake {args} {cmake_path}'
343  ).format(
344  p=self.option('prefix'),
345  args=shell_join(' ', (
346  ShellQuoted('-D{k}={v}').format(k=k, v=v)
347  for k, v in cmake_defines.items()
348  )),
349  cmake_path=cmake_path,
350  )),
351  ]
352 
353  def cmake_install(self, name, cmake_path='..'):
354  return self.step(
355  'Build and install {0}'.format(name),
356  self.cmake_configure(name, cmake_path) + self.make_and_install()
357  )
358 
359  def fb_github_autoconf_install(self, project_and_path, github_org='facebook'):
360  return [
361  self.fb_github_project_workdir(project_and_path, github_org),
362  self.autoconf_install(project_and_path),
363  ]
364 
365  def fb_github_cmake_install(self, project_and_path, cmake_path='..', github_org='facebook'):
366  return [
367  self.fb_github_project_workdir(project_and_path, github_org),
368  self.cmake_install(project_and_path, cmake_path),
369  ]
def fb_github_project_workdir(self, project_and_path, github_org='facebook')
void * object
Definition: AtFork.cpp:32
def __init__(self, kwargs)
def add_option(self, name, value)
def fb_github_cmake_install(self, project_and_path, cmake_path='..', github_org='facebook')
def run(self, shell_cmd)
def configure(self, name=None)
def copy_local_repo(self, dir, dest_name)
def github_project_workdir(self, project, path)
def cmake_install(self, name, cmake_path='..')
def _make_vars(self, make_vars)
def step(self, name, actions)
#define join
def path_join(args)
Formatter< false, Args... > format(StringPiece fmt, Args &&...args)
Definition: Format.h:271
def fb_github_autoconf_install(self, project_and_path, github_org='facebook')
def _read_project_github_hashes()
def autoconf_install(self, name)
def option(self, name, default=None)
def make_and_install(self, make_vars=None)
Definition: Traits.h:592
def cmake_configure(self, name, cmake_path='..')
def parallel_make(self, make_vars=None)
def shell_join(delim, it)