Subprocess.h

Go to the documentation of this file.

/*
 * Copyright 2012-present Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#pragma once

#include <signal.h>
#include <sys/types.h>

#if __APPLE__
#include <sys/wait.h>
#else
#include <wait.h>
#endif

#include <exception>
#include <string>
#include <vector>

#include <boost/container/flat_map.hpp>
#include <boost/operators.hpp>

#include <folly/Exception.h>
#include <folly/File.h>
#include <folly/FileUtil.h>
#include <folly/Function.h>
#include <folly/MapUtil.h>
#include <folly/Optional.h>
#include <folly/Portability.h>
#include <folly/Range.h>
#include <folly/gen/String.h>
#include <folly/io/IOBufQueue.h>
#include <folly/portability/SysResource.h>

namespace folly {

class Subprocess;
class ProcessReturnCode {
 public:
  enum State {
    // Subprocess starts in the constructor, so this state designates only
    // default-initialized or moved-out ProcessReturnCodes.
    NOT_STARTED,
    RUNNING,
    EXITED,
    KILLED,
  };

  static ProcessReturnCode makeNotStarted() {
    return ProcessReturnCode(RV_NOT_STARTED);
  }

  static ProcessReturnCode makeRunning() {
    return ProcessReturnCode(RV_RUNNING);
  }

  static ProcessReturnCode make(int status);

  // Default-initialized for convenience. Subprocess::returnCode() will
  // never produce this value.
  ProcessReturnCode() : rawStatus_(RV_NOT_STARTED) {}

  // Trivially copyable
  ProcessReturnCode(const ProcessReturnCode& p) = default;
  ProcessReturnCode& operator=(const ProcessReturnCode& p) = default;
  // Non-default move: In order for Subprocess to be movable, the "moved
  // out" state must not be "running", or ~Subprocess() will abort.
  ProcessReturnCode(ProcessReturnCode&& p) noexcept;
  ProcessReturnCode& operator=(ProcessReturnCode&& p) noexcept;

  State state() const;

  bool notStarted() const {
    return state() == NOT_STARTED;
  }
  bool running() const {
    return state() == RUNNING;
  }
  bool exited() const {
    return state() == EXITED;
  }
  bool killed() const {
    return state() == KILLED;
  }

  int exitStatus() const;

  int killSignal() const;

  bool coreDumped() const;

  std::string str() const;

  void enforce(State state) const;

 private:
  explicit ProcessReturnCode(int rv) : rawStatus_(rv) {}
  static constexpr int RV_NOT_STARTED = -2;
  static constexpr int RV_RUNNING = -1;

  int rawStatus_;
};

class FOLLY_EXPORT SubprocessError : public std::runtime_error {
 public:
  using std::runtime_error::runtime_error;
};

class FOLLY_EXPORT CalledProcessError : public SubprocessError {
 public:
  explicit CalledProcessError(ProcessReturnCode rc);
  ~CalledProcessError() throw() override = default;
  ProcessReturnCode returnCode() const {
    return returnCode_;
  }

 private:
  ProcessReturnCode returnCode_;
};

class FOLLY_EXPORT SubprocessSpawnError : public SubprocessError {
 public:
  SubprocessSpawnError(const char* executable, int errCode, int errnoValue);
  ~SubprocessSpawnError() throw() override = default;
  int errnoValue() const {
    return errnoValue_;
  }

 private:
  int errnoValue_;
};

class Subprocess {
 public:
  static const int CLOSE = -1;
  static const int PIPE = -2;
  static const int PIPE_IN = -3;
  static const int PIPE_OUT = -4;

  struct DangerousPostForkPreExecCallback {
    virtual ~DangerousPostForkPreExecCallback() {}
    // This must return 0 on success, or an `errno` error code.
    virtual int operator()() = 0;
  };

  class Options {
    friend class Subprocess;

   public:
    Options() {} // E.g. https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58328

    Options& fd(int fd, int action);

    Options& stdinFd(int action) {
      return fd(STDIN_FILENO, action);
    }

    Options& stdoutFd(int action) {
      return fd(STDOUT_FILENO, action);
    }

    Options& stderrFd(int action) {
      return fd(STDERR_FILENO, action);
    }

    Options& pipeStdin() {
      return fd(STDIN_FILENO, PIPE_IN);
    }
    Options& pipeStdout() {
      return fd(STDOUT_FILENO, PIPE_OUT);
    }
    Options& pipeStderr() {
      return fd(STDERR_FILENO, PIPE_OUT);
    }

    Options& closeOtherFds() {
      closeOtherFds_ = true;
      return *this;
    }

    Options& usePath() {
      usePath_ = true;
      return *this;
    }

    Options& chdir(const std::string& dir) {
      childDir_ = dir;
      return *this;
    }

#if __linux__

    Options& parentDeathSignal(int sig) {
      parentDeathSignal_ = sig;
      return *this;
    }
#endif

    Options& processGroupLeader() {
      processGroupLeader_ = true;
      return *this;
    }

    Options& detach() {
      detach_ = true;
      return *this;
    }

    Options& dangerousPostForkPreExecCallback(
        DangerousPostForkPreExecCallback* cob) {
      dangerousPostForkPreExecCallback_ = cob;
      return *this;
    }

#if __linux__

    using clone_flags_t = uint64_t;
    Options& useCloneWithFlags(clone_flags_t cloneFlags) noexcept {
      cloneFlags_ = cloneFlags;
      return *this;
    }
#endif

   private:
    typedef boost::container::flat_map<int, int> FdMap;
    FdMap fdActions_;
    bool closeOtherFds_{false};
    bool usePath_{false};
    bool processGroupLeader_{false};
    bool detach_{false};
    std::string childDir_; // "" keeps the parent's working directory
#if __linux__
    int parentDeathSignal_{0};
#endif
    DangerousPostForkPreExecCallback* dangerousPostForkPreExecCallback_{
        nullptr};
#if __linux__
    // none means `vfork()` instead of a custom `clone()`
    // Optional<> is used because value of '0' means do clone without any flags.
    Optional<clone_flags_t> cloneFlags_;
#endif
  };

  // Non-copiable, but movable
  Subprocess(const Subprocess&) = delete;
  Subprocess& operator=(const Subprocess&) = delete;
  Subprocess(Subprocess&&) = default;
  Subprocess& operator=(Subprocess&&) = default;

  Subprocess();

  explicit Subprocess(
      const std::vector<std::string>& argv,
      const Options& options = Options(),
      const char* executable = nullptr,
      const std::vector<std::string>* env = nullptr);
  ~Subprocess();

  // clang-format off
  [[deprecated(
      "Prefer not running in a shell or use `shellify`.")]]
  explicit Subprocess(
      const std::string& cmd,
      const Options& options = Options(),
      const std::vector<std::string>* env = nullptr);
  // clang-format on


  pid_t pid() const;

  ProcessReturnCode returnCode() const {
    return returnCode_;
  }

  ProcessReturnCode poll(struct rusage* ru = nullptr);

  bool pollChecked();

  ProcessReturnCode wait();

  void waitChecked();

  void sendSignal(int signal);
  void terminate() {
    sendSignal(SIGTERM);
  }
  void kill() {
    sendSignal(SIGKILL);
  }


  std::pair<IOBufQueue, IOBufQueue> communicateIOBuf(
      IOBufQueue input = IOBufQueue());

  std::pair<std::string, std::string> communicate(
      StringPiece input = StringPiece());

  using FdCallback = folly::Function<bool(int, int)>;
  void communicate(FdCallback readCallback, FdCallback writeCallback);

  template <class Callback>
  class ReadLinesCallback {
   private:
    // Binds an FD to the client-provided FD+line callback
    struct StreamSplitterCallback {
      StreamSplitterCallback(Callback& cb, int fd) : cb_(cb), fd_(fd) {}
      // The return value semantics are inverted vs StreamSplitter
      bool operator()(StringPiece s) {
        return !cb_(fd_, s);
      }
      Callback& cb_;
      int fd_;
    };
    typedef gen::StreamSplitter<StreamSplitterCallback> LineSplitter;

   public:
    explicit ReadLinesCallback(
        Callback&& fdLineCb,
        uint64_t maxLineLength = 0, // No line length limit by default
        char delimiter = '\n',
        uint64_t bufSize = 1024)
        : fdLineCb_(std::forward<Callback>(fdLineCb)),
          maxLineLength_(maxLineLength),
          delimiter_(delimiter),
          bufSize_(bufSize) {}

    bool operator()(int pfd, int cfd) {
      // Make a splitter for this cfd if it doesn't already exist
      auto it = fdToSplitter_.find(cfd);
      auto& splitter = (it != fdToSplitter_.end())
          ? it->second
          : fdToSplitter_
                .emplace(
                    cfd,
                    LineSplitter(
                        delimiter_,
                        StreamSplitterCallback(fdLineCb_, cfd),
                        maxLineLength_))
                .first->second;
      // Read as much as we can from this FD
      char buf[bufSize_];
      while (true) {
        ssize_t ret = readNoInt(pfd, buf, bufSize_);
        if (ret == -1 && errno == EAGAIN) { // No more data for now
          return false;
        }
        checkUnixError(ret, "read");
        if (ret == 0) { // Reached end-of-file
          splitter.flush(); // Ignore return since the file is over anyway
          return true;
        }
        if (!splitter(StringPiece(buf, ret))) {
          return true; // The callback told us to stop
        }
      }
    }

   private:
    Callback fdLineCb_;
    const uint64_t maxLineLength_;
    const char delimiter_;
    const uint64_t bufSize_;
    // We lazily make splitters for all cfds that get used.
    std::unordered_map<int, LineSplitter> fdToSplitter_;
  };

  // Helper to enable template deduction
  template <class Callback>
  static auto readLinesCallback(
      Callback&& fdLineCb,
      uint64_t maxLineLength = 0, // No line length limit by default
      char delimiter = '\n',
      uint64_t bufSize = 1024)
      -> ReadLinesCallback<typename std::decay<Callback>::type> {
    return ReadLinesCallback<typename std::decay<Callback>::type>(
        std::forward<Callback>(fdLineCb), maxLineLength, delimiter, bufSize);
  }

  void enableNotifications(int childFd, bool enabled);

  bool notificationsEnabled(int childFd) const;


  void closeParentFd(int childFd);

  void setAllNonBlocking();

  int parentFd(int childFd) const {
    return pipes_[findByChildFd(childFd)].pipe.fd();
  }
  int stdinFd() const {
    return parentFd(0);
  }
  int stdoutFd() const {
    return parentFd(1);
  }
  int stderrFd() const {
    return parentFd(2);
  }

  struct ChildPipe {
    ChildPipe(int fd, folly::File&& ppe) : childFd(fd), pipe(std::move(ppe)) {}
    int childFd;
    folly::File pipe; // Owns the parent FD
  };
  std::vector<ChildPipe> takeOwnershipOfPipes();

 private:
  // spawn() sets up a pipe to read errors from the child,
  // then calls spawnInternal() to do the bulk of the work.  Once
  // spawnInternal() returns it reads the error pipe to see if the child
  // encountered any errors.
  void spawn(
      std::unique_ptr<const char*[]> argv,
      const char* executable,
      const Options& options,
      const std::vector<std::string>* env);
  void spawnInternal(
      std::unique_ptr<const char*[]> argv,
      const char* executable,
      Options& options,
      const std::vector<std::string>* env,
      int errFd);

  // Actions to run in child.
  // Note that this runs after vfork(), so tread lightly.
  // Returns 0 on success, or an errno value on failure.
  int prepareChild(
      const Options& options,
      const sigset_t* sigmask,
      const char* childDir) const;
  int runChild(
      const char* executable,
      char** argv,
      char** env,
      const Options& options) const;

  void readChildErrorPipe(int pfd, const char* executable);

  // Returns an index into pipes_. Throws std::invalid_argument if not found.
  size_t findByChildFd(const int childFd) const;

  pid_t pid_{-1};
  ProcessReturnCode returnCode_;

  struct Pipe : private boost::totally_ordered<Pipe> {
    folly::File pipe; // Our end of the pipe, wrapped in a File to auto-close.
    int childFd = -1; // Identifies the pipe: what FD is this in the child?
    int direction = PIPE_IN; // one of PIPE_IN / PIPE_OUT
    bool enabled = true; // Are notifications enabled in communicate()?

    bool operator<(const Pipe& other) const {
      return childFd < other.childFd;
    }
    bool operator==(const Pipe& other) const {
      return childFd == other.childFd;
    }
  };

  // Populated at process start according to fdActions, empty after
  // takeOwnershipOfPipes().  Sorted by childFd.  Can only have elements
  // erased, but not inserted, after being populated.
  //
  // The number of pipes between parent and child is assumed to be small,
  // so we're happy with a vector here, even if it means linear erase.
  std::vector<Pipe> pipes_;
};

} // namespace folly