AsyncFileWriter.h

Go to the documentation of this file.

/*
 * Copyright 2017-present Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#pragma once

#include <condition_variable>
#include <mutex>
#include <thread>

#include <folly/File.h>
#include <folly/Range.h>
#include <folly/Synchronized.h>
#include <folly/logging/LogWriter.h>

namespace folly {

class AsyncFileWriter : public LogWriter {
 public:
  static constexpr size_t kDefaultMaxBufferSize = 1024 * 1024;

  explicit AsyncFileWriter(folly::StringPiece path);

  explicit AsyncFileWriter(folly::File&& file);

  ~AsyncFileWriter();

  void writeMessage(folly::StringPiece buffer, uint32_t flags = 0) override;
  void writeMessage(std::string&& buffer, uint32_t flags = 0) override;

  void flush() override;

  bool ttyOutput() const override;

  void setMaxBufferSize(size_t size);

  size_t getMaxBufferSize() const;

  const folly::File& getFile() const {
    return file_;
  }

 private:
  enum Flags : uint32_t {
    // FLAG_IO_THREAD_STARTED indicates that the constructor has started the
    // I/O thread.
    FLAG_IO_THREAD_STARTED = 0x01,
    // FLAG_DESTROYING indicates that the destructor is running and destroying
    // the I/O thread.
    FLAG_DESTROYING = 0x02,
    // FLAG_STOP indicates that the I/O thread has been asked to stop.
    // This is set either by the destructor or by preFork()
    FLAG_STOP = 0x04,
    // FLAG_IO_THREAD_STOPPED indicates that the I/O thread is about to return
    // and can now be joined.  ioCV_ will be signalled when this flag is set.
    FLAG_IO_THREAD_STOPPED = 0x08,
    // FLAG_IO_THREAD_JOINED indicates that the I/O thread has been joined.
    FLAG_IO_THREAD_JOINED = 0x10,
  };

  /*
   * A simple implementation using two queues.
   * All writer threads enqueue into one queue while the I/O thread is
   * processing the other.
   *
   * We could potentially also provide an implementation using folly::MPMCQueue
   * in the future, which may improve contention under very high write loads.
   */
  struct Data {
    std::array<std::vector<std::string>, 2> queues;
    uint32_t flags{0};
    uint64_t ioThreadCounter{0};
    size_t maxBufferBytes{kDefaultMaxBufferSize};
    size_t currentBufferSize{0};
    size_t numDiscarded{0};
    std::thread ioThread;

    std::vector<std::string>* getCurrentQueue() {
      return &queues[ioThreadCounter & 0x1];
    }
  };

  void ioThread();
  void performIO(std::vector<std::string>* ioQueue, size_t numDiscarded);

  void onIoError(const std::exception& ex);
  std::string getNumDiscardedMsg(size_t numDiscarded);

  bool preFork();
  void postForkParent();
  void postForkChild();
  void stopIoThread(
      folly::Synchronized<Data, std::mutex>::LockedPtr& data,
      uint32_t extraFlags);
  void restartThread();

  folly::File file_;

  folly::Synchronized<Data, std::mutex> data_;
  std::condition_variable messageReady_;
  std::condition_variable ioCV_;

  folly::Synchronized<Data, std::mutex>::LockedPtr lockedData_;
};
} // namespace folly