String-inl.h

Go to the documentation of this file.

/*
 * Copyright 2014-present Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef FOLLY_GEN_STRING_H_
#error This file may only be included from folly/gen/String.h
#endif

#include <folly/Conv.h>
#include <folly/Portability.h>
#include <folly/String.h>

namespace folly {
namespace gen {
namespace detail {

inline size_t
splitPrefix(StringPiece& in, StringPiece& prefix, char delimiter) {
  size_t found = in.find(delimiter);
  if (found != StringPiece::npos) {
    ++found;
    prefix.assign(in.data(), in.data() + found);
    in.advance(found);
    return 1;
  }
  prefix.clear();
  return 0;
}

inline size_t
splitPrefix(StringPiece& in, StringPiece& prefix, StringPiece delimiter) {
  auto found = in.find(delimiter);
  if (found != StringPiece::npos) {
    found += delimiter.size();
    prefix.assign(in.data(), in.data() + found);
    in.advance(found);
    return delimiter.size();
  }
  prefix.clear();
  return 0;
}

inline size_t splitPrefix(StringPiece& in, StringPiece& prefix, MixedNewlines) {
  const auto kCRLF = "\r\n";
  const size_t kLenCRLF = 2;

  auto p = in.find_first_of(kCRLF);
  if (p != std::string::npos) {
    const auto in_start = in.data();
    size_t delim_len = 1;
    in.advance(p);
    // Either remove an MS-DOS CR-LF 2-byte newline, or eat 1 byte at a time.
    if (in.removePrefix(kCRLF)) {
      delim_len = kLenCRLF;
    } else {
      in.advance(delim_len);
    }
    prefix.assign(in_start, in.data());
    return delim_len;
  }
  prefix.clear();
  return 0;
}

inline const char* ch(const unsigned char* p) {
  return reinterpret_cast<const char*>(p);
}

// Chop s into pieces of at most maxLength, feed them to cb
template <class Callback>
bool consumeFixedSizeChunks(Callback& cb, StringPiece& s, uint64_t maxLength) {
  while (!s.empty()) {
    auto num_to_add = s.size();
    if (maxLength) {
      num_to_add = std::min<uint64_t>(num_to_add, maxLength);
    }
    if (!cb(StringPiece(s.begin(), num_to_add))) {
      return false;
    }
    s.advance(num_to_add);
  }
  return true;
}

// Consumes all of buffer, plus n chars from s.
template <class Callback>
bool consumeBufferPlus(Callback& cb, IOBuf& buf, StringPiece& s, uint64_t n) {
  buf.reserve(0, n);
  memcpy(buf.writableTail(), s.data(), n);
  buf.append(n);
  s.advance(n);
  if (!cb(StringPiece(detail::ch(buf.data()), buf.length()))) {
    return false;
  }
  buf.clear();
  return true;
}

} // namespace detail

template <class Callback>
bool StreamSplitter<Callback>::flush() {
  CHECK(maxLength_ == 0 || buffer_.length() < maxLength_);
  if (!pieceCb_(StringPiece(detail::ch(buffer_.data()), buffer_.length()))) {
    return false;
  }
  // We are ready to handle another stream now.
  buffer_.clear();
  return true;
}

template <class Callback>
bool StreamSplitter<Callback>::operator()(StringPiece in) {
  StringPiece prefix;
  // NB This code assumes a 1-byte delimiter. It's not too hard to support
  // multibyte delimiters, just remember that maxLength_ chunks can end up
  // falling in the middle of a delimiter.
  bool found = detail::splitPrefix(in, prefix, delimiter_);
  if (buffer_.length() != 0) {
    if (found) {
      uint64_t num_to_add = prefix.size();
      if (maxLength_) {
        CHECK(buffer_.length() < maxLength_);
        // Consume as much of prefix as possible without exceeding maxLength_
        num_to_add = std::min(maxLength_ - buffer_.length(), num_to_add);
      }

      // Append part of the prefix to the buffer, and send it to the callback
      if (!detail::consumeBufferPlus(pieceCb_, buffer_, prefix, num_to_add)) {
        return false;
      }

      if (!detail::consumeFixedSizeChunks(pieceCb_, prefix, maxLength_)) {
        return false;
      }

      found = detail::splitPrefix(in, prefix, delimiter_);
      // Post-conditions:
      //  - we consumed all of buffer_ and all of the first prefix.
      //  - found, in, and prefix reflect the second delimiter_ search
    } else if (maxLength_ && buffer_.length() + in.size() >= maxLength_) {
      // Send all of buffer_, plus a bit of in, to the callback
      if (!detail::consumeBufferPlus(
              pieceCb_, buffer_, in, maxLength_ - buffer_.length())) {
        return false;
      }
      // Post-conditions:
      //  - we consumed all of buffer, and the minimal # of bytes from in
      //  - found is false
    } // Otherwise: found is false & we cannot invoke the callback this turn
  }
  // Post-condition: buffer_ is nonempty only if found is false **and**
  // len(buffer + in) < maxLength_.

  // Send lines to callback directly from input (no buffer)
  while (found) { // Buffer guaranteed to be empty
    if (!detail::consumeFixedSizeChunks(pieceCb_, prefix, maxLength_)) {
      return false;
    }
    found = detail::splitPrefix(in, prefix, delimiter_);
  }

  // No more delimiters left; consume 'in' until it is shorter than maxLength_
  if (maxLength_) {
    while (in.size() >= maxLength_) { // Buffer is guaranteed to be empty
      if (!pieceCb_(StringPiece(in.begin(), maxLength_))) {
        return false;
      }
      in.advance(maxLength_);
    }
  }

  if (!in.empty()) { // Buffer may be nonempty
    // Incomplete line left, append to buffer
    buffer_.reserve(0, in.size());
    memcpy(buffer_.writableTail(), in.data(), in.size());
    buffer_.append(in.size());
  }
  CHECK(maxLength_ == 0 || buffer_.length() < maxLength_);
  return true;
}

namespace detail {

class StringResplitter : public Operator<StringResplitter> {
  char delimiter_;
  bool keepDelimiter_;

 public:
  explicit StringResplitter(char delimiter, bool keepDelimiter = false)
      : delimiter_(delimiter), keepDelimiter_(keepDelimiter) {}

  template <class Source>
  class Generator : public GenImpl<StringPiece, Generator<Source>> {
    Source source_;
    char delimiter_;
    bool keepDelimiter_;

   public:
    Generator(Source source, char delimiter, bool keepDelimiter)
        : source_(std::move(source)),
          delimiter_(delimiter),
          keepDelimiter_(keepDelimiter) {}

    template <class Body>
    bool apply(Body&& body) const {
      auto splitter =
          streamSplitter(this->delimiter_, [this, &body](StringPiece s) {
            // The stream ended with a delimiter; our contract is to swallow
            // the final empty piece.
            if (s.empty()) {
              return true;
            }
            if (s.back() != this->delimiter_) {
              return body(s);
            }
            if (!keepDelimiter_) {
              s.pop_back(); // Remove the 1-character delimiter
            }
            return body(s);
          });
      if (!source_.apply(splitter)) {
        return false;
      }
      return splitter.flush();
    }

    static constexpr bool infinite = Source::infinite;
  };

  template <class Source, class Value, class Gen = Generator<Source>>
  Gen compose(GenImpl<Value, Source>&& source) const {
    return Gen(std::move(source.self()), delimiter_, keepDelimiter_);
  }

  template <class Source, class Value, class Gen = Generator<Source>>
  Gen compose(const GenImpl<Value, Source>& source) const {
    return Gen(source.self(), delimiter_, keepDelimiter_);
  }
};

template <class DelimiterType = char>
class SplitStringSource
    : public GenImpl<StringPiece, SplitStringSource<DelimiterType>> {
  StringPiece source_;
  DelimiterType delimiter_;

 public:
  SplitStringSource(const StringPiece source, DelimiterType delimiter)
      : source_(source), delimiter_(std::move(delimiter)) {}

  template <class Body>
  bool apply(Body&& body) const {
    StringPiece rest(source_);
    StringPiece prefix;
    while (size_t delim_len = splitPrefix(rest, prefix, this->delimiter_)) {
      prefix.subtract(delim_len); // Remove the delimiter
      if (!body(prefix)) {
        return false;
      }
    }
    if (!rest.empty()) {
      if (!body(rest)) {
        return false;
      }
    }
    return true;
  }
};

template <class Delimiter, class Output>
class Unsplit : public Operator<Unsplit<Delimiter, Output>> {
  Delimiter delimiter_;

 public:
  explicit Unsplit(const Delimiter& delimiter) : delimiter_(delimiter) {}

  template <class Source, class Value>
  Output compose(const GenImpl<Value, Source>& source) const {
    Output outputBuffer;
    UnsplitBuffer<Delimiter, Output> unsplitter(delimiter_, &outputBuffer);
    unsplitter.compose(source);
    return outputBuffer;
  }
};

template <class Delimiter, class OutputBuffer>
class UnsplitBuffer : public Operator<UnsplitBuffer<Delimiter, OutputBuffer>> {
  Delimiter delimiter_;
  OutputBuffer* outputBuffer_;

 public:
  UnsplitBuffer(const Delimiter& delimiter, OutputBuffer* outputBuffer)
      : delimiter_(delimiter), outputBuffer_(outputBuffer) {
    CHECK(outputBuffer);
  }

  template <class Source, class Value>
  void compose(const GenImpl<Value, Source>& source) const {
    // If the output buffer is empty, we skip inserting the delimiter for the
    // first element.
    bool skipDelim = outputBuffer_->empty();
    source | [&](Value v) {
      if (skipDelim) {
        skipDelim = false;
        toAppend(std::forward<Value>(v), outputBuffer_);
      } else {
        toAppend(delimiter_, std::forward<Value>(v), outputBuffer_);
      }
    };
  }
};

template <class Target, class = void>
inline Target passthrough(Target target) {
  return target;
}

FOLLY_PUSH_WARNING
#ifdef __clang__
// Clang isn't happy with eatField() hack below.
#pragma GCC diagnostic ignored "-Wreturn-stack-address"
#endif // __clang__

template <class TargetContainer, class Delimiter, class... Targets>
class SplitTo {
  Delimiter delimiter_;

 public:
  explicit SplitTo(Delimiter delimiter) : delimiter_(delimiter) {}

  TargetContainer operator()(StringPiece line) const {
    int i = 0;
    StringPiece fields[sizeof...(Targets)];
    // HACK(tjackson): Used for referencing fields[] corresponding to variadic
    // template parameters.
    auto eatField = [&]() -> StringPiece& { return fields[i++]; };
    if (!split(
            delimiter_,
            line,
            detail::passthrough<StringPiece&, Targets>(eatField())...)) {
      throw std::runtime_error("field count mismatch");
    }
    i = 0;
    return TargetContainer(To<Targets>()(eatField())...);
  }
};

FOLLY_POP_WARNING

} // namespace detail

} // namespace gen
} // namespace folly