#include "tokenizer.hpp"
namespace pk {
// U+2581 LOWER ONE EIGHTH BLOCK — SentencePiece meta-space marker.
// UTF-8 encoding: 0xE2 0x96 0x81  (3 bytes)
static const char META_SPACE[] = "\xe2\x96\x81";
static const size_t META_SPACE_LEN = 3;

std::string detokenize(const std::vector<std::string>& pieces,
                       const std::vector<int32_t>& ids) {
    // Step 1: concatenate the piece strings for each id.
    std::string result;
    result.reserve(ids.size() * 4);
    for (int32_t id : ids) {
        if (id >= 0 && (size_t)id < pieces.size()) {
            result += pieces[(size_t)id];
        }
    }

    // Step 2: replace every occurrence of META_SPACE (▁) with a regular space.
    std::string out;
    out.reserve(result.size());
    for (size_t i = 0; i < result.size(); ) {
        // Check for the 3-byte UTF-8 sequence 0xE2 0x96 0x81
        if (i + META_SPACE_LEN <= result.size() &&
            (unsigned char)result[i]   == 0xE2 &&
            (unsigned char)result[i+1] == 0x96 &&
            (unsigned char)result[i+2] == 0x81) {
            out += ' ';
            i += META_SPACE_LEN;
        } else {
            out += result[i++];
        }
    }

    // Step 3: strip a single leading space (SentencePiece decode_ids behavior).
    if (!out.empty() && out[0] == ' ') {
        out.erase(0, 1);
    }
    return out;
}
} // namespace pk