/* Copyright 2025 Google LLC. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ #ifndef THIRD_PARTY_ODML_LITERT_TENSOR_BUFFER_H_ #define THIRD_PARTY_ODML_LITERT_TENSOR_BUFFER_H_ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "absl/status/status.h" // from @com_google_absl #include "absl/status/statusor.h" // from @com_google_absl #include "absl/types/span.h" // from @com_google_absl #include "tensor/datatypes.h" #include "tensor/internal/type_id.h" namespace litert::tensor { static constexpr size_t kCpuBufferAlignment = 64; // Provides access to data stored in a buffer. // // Unlocks the buffer when destroyed. template class LockedBufferSpan { public: using value_type = T; using pointer = T*; using const_pointer = const T*; using reference = T&; using const_reference = const T&; using iterator = T*; using const_iterator = const T*; template LockedBufferSpan(T* data, Unlock&& unlock, size_t count) : data_(data, std::forward(unlock)), bytes_(count * sizeof(T)) {} template LockedBufferSpan(std::unique_ptr data, size_t count) : data_(data.release(), data.get_deleter()), bytes_(count * sizeof(T)) {} static LockedBufferSpan Empty() { return LockedBufferSpan(nullptr, [](T*) {}, 0); } // Casts the span to a specific type. template [[nodiscard]] LockedBufferSpan As() const { static_assert( std::is_const_v || !std::is_const_v, "Cannot cast from a constant buffer span to a non constant one."); return LockedBufferSpan(data_, reinterpret_cast(data_.get()), bytes_ / sizeof(U)); } [[nodiscard]] LockedBufferSpan SubSpan(size_t offset, size_t count = SIZE_MAX) const { if (offset >= size()) { return Empty(); } const size_t sub_count = std::min(count, size() - offset); return LockedBufferSpan(data_, data_.get() + offset, sub_count); } T* data() const& { return data_.get(); } size_t size() const { return bytes_ / sizeof(T); } T* begin() & { return data(); } T* end() & { return data() + size(); } const T* begin() const& { return data(); } const T* end() const& { return data() + size(); } const T* cbegin() const& { return data(); } const T* cend() const& { return data() + size(); } // We delete these overloads to prevent users from keeping dangling references // to data by using one-liners that lock a buffer and get the data at once. T* data() const&& = delete; const T* begin() const&& = delete; const T* end() const&& = delete; const T* cbegin() const&& = delete; const T* cend() const&& = delete; private: template friend class LockedBufferSpan; template LockedBufferSpan(const std::shared_ptr& data, T* ptr, size_t count) : data_(data, ptr), bytes_(count * sizeof(T)) {} std::shared_ptr data_; size_t bytes_ = 0; }; // The main interface for buffers. class Buffer { public: virtual ~Buffer() = default; // Returns the type identifier of the buffer. // // This should generally be implemented as: // // ```cpp // internal::TypeId GetTypeId() const override { // return internal::TypeId::Get(); // } // ``` virtual internal::TypeId GetTypeId() const = 0; // Returns true if the buffer concrete type corresponds to the given type // identifier. // // This should generally be implemented as: // // ```cpp // bool IsA(internal::TypeId id) const override { // return id == internal::TypeId::Get() || // BASE_CLASS::IsA(id); // } // ``` virtual bool IsA(internal::TypeId id) const = 0; // Returns the size of the buffer in bytes. // // This is the size of the span returned by `Lock()` and `LockMutable()`. // // Note: implementations should be able to answer this **without locking**. // The size is metadata that is expected to be available without making the // data accessible from the CPU, which may be costly for non-CPU buffers. virtual absl::StatusOr ByteSize() const = 0; // Locks the buffer so that it's accessible from the CPU and returns an RAII // object that allows reading the data. // // Warning: the RAII object may not manage the data lifetime, only whether // it's accessible from the cpu or not. virtual LockedBufferSpan Lock() = 0; // Locks the buffer so that it's accessible from the CPU and returns an RAII // object that allows reading and writing the data. // // On error, returns an empty span. virtual LockedBufferSpan LockMutable() { return LockedBufferSpan::Empty(); } // Tries to cast the buffer to the given type T. template absl::StatusOr As() { if (IsA(internal::TypeId::Get())) { return *static_cast(this); } return absl::InvalidArgumentError("Wrong buffer type."); } // Tries to cast the buffer to the given type T. template absl::StatusOr As() const { if (IsA(internal::TypeId::Get())) { return *static_cast(this); } return absl::InvalidArgumentError("Wrong buffer type."); } }; // Provides a view to constant data. class SpanCpuBuffer : public Buffer { public: SpanCpuBuffer() = default; ~SpanCpuBuffer() override = default; // Creates a viewing buffer holding `data` of size `bytes`. SpanCpuBuffer(const std::byte* data, size_t bytes) : bytes_(bytes), data_(const_cast(data)) {} // Creates a viewing buffer from a vector. template explicit SpanCpuBuffer(const std::vector& vec) : SpanCpuBuffer(reinterpret_cast(vec.data()), sizeof(T) * vec.size()) {} // Creates a viewing buffer from a C++ array. template explicit SpanCpuBuffer(const std::array& array) : SpanCpuBuffer(reinterpret_cast(array.data()), sizeof(T) * N) {} // Creates a viewing buffer from a C array. template explicit SpanCpuBuffer(const T (&arr)[N]) : SpanCpuBuffer(reinterpret_cast(arr), sizeof(arr)) {} // Returns the type id for this class. static internal::TypeId TypeId() { return internal::TypeId::Get(); } // Returns the type id for this instance. internal::TypeId GetTypeId() const override { return TypeId(); } // Checks if this instance is of the given type id. bool IsA(internal::TypeId id) const override { return id == TypeId(); } // Returns the size of the buffer in bytes. absl::StatusOr ByteSize() const override { return size(); } // Locks the buffer so that it's accessible from the CPU and returns an RAII // object that allows reading the data. // // Note: For CPU buffers, (un)locking is a no-op. LockedBufferSpan Lock() override { return {data_, [](const std::byte*) {}, bytes_}; } // Because we know that a CPU buffer is always accessible, we can provide // direct accessors to the underlying data. const std::byte* data() const { return data_; } const std::byte* begin() const { return data_; } const std::byte* cbegin() const { return data_; } const std::byte* end() const { return data_ + bytes_; } const std::byte* cend() const { return data_ + bytes_; } size_t size() const { return bytes_; } template absl::Span Span() const { static_assert(std::is_const_v, "SpanCpuBuffer data is not mutable."); return absl::Span(reinterpret_cast(data_), bytes_ / sizeof(T)); } protected: size_t bytes_ = 0; std::byte* data_ = nullptr; }; // Provides a view to mutable data. class MutableSpanCpuBuffer : public SpanCpuBuffer { public: MutableSpanCpuBuffer() = default; ~MutableSpanCpuBuffer() override = default; // Creates a viewing buffer holding `data` of size `bytes`. MutableSpanCpuBuffer(std::byte* data, size_t bytes) : SpanCpuBuffer(data, bytes) {} // Creates a viewing buffer from a C array. template explicit MutableSpanCpuBuffer(T (&arr)[N]) : SpanCpuBuffer(arr) {} // Don't create a mutable span buffer over constant data. template explicit MutableSpanCpuBuffer(const T (&arr)[N]) = delete; // Returns the type id for this class. static internal::TypeId TypeId() { return internal::TypeId::Get(); } // Returns the type id for this instance. internal::TypeId GetTypeId() const override { return TypeId(); } // Checks if this instance is of the given type id. bool IsA(internal::TypeId id) const override { return id == TypeId() || SpanCpuBuffer::IsA(id); } // Locks the buffer so that it's accessible from the CPU and returns an RAII // object that allows reading the data. // // Note: For CPU buffers, (un)locking is a no-op. LockedBufferSpan LockMutable() override { return {data_, [](std::byte*) {}, bytes_}; } // Because we know that a CPU buffer is always accessible, we can provide // direct accessors to the underlying data. std::byte* data() const { return data_; } std::byte* begin() const { return data_; } std::byte* end() const { return data_ + bytes_; } template absl::Span Span() const { return absl::Span(reinterpret_cast(data_), bytes_ / sizeof(T)); } }; namespace internal { // Detects sequences that store their elements contiguously, i.e. the ones that // can be viewed through an `absl::Span`. template struct IsContiguousSequence : std::false_type {}; template struct IsContiguousSequence< Sequence, std::void_t()))>> : std::true_type {}; // Detects the span type that contiguous sequences are narrowed down to before // the run time type dispatch happens. template struct IsConstAbslSpan : std::false_type {}; template struct IsConstAbslSpan> : std::true_type {}; } // namespace internal // Manages tensor data. class OwningCpuBuffer : public Buffer { protected: static constexpr struct PassKey { } kPass{}; public: using CustomAllocPtr = std::unique_ptr; ~OwningCpuBuffer() override = default; // Builds an owning cpu buffer. // // Note: This is an internal constructor that is made public to allow building // smart pointers in the factory functions that are below by using the passkey // idiom. OwningCpuBuffer(PassKey, std::shared_ptr data, size_t bytes) : data_(std::move(data)), bytes_(bytes) {} // We want to avoid unintentional copies. OwningCpuBuffer(const OwningCpuBuffer&) = delete; OwningCpuBuffer& operator=(const OwningCpuBuffer&) = delete; OwningCpuBuffer(OwningCpuBuffer&& other) : data_(std::move(other.data_)), bytes_(std::exchange(other.bytes_, 0)) {} OwningCpuBuffer& operator=(OwningCpuBuffer&& other) { data_ = std::move(other.data_); bytes_ = std::exchange(other.bytes_, 0); return *this; } // Returns the type id for this class. static internal::TypeId TypeId() { return internal::TypeId::Get(); } // Returns the type id for this instance. internal::TypeId GetTypeId() const override { return TypeId(); } // Checks if this instance is of the given type id. bool IsA(internal::TypeId id) const override { return id == TypeId(); } // Returns the size of the buffer in bytes. absl::StatusOr ByteSize() const override { return size(); } // Locks the buffer so that it's accessible from the CPU and returns an RAII // object that allows reading the data. LockedBufferSpan Lock() override { return {data(), [d = data_](const std::byte*) mutable { d.reset(); }, size()}; } // Locks the buffer so that it's accessible from the CPU and returns an RAII // object that allows reading and writing the data. LockedBufferSpan LockMutable() override { return {data(), [d = data_](std::byte*) mutable { d.reset(); }, size()}; } template >> absl::Span Span() const { return absl::Span(reinterpret_cast(data()), size() / sizeof(T)); } template absl::Span Span() { return absl::Span(reinterpret_cast(data()), size() / sizeof(T)); } // Because we know that a CPU buffer is always accessible, we can provide // direct accessors to the underlying data. std::byte* data() { return data_.get(); } const std::byte* data() const { return data_.get(); } size_t size() const { return bytes_; } std::byte* begin() { return data_.get(); } std::byte* end() { return data_.get() + bytes_; } const std::byte* begin() const { return data_.get(); } const std::byte* end() const { return data_.get() + bytes_; } const std::byte* cbegin() const { return begin(); } const std::byte* cend() const { return end(); } // Transfers ownership of `data` to a new `OwningCpuBuffer`. static absl::StatusOr> Own( CustomAllocPtr data, size_t bytes); // Transfers ownership of `data` to a new `OwningCpuBuffer`. static absl::StatusOr> Own( std::shared_ptr data, size_t bytes); #define LITERT_TENSOR_BUFFER_OP_AS_CASE(OP, TYPE, ...) \ case Type::k##TYPE: \ return OP(__VA_ARGS__) #define LITERT_TENSOR_BUFFER_OP_AS_SWITCH(OP, ...) \ switch (type) { \ LITERT_TENSOR_BUFFER_OP_AS_CASE(OP, BOOL, __VA_ARGS__); \ LITERT_TENSOR_BUFFER_OP_AS_CASE(OP, I2, __VA_ARGS__); \ LITERT_TENSOR_BUFFER_OP_AS_CASE(OP, I4, __VA_ARGS__); \ LITERT_TENSOR_BUFFER_OP_AS_CASE(OP, I8, __VA_ARGS__); \ LITERT_TENSOR_BUFFER_OP_AS_CASE(OP, I16, __VA_ARGS__); \ LITERT_TENSOR_BUFFER_OP_AS_CASE(OP, I32, __VA_ARGS__); \ LITERT_TENSOR_BUFFER_OP_AS_CASE(OP, I64, __VA_ARGS__); \ LITERT_TENSOR_BUFFER_OP_AS_CASE(OP, U2, __VA_ARGS__); \ LITERT_TENSOR_BUFFER_OP_AS_CASE(OP, U4, __VA_ARGS__); \ LITERT_TENSOR_BUFFER_OP_AS_CASE(OP, U8, __VA_ARGS__); \ LITERT_TENSOR_BUFFER_OP_AS_CASE(OP, U16, __VA_ARGS__); \ LITERT_TENSOR_BUFFER_OP_AS_CASE(OP, U32, __VA_ARGS__); \ LITERT_TENSOR_BUFFER_OP_AS_CASE(OP, U64, __VA_ARGS__); \ LITERT_TENSOR_BUFFER_OP_AS_CASE(OP, FP16, __VA_ARGS__); \ LITERT_TENSOR_BUFFER_OP_AS_CASE(OP, FP32, __VA_ARGS__); \ LITERT_TENSOR_BUFFER_OP_AS_CASE(OP, FP64, __VA_ARGS__); \ LITERT_TENSOR_BUFFER_OP_AS_CASE(OP, BF16, __VA_ARGS__); \ case Type::kUnknown: \ return nullptr; \ } \ return nullptr; // Some compilers complain even if we handle all cases. // Allocates a buffer that can hold `count` elements. template static std::shared_ptr Allocate(size_t count) { const size_t bytes = NativeStorage::BufferSize(count); CustomAllocPtr copied_data = AlignedAlloc(bytes); return std::make_shared(kPass, std::move(copied_data), bytes); } template static std::shared_ptr Allocate(std::vector shape) { return Allocate( std::accumulate(shape.begin(), shape.end(), 1, std::multiplies<>())); } static std::shared_ptr AllocateAs(Type type, size_t count) { LITERT_TENSOR_BUFFER_OP_AS_SWITCH(Allocate, count); } static std::shared_ptr AllocateAs( Type type, std::vector shape) { LITERT_TENSOR_BUFFER_OP_AS_SWITCH(Allocate, std::move(shape)); } // Builds an `OwningCpuBuffer` and copy the given data to it. // // Note: This is not done with a constructor to force copies to be explicit. static std::shared_ptr Copy(const char* data, size_t bytes); // Builds an `OwningCpuBuffer` by copying the elements in the given sequence. // // - `type`: The underlying storage type. template static std::shared_ptr Copy(Sequence&& seq) { using std::begin; using std::end; using std::size; using NativeType = typename NativeStorage::type; constexpr Type seq_type = ApiType>::value; const size_t num_elements = size(seq) * NativeStorage::kNumElements; std::shared_ptr copied_data = Allocate(num_elements); if constexpr (type == seq_type) { std::copy(begin(seq), end(seq), copied_data->Span().data()); } else { Convert(seq, copied_data->Span()).IgnoreError(); } return copied_data; } // Builds an `OwningCpuBuffer` by copying the elements of the given // initializer list. template static std::shared_ptr Copy(std::initializer_list&& seq) { return OwningCpuBuffer::Copy(seq); } // Builds an `OwningCpuBuffer` by copying the elements in the given sequence // with run time type dispatching. template static std::shared_ptr CopyAs(Type type, Sequence&& seq) { using DecayedSequence = std::decay_t; if constexpr (internal::IsContiguousSequence::value && !internal::IsConstAbslSpan::value) { // Narrowing to a span lets contiguous containers share a single dispatch // table per element type instead of instantiating one per container type. return CopyAs(type, absl::MakeConstSpan(seq)); } else { LITERT_TENSOR_BUFFER_OP_AS_SWITCH(Copy, std::forward(seq)); } } // Builds an `OwningCpuBuffer` by copying the elements of the given // initializer list with run time dispatching. template static std::shared_ptr CopyAs(Type type, std::initializer_list seq) { return CopyAs(type, absl::Span(seq.begin(), seq.size())); } // Builds an `OwningCpuBuffer` by applying the given `transform` to elements // of a sequence. template static std::shared_ptr Transform(Sequence&& seq, F&& transform) { using NativeType = typename NativeStorage::type; using std::begin; using std::end; using std::size; constexpr Type seq_type = ApiType>::value; const size_t num_elements = size(seq) * NativeStorage::kNumElements; std::shared_ptr copied_data = Allocate(num_elements); std::transform(begin(seq), end(seq), copied_data->Span().data(), std::forward(transform)); return copied_data; } // Builds an `OwningCpuBuffer` by applying the given `transform` to elements // of a sequence with run time type dispatching. template static std::shared_ptr TransformAs(Type type, Sequence&& seq, F&& transform) { LITERT_TENSOR_BUFFER_OP_AS_SWITCH(Transform, std::forward(seq), std::forward(transform)); } // Allocates an array with an alignment of `kBufferAlignment`. // // Note: This is different from `std::aligned_alloc` because it doesn't // require the array size to be a multiple of the alignment. // // To do so we allocate a bigger buffer than requested, get the first // aligned address in it and prepend the offset to the real allocation. // // ``` // [data ][off][aligned_data... ] // ↑ // This pointer is returned. // ``` static CustomAllocPtr AlignedAlloc(size_t bytes); // Frees an array allocated with `AlignedAlloc`. static void AlignedFree(std::byte* ptr); // Returns true if the given pointer is aligned to `kBufferAlignment`. static bool IsAligned(const void* ptr) { return !(reinterpret_cast(ptr) % kCpuBufferAlignment); } protected: // We use a shared_ptr to be able to lock the buffer and keep the data alive. std::shared_ptr data_; size_t bytes_; }; } // namespace litert::tensor #endif // THIRD_PARTY_ODML_LITERT_TENSOR_BUFFER_H_