#include #include #include #include #include "common/common.h" #include "llama-context.h" #include "llama-vocab.h" #include "llama.h" #include "addonGlobals.h" #include "AddonModel.h" #include "AddonModelLora.h" #include "AddonGrammarEvaluationState.h" #include "AddonContext.h" static uint64_t calculateBatchMemorySize(int32_t n_tokens_alloc, int32_t embd, int32_t n_seq_max) { uint64_t totalSize = 0; if (embd) { totalSize += sizeof(float) * n_tokens_alloc * embd; } else { totalSize += sizeof(llama_token) * n_tokens_alloc; } totalSize += sizeof(llama_pos) * n_tokens_alloc; totalSize += sizeof(int32_t) * n_tokens_alloc; totalSize += sizeof(llama_seq_id *) * (n_tokens_alloc + 1); totalSize += sizeof(llama_seq_id) * n_seq_max * n_tokens_alloc; totalSize += sizeof(int8_t) * n_tokens_alloc; return totalSize; } static void addTokenToBatch(llama_batch& batch, llama_token token, llama_pos pos, llama_seq_id sequenceId, bool logits) { GGML_ASSERT(batch.seq_id[batch.n_tokens] && "llama_batch size exceeded"); const int32_t batchIndex = batch.n_tokens; batch.token[batchIndex] = token; batch.pos[batchIndex] = pos; batch.n_seq_id[batchIndex] = 1; batch.seq_id[batchIndex][0] = sequenceId; batch.logits[batchIndex] = logits; batch.n_tokens++; } class AddonContextDecodeBatchWorker : public Napi::AsyncWorker { public: AddonContext* ctx; AddonContextDecodeBatchWorker(const Napi::Env& env, AddonContext* ctx) : Napi::AsyncWorker(env, "AddonContextDecodeBatchWorker"), ctx(ctx), deferred(Napi::Promise::Deferred::New(env)) { ctx->Ref(); } ~AddonContextDecodeBatchWorker() { ctx->Unref(); } Napi::Promise GetPromise() { return deferred.Promise(); } protected: Napi::Promise::Deferred deferred; void Execute() { try { ctx->sharedSamplerData.gotLogit = false; ctx->sharedSamplerData.hasLogits = false; // Perform the evaluation using llama_decode. int r = llama_decode(ctx->ctx, ctx->batch); if (r != 0) { if (r == 1) { SetError("could not find a KV slot for the batch (try reducing the size of the batch or increase the context)"); } else { SetError("Eval has failed"); } return; } llama_synchronize(ctx->ctx); } catch (const std::exception& e) { SetError(e.what()); } catch(...) { SetError("Unknown error when calling \"llama_decode\""); } } void OnOK() { deferred.Resolve(Env().Undefined()); } void OnError(const Napi::Error& err) { deferred.Reject(err.Value()); } }; class AddonContextLoadContextWorker : public Napi::AsyncWorker { public: AddonContext* context; AddonContextLoadContextWorker(const Napi::Env& env, AddonContext* context) : Napi::AsyncWorker(env, "AddonContextLoadContextWorker"), context(context), deferred(Napi::Promise::Deferred::New(env)) { context->Ref(); } ~AddonContextLoadContextWorker() { context->Unref(); } Napi::Promise GetPromise() { return deferred.Promise(); } protected: Napi::Promise::Deferred deferred; void Execute() { try { context->ctx = llama_init_from_model(context->model->model, context->context_params); context->contextLoaded = context->ctx != nullptr; } catch (const std::exception& e) { SetError(e.what()); } catch(...) { SetError("Unknown error when calling \"llama_init_from_model\""); } } void OnOK() { if (context->contextLoaded && !context->model->model_params.no_alloc) { uint64_t contextMemorySize = llama_state_get_size(context->ctx); adjustNapiExternalMemoryAdd(context->Env(), contextMemorySize); context->loadedContextMemorySize = contextMemorySize; } deferred.Resolve(Napi::Boolean::New(Env(), context->contextLoaded)); } void OnError(const Napi::Error& err) { deferred.Reject(err.Value()); } }; class AddonContextUnloadContextWorker : public Napi::AsyncWorker { public: AddonContext* context; AddonContextUnloadContextWorker(const Napi::Env& env, AddonContext* context) : Napi::AsyncWorker(env, "AddonContextUnloadContextWorker"), context(context), deferred(Napi::Promise::Deferred::New(env)) { context->Ref(); } ~AddonContextUnloadContextWorker() { context->Unref(); } Napi::Promise GetPromise() { return deferred.Promise(); } protected: Napi::Promise::Deferred deferred; void Execute() { try { context->disposeMemory(); } catch (const std::exception& e) { SetError(e.what()); } catch(...) { SetError("Unknown error while disposing context memory"); } } void OnOK() { context->disposeMT(); deferred.Resolve(Env().Undefined()); } void OnError(const Napi::Error& err) { deferred.Reject(err.Value()); } }; class AddonContextSampleTokenWorker : public Napi::AsyncWorker { public: AddonContext* ctx; AddonSampler* sampler; bool arrayResult = false; bool returnConfidence = false; float tokenConfidence = -1; int32_t batchLogitIndex; llama_token result; bool no_output = false; struct TokenValuePair { llama_token token = LLAMA_TOKEN_NULL; float value = 0.00f; TokenValuePair() = default; TokenValuePair(llama_token token, float value) : token(token), value(value) {} }; bool returnProbabilities = false; struct ProbabilitiesMap { bool isSet = false; std::vector probs; } probabilities; struct ReturnLogits { enum ReturnLogitsEnabled { Disabled = 0, Enabled = 1, WithFilter = 2 } enabled = ReturnLogits::Disabled; std::vector filter; bool includeHighest = false; bool includeLowest = false; bool includeSelected = false; size_t includeTop = 0; } returnLogits; struct LogitsMap { bool isSet = false; std::vector logits; } logits; bool returnTotalLogitWeight = false; struct TotalLogitWeight { bool isSet = false; float value = 0.0f; } totalLogitWeight; AddonContextSampleTokenWorker(const Napi::CallbackInfo& info, AddonContext* ctx) : Napi::AsyncWorker(info.Env(), "AddonContextSampleTokenWorker"), ctx(ctx), deferred(Napi::Promise::Deferred::New(info.Env())) { ctx->Ref(); batchLogitIndex = info[0].As().Int32Value(); sampler = Napi::ObjectWrap::Unwrap(info[1].As()); arrayResult = info.Length() > 2 && info[2].IsBoolean(); returnProbabilities = arrayResult ? info[2].As().Value() : false; returnConfidence = arrayResult && info.Length() > 3 && info[3].IsBoolean() ? info[3].As().Value() : false; if (info.Length() > 4) { const auto option = info[4]; if (option.IsBoolean()) { returnLogits.enabled = option.As().Value() ? ReturnLogits::Enabled : ReturnLogits::Disabled; } else if (option.IsArray()) { const auto arr = option.As(); if (arr.Length() == 5) { const auto tokensOption = arr.Get(static_cast(0)); const auto includeHighestOption = arr.Get(static_cast(1)); const auto includeLowestOption = arr.Get(static_cast(2)); const auto includeSelectedOption = arr.Get(static_cast(3)); const auto includeTopOption = arr.Get(static_cast(4)); size_t logitsReserveSize = 0; if (includeHighestOption.IsBoolean()) { returnLogits.includeHighest = includeHighestOption.As().Value(); logitsReserveSize++; } if (includeLowestOption.IsBoolean()) { returnLogits.includeLowest = includeLowestOption.As().Value(); logitsReserveSize++; } if (includeSelectedOption.IsBoolean()) { returnLogits.includeSelected = includeSelectedOption.As().Value(); logitsReserveSize++; } if (includeTopOption.IsNumber()) { returnLogits.includeTop = includeTopOption.As().Uint32Value(); logitsReserveSize += returnLogits.includeTop; } if (tokensOption.IsArray()) { returnLogits.enabled = ReturnLogits::WithFilter; const auto tokensArr = tokensOption.As(); returnLogits.filter.clear(); returnLogits.filter.reserve(tokensArr.Length() + (returnLogits.includeSelected ? 1 : 0)); for (size_t i = 0; i < tokensArr.Length(); ++i) { const auto val = tokensArr.Get(i); if (!val.IsNumber()) { continue; } returnLogits.filter.push_back(val.As().Int32Value()); } logitsReserveSize = returnLogits.filter.size(); } this->logits.logits.reserve(logitsReserveSize); } } } returnTotalLogitWeight = info.Length() > 5 && info[5].As().Value(); sampler->Ref(); } ~AddonContextSampleTokenWorker() { ctx->Unref(); sampler->Unref(); } Napi::Promise GetPromise() { return deferred.Promise(); } protected: Napi::Promise::Deferred deferred; void Execute() { try { SampleToken(); } catch (const std::exception& e) { SetError(e.what()); } catch(...) { SetError("Unknown error when calling \"SampleToken\""); } } void SampleToken() { sampler->rebuildChainIfNeeded(); std::unique_lock samplingLock(ctx->sharedSamplerData.mutex); if (!ctx->sharedSamplerData.gotLogit) { ctx->sharedSamplerData.hasLogits = llama_get_logits(ctx->ctx) != nullptr; ctx->sharedSamplerData.gotLogit = true; } if (!ctx->sharedSamplerData.hasLogits) { SetError("This model does not support token generation"); return; } llama_token_data_array cur_p; sampler->sampleAndReleaseLock(ctx->sharedSamplerData.mutex, samplingLock, ctx->ctx, batchLogitIndex, cur_p, returnProbabilities || returnConfidence || returnLogits.enabled != ReturnLogits::Disabled); if (cur_p.size == 0 || !(cur_p.selected >= 0 && cur_p.selected < (int32_t)cur_p.size)) { no_output = true; return; } auto new_token_id = cur_p.data[cur_p.selected].id; if (returnProbabilities || returnConfidence || returnLogits.enabled == ReturnLogits::Enabled || returnLogits.includeTop != 0) { if (!cur_p.sorted) { std::sort(cur_p.data, cur_p.data + cur_p.size, [](const llama_token_data & a, const llama_token_data & b) { return a.logit > b.logit; }); cur_p.sorted = true; for (size_t i = 0; i < cur_p.size; i++) { if (cur_p.data[i].id == new_token_id) { cur_p.selected = i; break; } } } } if (returnLogits.includeTop != 0) { const auto topCount = std::min(returnLogits.includeTop, cur_p.size); for (int32_t i = 0; i < topCount; i++) { this->logits.logits.emplace_back(cur_p.data[i].id, cur_p.data[i].logit); } } if (returnLogits.enabled == ReturnLogits::WithFilter && returnLogits.includeSelected && new_token_id != LLAMA_TOKEN_NULL && std::find(returnLogits.filter.begin(), returnLogits.filter.end(), new_token_id) == returnLogits.filter.end() ) { returnLogits.filter.emplace_back(new_token_id); } float maxLogit = -INFINITY; bool maxLogitSet = false; if (returnProbabilities || returnLogits.enabled != ReturnLogits::Disabled) { std::vector logits; logits.resize(cur_p.size); maxLogit = cur_p.size > 0 ? cur_p.data[0].logit : -INFINITY; llama_token maxLogitToken = cur_p.size > 0 ? cur_p.data[0].id : LLAMA_TOKEN_NULL; maxLogitSet = true; float minLogit = cur_p.size > 0 ? cur_p.data[0].logit : INFINITY; llama_token minLogitToken = cur_p.size > 0 ? cur_p.data[0].id : LLAMA_TOKEN_NULL; for (size_t i = 0; i < cur_p.size; i++) { auto & logit = cur_p.data[i].logit; auto & token = cur_p.data[i].id; logits[i].token = token; logits[i].value = logit; if (logit > maxLogit) { maxLogit = logit; maxLogitToken = token; } if (logit < minLogit) { minLogit = logit; minLogitToken = token; } if (returnLogits.enabled == ReturnLogits::WithFilter && std::find(returnLogits.filter.begin(), returnLogits.filter.end(), token) != returnLogits.filter.end()) { this->logits.logits.emplace_back(token, logit); } } if (returnLogits.enabled == ReturnLogits::WithFilter) { if (returnLogits.includeHighest && maxLogitToken != LLAMA_TOKEN_NULL && std::any_of( this->logits.logits.begin(), this->logits.logits.end(), [maxLogitToken](const TokenValuePair& item) { return item.token == maxLogitToken; } ) ) { if (cur_p.sorted) { this->logits.logits.emplace(this->logits.logits.begin(), maxLogitToken, maxLogit); } else { this->logits.logits.emplace_back(maxLogitToken, maxLogit); } } if (returnLogits.includeLowest && minLogitToken != LLAMA_TOKEN_NULL && std::any_of( this->logits.logits.begin(), this->logits.logits.end(), [minLogitToken](const TokenValuePair& item) { return item.token == minLogitToken; } ) ) { this->logits.logits.emplace_back(minLogitToken, minLogit); } this->logits.isSet = true; if (!cur_p.sorted) { std::sort(this->logits.logits.begin(), this->logits.logits.end(), [](const TokenValuePair& a, const TokenValuePair& b) { return a.value > b.value; }); } } else if (returnProbabilities && returnLogits.enabled == ReturnLogits::Enabled) { this->logits.logits = logits; this->logits.isSet = true; } else if (!returnProbabilities && returnLogits.enabled == ReturnLogits::Enabled) { this->logits.logits.swap(logits); this->logits.isSet = true; } if (returnProbabilities && cur_p.size > 0 && maxLogit != -INFINITY) { float sum = 0.0f; for (size_t i = 0; i < cur_p.size; i++) { float prob = expf(logits[i].value - maxLogit); logits[i].value = prob; sum += prob; } for (size_t i = 0; i < cur_p.size; i++) { logits[i].value /= sum; } probabilities.probs.swap(logits); probabilities.isSet = true; } } if (returnConfidence || returnTotalLogitWeight) { if (!returnTotalLogitWeight && probabilities.isSet && cur_p.selected < probabilities.probs.size()) { tokenConfidence = probabilities.probs[cur_p.selected].value; } else { if (!maxLogitSet) { maxLogit = cur_p.data[0].logit; maxLogitSet = true; for (size_t i = 0; i < cur_p.size; i++) { auto logit = cur_p.data[i].logit; if (logit > maxLogit) { maxLogit = logit; } } } float sum = 0.0f; for (size_t i = 0; i < cur_p.size; i++) { sum += expf(cur_p.data[i].logit - maxLogit); } if (returnTotalLogitWeight) { totalLogitWeight.value = sum; totalLogitWeight.isSet = true; } tokenConfidence = expf(cur_p.data[cur_p.selected].logit - maxLogit) / sum; } } try { sampler->acceptToken(new_token_id); result = new_token_id; } catch (const std::exception& e) { SetError(std::string("Failed to accept token in sampler: ") + e.what()); } catch(...) { SetError("Unknown error when calling \"acceptToken\""); } } void OnOK() { Napi::Number resultToken; if (no_output) { resultToken = Napi::Number::New(Env(), -1); } else { resultToken = Napi::Number::New(Env(), static_cast(result)); } if (!arrayResult) { deferred.Resolve(resultToken); return; } Napi::Array resultArray = Napi::Array::New(Env(), 2); resultArray.Set(Napi::Number::New(Env(), 0), resultToken); if (this->probabilities.isSet) { const auto size = this->probabilities.probs.size(); Napi::Array probabilities = Napi::Array::New(Env(), size * 2); for (size_t i = 0; i < size; i++) { auto & prob = this->probabilities.probs[i]; probabilities.Set(i * 2, Napi::Number::New(Env(), prob.token)); probabilities.Set(i * 2 + 1, Napi::Number::New(Env(), prob.value)); } resultArray.Set(1, probabilities); } if (returnConfidence && tokenConfidence != -1) { resultArray.Set(2, Napi::Number::New(Env(), tokenConfidence)); } if (this->logits.isSet) { const auto size = this->logits.logits.size(); Napi::Array logits = Napi::Array::New(Env(), size * 2); for (size_t i = 0; i < size; i++) { auto& prob = this->logits.logits[i]; logits.Set(i * 2, Napi::Number::New(Env(), prob.token)); logits.Set(i * 2 + 1, Napi::Number::New(Env(), prob.value)); } resultArray.Set(3, logits); } if (totalLogitWeight.isSet) { resultArray.Set(4, Napi::Number::New(Env(), totalLogitWeight.value)); } deferred.Resolve(resultArray); } void OnError(const Napi::Error& err) { deferred.Reject(err.Value()); } }; AddonContext::AddonContext(const Napi::CallbackInfo& info) : Napi::ObjectWrap(info) { model = Napi::ObjectWrap::Unwrap(info[0].As()); model->Ref(); context_params = llama_context_default_params(); context_params.n_ctx = 4096; context_params.n_threads = std::max(common_cpu_get_num_math(), 1); context_params.n_threads_batch = context_params.n_threads; context_params.no_perf = true; context_params.swa_full = false; context_params.kv_unified = false; if (info.Length() > 1 && info[1].IsObject()) { const auto options = info[1].As(); if (options.Has("contextSize")) { context_params.n_ctx = options.Get("contextSize").As().Uint32Value(); } if (options.Has("batchSize")) { context_params.n_batch = options.Get("batchSize").As().Uint32Value(); context_params.n_ubatch = context_params.n_batch; // the batch queue is managed in the JS side, so there's no need for managing it on the C++ side } if (options.Has("sequences")) { context_params.n_seq_max = options.Get("sequences").As().Uint32Value(); } if (options.Has("embeddings")) { context_params.embeddings = options.Get("embeddings").As().Value(); } if (options.Has("ranking") && options.Get("ranking").As().Value()) { context_params.pooling_type = LLAMA_POOLING_TYPE_RANK; } if (options.Has("flashAttention")) { const auto flashAttention = options.Get("flashAttention"); if (flashAttention.IsString() && flashAttention.As().Utf8Value() == "auto") { context_params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_AUTO; } else { const bool flashAttentionEnabled = flashAttention.As().Value(); context_params.flash_attn_type = flashAttentionEnabled ? LLAMA_FLASH_ATTN_TYPE_ENABLED : LLAMA_FLASH_ATTN_TYPE_DISABLED; } } if (options.Has("threads")) { const auto threads = options.Get("threads").As().Int32Value(); const auto resolvedThreads = threads == 0 ? std::max((int32_t)std::thread::hardware_concurrency(), context_params.n_threads) : threads; context_params.n_threads = resolvedThreads; context_params.n_threads_batch = resolvedThreads; } if (options.Has("performanceTracking")) { context_params.no_perf = !(options.Get("performanceTracking").As().Value()); } if (options.Has("kvCacheKeyType") && options.Get("kvCacheKeyType").IsNumber()) { const auto keyType = options.Get("kvCacheKeyType").As().Int32Value(); if (keyType >= 0 && keyType < GGML_TYPE_COUNT) { context_params.type_k = static_cast(keyType); } } if (options.Has("kvCacheValueType") && options.Get("kvCacheValueType").IsNumber()) { const auto valueType = options.Get("kvCacheValueType").As().Int32Value(); if (valueType >= 0 && valueType < GGML_TYPE_COUNT) { context_params.type_v = static_cast(valueType); } } if (options.Has("swaFullCache")) { context_params.swa_full = options.Get("swaFullCache").As().Value(); } } } AddonContext::~AddonContext() { disposeMT(); } void AddonContext::disposeBatchMemory() { std::lock_guard lock(disposeMutex); if (!has_batch) { return; } llama_batch_free(batch); has_batch = false; batch_n_tokens = 0; } void AddonContext::disposeBatchMT() { uint64_t currentBatchMemorySize = 0; disposeBatchMemory(); { std::lock_guard lock(disposeMutex); currentBatchMemorySize = batchMemorySize; batchMemorySize = 0; } if (currentBatchMemorySize > 0) { adjustNapiExternalMemorySubtract(Env(), currentBatchMemorySize); } } void AddonContext::disposeMemory() { llama_context* currentCtx = nullptr; { std::lock_guard lock(disposeMutex); if (!memoryDisposed) { memoryDisposed = true; currentCtx = ctx; ctx = nullptr; contextLoaded = false; } } disposeBatchMemory(); if (currentCtx != nullptr) { llama_free(currentCtx); } } void AddonContext::disposeMT() { uint64_t currentLoadedContextMemorySize = 0; uint64_t currentBatchMemorySize = 0; bool shouldUnrefModel = false; disposeMemory(); { std::lock_guard lock(disposeMutex); if (disposed) { return; } disposed = true; shouldUnrefModel = true; if (!model->model_params.no_alloc) { currentLoadedContextMemorySize = loadedContextMemorySize; loadedContextMemorySize = 0; } currentBatchMemorySize = batchMemorySize; batchMemorySize = 0; } if (currentLoadedContextMemorySize > 0) { adjustNapiExternalMemorySubtract(Env(), currentLoadedContextMemorySize); } if (currentBatchMemorySize > 0) { adjustNapiExternalMemorySubtract(Env(), currentBatchMemorySize); } if (shouldUnrefModel) { model->Unref(); } } Napi::Value AddonContext::Dispose(const Napi::CallbackInfo& info) { if (disposed) { return info.Env().Undefined(); } if (contextLoaded || has_batch) { AddonContextUnloadContextWorker* worker = new AddonContextUnloadContextWorker(this->Env(), this); worker->Queue(); return worker->GetPromise(); } disposeMT(); Napi::Promise::Deferred deferred = Napi::Promise::Deferred::New(info.Env()); deferred.Resolve(info.Env().Undefined()); return deferred.Promise(); } Napi::Value AddonContext::Init(const Napi::CallbackInfo& info) { if (disposed) { Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException(); return info.Env().Undefined(); } else { AddonContextLoadContextWorker* worker = new AddonContextLoadContextWorker(this->Env(), this); worker->Queue(); return worker->GetPromise(); } } Napi::Value AddonContext::GetContextSize(const Napi::CallbackInfo& info) { if (disposed) { Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException(); return info.Env().Undefined(); } return Napi::Number::From(info.Env(), llama_n_ctx(ctx)); } Napi::Value AddonContext::InitBatch(const Napi::CallbackInfo& info) { if (disposed) { Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException(); return info.Env().Undefined(); } int32_t n_tokens = info[0].As().Int32Value(); if (n_tokens <= 0 || static_cast(n_tokens) > context_params.n_batch) { Napi::RangeError::New(info.Env(), "Invalid batch size").ThrowAsJavaScriptException(); return info.Env().Undefined(); } if (has_batch) { llama_batch_free(batch); } batch = llama_batch_init(n_tokens, 0, 1); has_batch = true; batch_n_tokens = n_tokens; uint64_t newBatchMemorySize = calculateBatchMemorySize(n_tokens, 0, 1); if (newBatchMemorySize > batchMemorySize) { adjustNapiExternalMemoryAdd(Env(), newBatchMemorySize - batchMemorySize); batchMemorySize = newBatchMemorySize; } else if (newBatchMemorySize < batchMemorySize) { adjustNapiExternalMemorySubtract(Env(), batchMemorySize - newBatchMemorySize); batchMemorySize = newBatchMemorySize; } return info.Env().Undefined(); } Napi::Value AddonContext::DisposeBatch(const Napi::CallbackInfo& info) { if (disposed) { Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException(); return info.Env().Undefined(); } disposeBatchMT(); return info.Env().Undefined(); } Napi::Value AddonContext::AddToBatch(const Napi::CallbackInfo& info) { if (!has_batch) { Napi::Error::New(info.Env(), "No batch is initialized").ThrowAsJavaScriptException(); return info.Env().Undefined(); } int32_t sequenceId = info[0].As().Int32Value(); int32_t firstTokenContextIndex = info[1].As().Int32Value(); Napi::Uint32Array tokens = info[2].As(); Napi::Uint32Array tokenLogitIndexes = info[3].As(); auto tokensLength = tokens.ElementLength(); auto tokenLogitIndexesLength = tokenLogitIndexes.ElementLength(); if (tokensLength > static_cast(batch_n_tokens - batch.n_tokens)) { Napi::RangeError::New(info.Env(), "Tokens exceed the initialized batch size").ThrowAsJavaScriptException(); return info.Env().Undefined(); } Napi::Uint32Array resLogitIndexes = Napi::Uint32Array::New(info.Env(), tokenLogitIndexesLength); for (size_t i = 0, l = 0; i < tokensLength; i++) { if (l < tokenLogitIndexesLength && tokenLogitIndexes[l] == i) { addTokenToBatch(batch, static_cast(tokens[i]), firstTokenContextIndex + i, sequenceId, true); resLogitIndexes[l] = batch.n_tokens - 1; l++; } else { addTokenToBatch(batch, static_cast(tokens[i]), firstTokenContextIndex + i, sequenceId, false); } } return resLogitIndexes; } Napi::Value AddonContext::DisposeSequence(const Napi::CallbackInfo& info) { if (disposed) { Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException(); return info.Env().Undefined(); } int32_t sequenceId = info[0].As().Int32Value(); bool result = llama_memory_seq_rm(llama_get_memory(ctx), sequenceId, -1, -1); if (!result) { Napi::Error::New(info.Env(), "Failed to dispose sequence").ThrowAsJavaScriptException(); return info.Env().Undefined(); } return info.Env().Undefined(); } Napi::Value AddonContext::RemoveTokenCellsFromSequence(const Napi::CallbackInfo& info) { if (disposed) { Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException(); return info.Env().Undefined(); } int32_t sequenceId = info[0].As().Int32Value(); int32_t startPos = info[1].As().Int32Value(); int32_t endPos = info[2].As().Int32Value(); bool result = llama_memory_seq_rm(llama_get_memory(ctx), sequenceId, startPos, endPos); return Napi::Boolean::New(info.Env(), result); } Napi::Value AddonContext::ShiftSequenceTokenCells(const Napi::CallbackInfo& info) { if (disposed) { Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException(); return info.Env().Undefined(); } int32_t sequenceId = info[0].As().Int32Value(); int32_t startPos = info[1].As().Int32Value(); int32_t endPos = info[2].As().Int32Value(); int32_t shiftDelta = info[3].As().Int32Value(); llama_memory_seq_add(llama_get_memory(ctx), sequenceId, startPos, endPos, shiftDelta); return info.Env().Undefined(); } Napi::Value AddonContext::GetSequenceKvCacheMinPosition(const Napi::CallbackInfo& info) { if (disposed) { Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException(); return info.Env().Undefined(); } int32_t sequenceId = info[0].As().Int32Value(); const auto minPosition = llama_memory_seq_pos_min(llama_get_memory(ctx), sequenceId); return Napi::Number::New(info.Env(), minPosition); } Napi::Value AddonContext::GetSequenceKvCacheMaxPosition(const Napi::CallbackInfo& info) { if (disposed) { Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException(); return info.Env().Undefined(); } int32_t sequenceId = info[0].As().Int32Value(); const auto maxPosition = llama_memory_seq_pos_max(llama_get_memory(ctx), sequenceId); return Napi::Number::New(info.Env(), maxPosition); } Napi::Value AddonContext::DecodeBatch(const Napi::CallbackInfo& info) { AddonContextDecodeBatchWorker* worker = new AddonContextDecodeBatchWorker(info.Env(), this); worker->Queue(); return worker->GetPromise(); } Napi::Value AddonContext::SampleToken(const Napi::CallbackInfo& info) { AddonContextSampleTokenWorker* worker = new AddonContextSampleTokenWorker(info, this); worker->Queue(); return worker->GetPromise(); } Napi::Value AddonContext::GetEmbedding(const Napi::CallbackInfo& info) { if (disposed) { Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException(); return info.Env().Undefined(); } int32_t inputTokensLength = info[0].As().Int32Value(); const double maxVectorSize = (info.Length() > 1 && info[1].IsNumber()) ? info[1].As().DoubleValue() : 0; if (inputTokensLength <= 0) { Napi::Error::New(info.Env(), "Invalid input tokens length").ThrowAsJavaScriptException(); return info.Env().Undefined(); } if (!std::isfinite(maxVectorSize) || maxVectorSize < 0 || std::floor(maxVectorSize) != maxVectorSize) { Napi::Error::New(info.Env(), "Invalid maximum embedding vector size").ThrowAsJavaScriptException(); return info.Env().Undefined(); } const enum llama_pooling_type pooling_type = llama_pooling_type(ctx); const int64_t n_embd = pooling_type == LLAMA_POOLING_TYPE_RANK ? static_cast(llama_model_n_cls_out(model->model)) : llama_model_n_embd_out(model->model); if (n_embd <= 0) { Napi::Error::New(info.Env(), "Invalid embedding vector size").ThrowAsJavaScriptException(); return info.Env().Undefined(); } const auto* embeddings = pooling_type == LLAMA_POOLING_TYPE_NONE ? NULL : llama_get_embeddings_seq(ctx, 0); if (embeddings == NULL) { embeddings = llama_get_embeddings_ith(ctx, -1); } if (embeddings == NULL) { Napi::Error::New(info.Env(), std::string("Failed to get embeddings for token ") + std::to_string(inputTokensLength - 1)).ThrowAsJavaScriptException(); return info.Env().Undefined(); } const size_t resultSize = maxVectorSize == 0 ? n_embd : std::min(n_embd, maxVectorSize); Napi::Float64Array result = Napi::Float64Array::New(info.Env(), resultSize); for (size_t i = 0; i < resultSize; i++) { result[i] = embeddings[i]; } return result; } Napi::Value AddonContext::GetStateSize(const Napi::CallbackInfo& info) { if (disposed) { Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException(); return info.Env().Undefined(); } return Napi::Number::From(info.Env(), llama_state_get_size(ctx)); } Napi::Value AddonContext::GetMemoryBreakdown(const Napi::CallbackInfo& info) { if (disposed) { Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException(); return info.Env().Undefined(); } if (!contextLoaded || ctx == nullptr) { Napi::Error::New(info.Env(), "Context is not loaded").ThrowAsJavaScriptException(); return info.Env().Undefined(); } std::size_t cpuRam = 0; std::size_t gpuVram = 0; for (const auto& [bufferType, memoryBreakdown] : ctx->memory_breakdown()) { const std::size_t size = memoryBreakdown.context + memoryBreakdown.compute; if (size == 0) { continue; } if (ggml_backend_buft_is_host(bufferType)) { cpuRam += size; } else { ggml_backend_dev_t device = ggml_backend_buft_get_device(bufferType); if (device != nullptr) { auto deviceType = ggml_backend_dev_type(device); if (deviceType == GGML_BACKEND_DEVICE_TYPE_GPU || deviceType == GGML_BACKEND_DEVICE_TYPE_IGPU) { gpuVram += size; } else { cpuRam += size; } } else { cpuRam += size; } } } Napi::Object result = Napi::Object::New(info.Env()); result.Set("cpuRam", Napi::Number::New(info.Env(), cpuRam)); result.Set("gpuVram", Napi::Number::New(info.Env(), gpuVram)); return result; } Napi::Value AddonContext::GetThreads(const Napi::CallbackInfo& info) { if (disposed) { Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException(); return info.Env().Undefined(); } return Napi::Number::From(info.Env(), llama_n_threads(ctx)); } Napi::Value AddonContext::SetThreads(const Napi::CallbackInfo& info) { if (disposed) { Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException(); return info.Env().Undefined(); } const auto threads = info[0].As().Int32Value(); const auto resolvedThreads = threads == 0 ? std::max((int32_t)std::thread::hardware_concurrency(), std::max(common_cpu_get_num_math(), 1)) : threads; if (llama_n_threads(ctx) != resolvedThreads) { llama_set_n_threads(ctx, resolvedThreads, resolvedThreads); } return info.Env().Undefined(); } class AddonContextSaveSequenceStateToFileWorker : public Napi::AsyncWorker { public: AddonContext* context; std::string filepath; llama_seq_id sequenceId; std::vector tokens; size_t savedFileSize = 0; AddonContextSaveSequenceStateToFileWorker(const Napi::CallbackInfo& info, AddonContext* context) : Napi::AsyncWorker(info.Env(), "AddonContextSaveSequenceStateToFileWorker"), context(context), deferred(Napi::Promise::Deferred::New(info.Env())) { context->Ref(); filepath = info[0].As().Utf8Value(); sequenceId = info[1].As().Int32Value(); Napi::Uint32Array inputTokens = info[2].As(); tokens.resize(inputTokens.ElementLength()); for (size_t i = 0; i < tokens.size(); i++) { tokens[i] = inputTokens[i]; } } ~AddonContextSaveSequenceStateToFileWorker() { context->Unref(); } Napi::Promise GetPromise() { return deferred.Promise(); } protected: Napi::Promise::Deferred deferred; void Execute() { try { savedFileSize = llama_state_seq_save_file(context->ctx, filepath.c_str(), sequenceId, tokens.data(), tokens.size()); if (savedFileSize == 0) { SetError("Failed to save state to file"); return; } } catch (const std::exception& e) { SetError(e.what()); } catch(...) { SetError("Unknown error when calling \"llama_state_seq_save_file\""); } } void OnOK() { deferred.Resolve(Napi::Number::New(Env(), savedFileSize)); } void OnError(const Napi::Error& err) { deferred.Reject(err.Value()); } }; Napi::Value AddonContext::SaveSequenceStateToFile(const Napi::CallbackInfo& info) { if (disposed) { Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException(); return info.Env().Undefined(); } AddonContextSaveSequenceStateToFileWorker* worker = new AddonContextSaveSequenceStateToFileWorker(info, this); worker->Queue(); return worker->GetPromise(); } class AddonContextLoadSequenceStateFromFileWorker : public Napi::AsyncWorker { public: AddonContext* context; std::string filepath; llama_seq_id sequenceId; size_t maxContextSize; std::vector tokens; AddonContextLoadSequenceStateFromFileWorker(const Napi::CallbackInfo& info, AddonContext* context) : Napi::AsyncWorker(info.Env(), "AddonContextLoadSequenceStateFromFileWorker"), context(context), deferred(Napi::Promise::Deferred::New(info.Env())) { context->Ref(); filepath = info[0].As().Utf8Value(); sequenceId = info[1].As().Int32Value(); maxContextSize = info[2].As().Uint32Value(); tokens.resize(maxContextSize); } ~AddonContextLoadSequenceStateFromFileWorker() { context->Unref(); } Napi::Promise GetPromise() { return deferred.Promise(); } protected: Napi::Promise::Deferred deferred; void Execute() { try { size_t tokenCount = 0; const size_t fileSize = llama_state_seq_load_file(context->ctx, filepath.c_str(), sequenceId, tokens.data(), tokens.size(), &tokenCount); if (fileSize == 0) { SetError("Failed to load state from file. Current context sequence size may be smaller that the state of the file"); return; } tokens.resize(tokenCount); } catch (const std::exception& e) { SetError(e.what()); } catch(...) { SetError("Unknown error when calling \"llama_state_seq_load_file\""); } } void OnOK() { size_t tokenCount = tokens.size(); Napi::Uint32Array result = Napi::Uint32Array::New(Env(), tokenCount); for (size_t i = 0; i < tokenCount; i++) { result[i] = tokens[i]; } deferred.Resolve(result); } void OnError(const Napi::Error& err) { deferred.Reject(err.Value()); } }; Napi::Value AddonContext::LoadSequenceStateFromFile(const Napi::CallbackInfo& info) { if (disposed) { Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException(); return info.Env().Undefined(); } AddonContextLoadSequenceStateFromFileWorker* worker = new AddonContextLoadSequenceStateFromFileWorker(info, this); worker->Queue(); return worker->GetPromise(); } Napi::Value AddonContext::PrintTimings(const Napi::CallbackInfo& info) { llama_perf_context_print(ctx); llama_perf_context_reset(ctx); return info.Env().Undefined(); } Napi::Value AddonContext::EnsureDraftContextIsCompatibleForSpeculative(const Napi::CallbackInfo& info) { constexpr auto vocabSizeMaxDifference = 128; // SPEC_VOCAB_MAX_SIZE_DIFFERENCE constexpr auto vocabCheckStartTokenId = 5; // SPEC_VOCAB_CHECK_START_TOKEN_ID const AddonContext * draftContext = Napi::ObjectWrap::Unwrap(info[0].As()); const auto currentCtx = ctx; const auto draftCtx = draftContext->ctx; const auto currentModel = model->model; const auto draftModel = draftContext->model->model; const auto currentVocab = model->vocab; const auto draftVocab = draftContext->model->vocab; if (llama_vocab_type(currentVocab) != llama_vocab_type(draftVocab)) { Napi::Error::New(info.Env(), "Speculative draft model vocabulary type must match the target model vocabulary type").ThrowAsJavaScriptException(); return info.Env().Undefined(); } if (llama_vocab_get_add_bos(currentVocab) != llama_vocab_get_add_bos(draftVocab) || llama_vocab_get_add_eos(currentVocab) != llama_vocab_get_add_eos(draftVocab) || llama_vocab_bos(currentVocab) != llama_vocab_bos(draftVocab) || llama_vocab_eos(currentVocab) != llama_vocab_eos(draftVocab) ) { Napi::Error::New(info.Env(), "Speculative draft model special tokens must match the target model special tokens").ThrowAsJavaScriptException(); return info.Env().Undefined(); } const int currentModelVocabSize = llama_vocab_n_tokens(currentVocab); const int draftModelVocabSize = llama_vocab_n_tokens(draftVocab); const int vocabDiff = std::abs(currentModelVocabSize - draftModelVocabSize); if (vocabDiff > vocabSizeMaxDifference) { Napi::Error::New( info.Env(), std::string("Speculative draft model vocabulary must closely match the target model vocabulary size (vocabulary size difference: ") + std::to_string(vocabDiff) + std::string(", max allowed: ") + std::to_string(vocabSizeMaxDifference) + std::string(")") ).ThrowAsJavaScriptException(); return info.Env().Undefined(); } const int minVocabSize = std::min(currentModelVocabSize, draftModelVocabSize); for (int i = vocabCheckStartTokenId; i < minVocabSize; ++i) { const char * currentTokenText = llama_vocab_get_text(currentVocab, i); const char * draftTokenText = llama_vocab_get_text(draftVocab, i); if (std::strcmp(currentTokenText, draftTokenText) != 0) { Napi::Error::New( info.Env(), std::string("Speculative draft model vocabulary must match the target model vocabulary, but token ") + std::to_string(i) + std::string(" content differs. Target: \"") + std::string(currentTokenText) + std::string("\", Draft: \"") + std::string(draftTokenText) + std::string("") ).ThrowAsJavaScriptException(); return info.Env().Undefined(); } } return info.Env().Undefined(); } Napi::Value AddonContext::SetLoras(const Napi::CallbackInfo& info) { Napi::Array loraArray = info[0].As(); Napi::Array scaleArray = info[1].As(); std::vector loras; std::vector scales; loras.reserve(loraArray.Length()); scales.reserve(scaleArray.Length()); for (size_t i = 0; i < loraArray.Length() && i < scaleArray.Length(); i++) { AddonModelLora* lora = Napi::ObjectWrap::Unwrap(loraArray.Get(i).As()); float scale = scaleArray.Get(i).As().FloatValue(); loras.push_back(lora->lora_adapter); scales.push_back(scale); } llama_set_adapters_lora(ctx, loras.data(), loras.size(), scales.data()); return info.Env().Undefined(); } class RestoreCheckpointWorker : public Napi::AsyncWorker { public: AddonContext* context; AddonContextSequenceCheckpoint* checkpoint; llama_pos maxPosIndex; llama_seq_id sequenceId; bool restoreSuccess = false; RestoreCheckpointWorker(const Napi::CallbackInfo& info, AddonContext* context, AddonContextSequenceCheckpoint* checkpoint, llama_pos maxPosIndex, llama_seq_id sequenceId) : Napi::AsyncWorker(info.Env(), "RestoreCheckpointWorker"), context(context), checkpoint(checkpoint), maxPosIndex(maxPosIndex), sequenceId(sequenceId), deferred(Napi::Promise::Deferred::New(info.Env())) { context->Ref(); checkpoint->Ref(); } ~RestoreCheckpointWorker() { context->Unref(); checkpoint->Unref(); } Napi::Promise GetPromise() { return deferred.Promise(); } protected: Napi::Promise::Deferred deferred; void Execute() { try { std::shared_lock lock(checkpoint->dataMutex); if (checkpoint->disposed) { SetError("Checkpoint is disposed"); return; } if (!checkpoint->initialized) { return; } if (checkpoint->maxPos < 0) { restoreSuccess = maxPosIndex == -1 && llama_memory_seq_rm(llama_get_memory(context->ctx), sequenceId, 0, -1); return; } std::size_t dataSize = checkpoint->data.size(); std::size_t restoreSize = llama_state_seq_set_data_ext(context->ctx, checkpoint->data.data(), dataSize, sequenceId, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); if (restoreSize == dataSize) { restoreSuccess = ( llama_memory_seq_rm(llama_get_memory(context->ctx), sequenceId, maxPosIndex + 1, -1) && llama_memory_seq_pos_max(llama_get_memory(context->ctx), sequenceId) == maxPosIndex ); } } catch (const std::exception& e) { SetError(e.what()); } catch(...) { SetError("Unknown error when calling \"llama_state_seq_set_data_ext\""); } } void OnOK() { deferred.Resolve(Napi::Boolean::New(Env(), restoreSuccess)); } void OnError(const Napi::Error& err) { deferred.Reject(err.Value()); } }; Napi::Value AddonContext::RestoreCheckpoint(const Napi::CallbackInfo& info) { if (disposed || !contextLoaded) { throw Napi::Error::New(info.Env(), "Context is disposed or not loaded"); } AddonContextSequenceCheckpoint* checkpoint = Napi::ObjectWrap::Unwrap(info[0].As()); const llama_pos maxPosIndex = static_cast(info[1].As().Int32Value()); const llama_seq_id sequenceId = static_cast(info[2].As().Int32Value()); RestoreCheckpointWorker* worker = new RestoreCheckpointWorker(info, this, checkpoint, maxPosIndex, sequenceId); worker->Queue(); return worker->GetPromise(); } class CloneSequenceStateWorker : public Napi::AsyncWorker { public: AddonContext* context; llama_seq_id targetSequenceId; llama_seq_id sourceSequenceId; llama_pos copied = 0; CloneSequenceStateWorker( const Napi::CallbackInfo& info, AddonContext* context, llama_seq_id targetSequenceId, llama_seq_id sourceSequenceId ) : Napi::AsyncWorker(info.Env(), "CloneSequenceStateWorker"), context(context), targetSequenceId(targetSequenceId), sourceSequenceId(sourceSequenceId), deferred(Napi::Promise::Deferred::New(info.Env())) { context->Ref(); } ~CloneSequenceStateWorker() { context->Unref(); } Napi::Promise GetPromise() { return deferred.Promise(); } protected: Napi::Promise::Deferred deferred; void Execute() override { try { llama_memory_t mem = llama_get_memory(context->ctx); llama_pos srcMax = llama_memory_seq_pos_max(mem, sourceSequenceId); if (srcMax < 0) { if (!llama_memory_seq_rm(mem, targetSequenceId, -1, -1)) { throw std::runtime_error("Failed to clear destination sequence"); } copied = 0; return; } llama_memory_seq_cp(llama_get_memory(context->ctx), sourceSequenceId, targetSequenceId, -1, -1); if (context->context_params.kv_unified) { if (!llama_memory_seq_rm(mem, targetSequenceId, srcMax + 1, -1)) { throw std::runtime_error("Failed to remove destination sequence KV tail"); } } copied = srcMax; } catch (const std::exception& e) { SetError(e.what()); } catch (...) { SetError("Unknown error when calling \"llama_memory_seq_cp\""); } } void OnOK() override { deferred.Resolve(Napi::Number::New(Env(), copied)); } void OnError(const Napi::Error& err) override { deferred.Reject(err.Value()); } }; Napi::Value AddonContext::CopySequenceStateFromOtherSequence(const Napi::CallbackInfo& info) { if (disposed || !contextLoaded) { throw Napi::Error::New(info.Env(), "Context is disposed or not loaded"); } const llama_seq_id targetSequenceId = static_cast(info[0].As().Int32Value()); const llama_seq_id sourceSequenceId = static_cast(info[1].As().Int32Value()); CloneSequenceStateWorker* worker = new CloneSequenceStateWorker(info, this, targetSequenceId, sourceSequenceId); worker->Queue(); return worker->GetPromise(); } void AddonContext::init(Napi::Object exports) { exports.Set( "AddonContext", DefineClass( exports.Env(), "AddonContext", { InstanceMethod("init", &AddonContext::Init), InstanceMethod("getContextSize", &AddonContext::GetContextSize), InstanceMethod("initBatch", &AddonContext::InitBatch), InstanceMethod("addToBatch", &AddonContext::AddToBatch), InstanceMethod("disposeSequence", &AddonContext::DisposeSequence), InstanceMethod("removeTokenCellsFromSequence", &AddonContext::RemoveTokenCellsFromSequence), InstanceMethod("shiftSequenceTokenCells", &AddonContext::ShiftSequenceTokenCells), InstanceMethod("getSequenceKvCacheMinPosition", &AddonContext::GetSequenceKvCacheMinPosition), InstanceMethod("getSequenceKvCacheMaxPosition", &AddonContext::GetSequenceKvCacheMaxPosition), InstanceMethod("decodeBatch", &AddonContext::DecodeBatch), InstanceMethod("sampleToken", &AddonContext::SampleToken), InstanceMethod("getEmbedding", &AddonContext::GetEmbedding), InstanceMethod("getStateSize", &AddonContext::GetStateSize), InstanceMethod("getMemoryBreakdown", &AddonContext::GetMemoryBreakdown), InstanceMethod("getThreads", &AddonContext::GetThreads), InstanceMethod("setThreads", &AddonContext::SetThreads), InstanceMethod("printTimings", &AddonContext::PrintTimings), InstanceMethod("ensureDraftContextIsCompatibleForSpeculative", &AddonContext::EnsureDraftContextIsCompatibleForSpeculative), InstanceMethod("saveSequenceStateToFile", &AddonContext::SaveSequenceStateToFile), InstanceMethod("loadSequenceStateFromFile", &AddonContext::LoadSequenceStateFromFile), InstanceMethod("setLoras", &AddonContext::SetLoras), InstanceMethod("restoreCheckpoint", &AddonContext::RestoreCheckpoint), InstanceMethod("copySequenceStateFromOtherSequence", &AddonContext::CopySequenceStateFromOtherSequence), InstanceMethod("dispose", &AddonContext::Dispose), } ) ); } AddonContextSequenceCheckpoint::AddonContextSequenceCheckpoint(const Napi::CallbackInfo& info) : Napi::ObjectWrap(info) { } AddonContextSequenceCheckpoint::~AddonContextSequenceCheckpoint() { dispose(); } class AddonContextSequenceCheckpointInitWorker : public Napi::AsyncWorker { public: AddonContextSequenceCheckpoint* checkpoint; AddonContext* context; const llama_seq_id sequenceId; llama_pos minPos = -1; llama_pos maxPos = -1; std::vector data; AddonContextSequenceCheckpointInitWorker(const Napi::CallbackInfo& info, AddonContextSequenceCheckpoint* checkpoint, AddonContext* context, llama_seq_id sequenceId) : Napi::AsyncWorker(info.Env(), "AddonContextSequenceCheckpointInitWorker"), checkpoint(checkpoint), context(context), sequenceId(sequenceId), deferred(Napi::Promise::Deferred::New(info.Env())) { checkpoint->Ref(); context->Ref(); } ~AddonContextSequenceCheckpointInitWorker() { checkpoint->Unref(); context->Unref(); } Napi::Promise GetPromise() { return deferred.Promise(); } protected: Napi::Promise::Deferred deferred; void Execute() { try { minPos = llama_memory_seq_pos_min(llama_get_memory(context->ctx), sequenceId); maxPos = llama_memory_seq_pos_max(llama_get_memory(context->ctx), sequenceId); if (maxPos >= 0) { const size_t checkpointSize = llama_state_seq_get_size_ext(context->ctx, sequenceId, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); data.resize(checkpointSize, 0); if (checkpointSize == 0 || llama_state_seq_get_data_ext(context->ctx, data.data(), checkpointSize, sequenceId, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY) != checkpointSize) { throw std::runtime_error("Failed to capture the complete checkpoint"); } } } catch (const std::exception& e) { SetError(e.what()); } catch(...) { SetError("Unknown error when calling \"llama_state_seq_get_data_ext\""); } } void OnOK() { if (checkpoint->disposed) { deferred.Reject(Napi::Error::New(Env(), "Checkpoint is disposed").Value()); return; } { std::unique_lock lock(checkpoint->dataMutex); checkpoint->data.swap(data); checkpoint->minPos = minPos; checkpoint->maxPos = maxPos; checkpoint->initialized = true; } adjustNapiExternalMemorySubtract(Env(), data.size()); adjustNapiExternalMemoryAdd(Env(), checkpoint->data.size()); deferred.Resolve(Env().Undefined()); } void OnError(const Napi::Error& err) { deferred.Reject(err.Value()); } }; Napi::Value AddonContextSequenceCheckpoint::Init(const Napi::CallbackInfo& info) { AddonContext * context = Napi::ObjectWrap::Unwrap(info[0].As()); if (disposed || context->disposed || !context->contextLoaded) { throw Napi::Error::New(info.Env(), "Checkpoint or context is disposed or not loaded"); } const llama_seq_id requestedSequenceId = info[1].As().Int32Value(); AddonContextSequenceCheckpointInitWorker* worker = new AddonContextSequenceCheckpointInitWorker(info, this, context, requestedSequenceId); worker->Queue(); return worker->GetPromise(); } Napi::Value AddonContextSequenceCheckpoint::Dispose(const Napi::CallbackInfo& info) { dispose(); return info.Env().Undefined(); } void AddonContextSequenceCheckpoint::dispose() { std::unique_lock lock(dataMutex); disposed = true; adjustNapiExternalMemorySubtract(Env(), data.size()); std::vector().swap(data); minPos = -1; maxPos = -1; } Napi::Value AddonContextSequenceCheckpoint::GetSize(const Napi::CallbackInfo& info) { return Napi::Number::New(info.Env(), data.size()); } Napi::Value AddonContextSequenceCheckpoint::GetMinPos(const Napi::CallbackInfo& info) { return Napi::Number::New(info.Env(), minPos); } Napi::Value AddonContextSequenceCheckpoint::GetMaxPos(const Napi::CallbackInfo& info) { return Napi::Number::New(info.Env(), maxPos); } void AddonContextSequenceCheckpoint::init(Napi::Object exports) { exports.Set( "AddonContextSequenceCheckpoint", DefineClass( exports.Env(), "AddonContextSequenceCheckpoint", { InstanceMethod("init", &AddonContextSequenceCheckpoint::Init), InstanceMethod("dispose", &AddonContextSequenceCheckpoint::Dispose), InstanceAccessor("size", &AddonContextSequenceCheckpoint::GetSize, nullptr), InstanceAccessor("minPos", &AddonContextSequenceCheckpoint::GetMinPos, nullptr), InstanceAccessor("maxPos", &AddonContextSequenceCheckpoint::GetMaxPos, nullptr), } ) ); }