---
layout: home
title: node-llama-cpp
titleTemplate: Run AI models locally on your machine
hero:
name: "node-llama-cpp"
text: "Run AI models locally on your machine"
tagline: node.js bindings for llama.cpp, and much more
actions:
- theme: brand
text: Get Started
link: /guide/
- theme: alt
text: API Reference
link: /api/functions/getLlama
image:
src: /logo.jpg
alt: node-llama-cpp Logo
width: 320
height: 320
features:
- icon: 🌟
title: Easy to use
details: |
Zero-config by default.
Works in Node.js, Bun, and Electron.
Bootstrap a project with a single command
link: /guide/
linkText: Learn more
- icon: 🚀
title: Metal, CUDA and Vulkan support
details: Adapts to your hardware automatically to run models with maximum performance
link: /guide/#gpu-support
linkText: Learn more
- icon: 📦
title: Native binaries
details: Pre-built binaries are provided, with a fallback to building from source without node-gyp or Python
link: /guide/building-from-source
linkText: Learn more
- icon:
title: Powerful features
details: Force a model to generate output according to a JSON schema, provide a model with functions it can call on demand, and much more
link: /guide/grammar#json-schema
linkText: Learn more
---
```shell
npx -y node-llama-cpp chat
```
```shell
npx -y node-llama-cpp inspect gpu
```
* [Embedding](./guide/embedding.md)
* [Grammar](./guide/grammar.md)
* [JSON schema grammar](./guide/index.md#chatbot-with-json-schema)
* [Function calling](./guide/index.md#chatbot-with-json-schema)
* [Structured decisions](./guide/structured-decisions.md)
* [CUDA support](./guide/CUDA.md)
* [Metal support](./guide/Metal.md)
* [Vulkan support](./guide/Vulkan.md)
* [Adapts to your hardware](./guide/index.md#gpu-support)
* [Model downloader](./guide/downloading-models.md)
* [Prebuilt binaries](./guide/building-from-source.md)
* [Electron support](./guide/electron.md)
* [Prompt preloading](./guide/chat-session.md#preload-prompt)
* [Automatic chat wrapper](./guide/chat-wrapper.md#chat-wrappers)
* [Template chat wrapper](./guide/chat-wrapper.md#template)
* [Text completion](./guide/text-completion.md#complete)
* [Fill in the middle (infill)](./guide/text-completion.md#infill)
* [Jinja support](./guide/chat-wrapper.md#jinja)
* [Smart context shift](./guide/chat-wrapper.md#smart-context-shift)
* [Token bias](./guide/token-bias.md)
* Windows on Arm support
* [Apple Silicon support](./guide/Metal.md)
* [Inspect GGUF files](./cli/inspect/gguf.md)
* [Custom CMake options](./guide/building-from-source.md#customize-build)
* [Automatic batching](./guide/batching.md)
* [TypeScript type-safety](./api/functions/getLlama.md)
* [LoRA](./api/type-aliases/LlamaContextOptions.md#lora)
* [Remote GGUF reader](./api/functions/readGgufFileInfo.md)
* [User input safety](./guide/llama-text.md#input-safety-in-node-llama-cpp)
* [Token prediction](./guide/token-prediction.md)
* [Reranking](./guide/embedding.md#reranking)
* [Thought segmentation](./guide/chat-session.md#stream-response-segments)
```TypeScript
import {fileURLToPath} from "url";
import path from "path";
import {getLlama, LlamaChatSession} from "node-llama-cpp";
const __dirname = path.dirname(
fileURLToPath(import.meta.url)
);
const llama = await getLlama();
const model = await llama.loadModel({
modelPath: path.join(__dirname, "my-model.gguf")
});
const context = await model.createContext();
const session = new LlamaChatSession({
contextSequence: context.getSequence()
});
const q1 = "Hi there, how are you?";
console.log("User: " + q1);
const a1 = await session.prompt(q1);
console.log("AI: " + a1);
```
```TypeScript
import {fileURLToPath} from "url";
import path from "path";
import {getLlama} from "node-llama-cpp";
const __dirname = path.dirname(
fileURLToPath(import.meta.url)
);
const llama = await getLlama();
const model = await llama.loadModel({
modelPath: path.join(__dirname, "my-model.gguf")
});
const context = await model.createEmbeddingContext();
const text = "Hello world";
console.log("Text:", text);
const embedding = await context.getEmbeddingFor(text);
console.log("Embedding vector:", embedding.vector);
```
```TypeScript
import {fileURLToPath} from "url";
import path from "path";
import {getLlama, LlamaChatSession} from "node-llama-cpp";
const __dirname = path.dirname(
fileURLToPath(import.meta.url)
);
const llama = await getLlama();
const model = await llama.loadModel({
modelPath: path.join(__dirname, "my-model.gguf")
});
const context = await model.createContext();
const session = new LlamaChatSession({
contextSequence: context.getSequence()
});
const grammar = await llama.createGrammarForJsonSchema({
type: "object",
properties: {
positiveWordsInUserMessage: {
type: "array",
items: {
type: "string"
}
},
userMessagePositivityScoreFromOneToTen: {
enum: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
},
nameOfUser: {
oneOf: [{
type: "null"
}, {
type: "string"
}]
}
}
});
const prompt = "Hi there! I'm John. Nice to meet you!";
const res = await session.prompt(prompt, {
grammar
});
const parsedRes = grammar.parse(res);
console.log("User name:", parsedRes.nameOfUser);
console.log(
"Positive words in user message:",
parsedRes.positiveWordsInUserMessage
);
console.log(
"User message positivity score:",
parsedRes.userMessagePositivityScoreFromOneToTen
);
```
```TypeScript
import {fileURLToPath} from "url";
import path from "path";
import {
getLlama,
LlamaChatSession,
defineChatSessionFunction
} from "node-llama-cpp";
const __dirname = path.dirname(
fileURLToPath(import.meta.url)
);
const llama = await getLlama();
const model = await llama.loadModel({
modelPath: path.join(__dirname, "my-model.gguf")
});
const context = await model.createContext();
const session = new LlamaChatSession({
contextSequence: context.getSequence()
});
const fruitPrices: Record = {
"apple": "$6",
"banana": "$4"
};
const functions = {
getFruitPrice: defineChatSessionFunction({
description: "Get the price of a fruit",
params: {
type: "object",
properties: {
name: {
type: "string"
}
}
},
async handler(params) {
const name = params.name.toLowerCase();
if (Object.keys(fruitPrices).includes(name))
return {
name: name,
price: fruitPrices[name]
};
return `Unrecognized fruit "${params.name}"`;
}
})
};
const q1 = "Is an apple more expensive than a banana?";
console.log("User: " + q1);
const a1 = await session.prompt(q1, {functions});
console.log("AI: " + a1);
```
```TypeScript
import {getLlama, resolveModelFile} from "node-llama-cpp";
const modelUri = "hf:giladgd/gemma-4-E2B-it-GGUF:Q8_0";
const llama = await getLlama();
const model = await llama.loadModel({
modelPath: await resolveModelFile(modelUri)
});
const context = await model.createDecisionContext();
await context.warmup(); // optional
const startTime = Date.now();
const ticket =
"I can't sign in after resetting my password. " +
"My whole team is locked out.";
const answers = await context.decide(ticket, {
troubleshooted: {
type: "noul",
instruction:
"Has the customer tried fixing the issue?"
},
team: {
type: "choice",
instruction:
"Which support team should handle this?",
criteria: {
accounts: "Signing in, account access",
billing: "Invoices, payments, refunds",
technical: "Problems after signing in"
}
},
impact: {
type: "score",
instruction:
"How much is the issue " +
"affecting the customer's work?",
criteria: [
"No interruption to work",
"Some tasks are slower or harder",
"The customer cannot continue their work"
]
}
});
console.log({
queue: answers.team.confidence >= 0.7
? answers.team.choice
: "triage",
priority: answers.impact.score >= 1.5 // 0-2 range
? "high"
: "normal",
nextStep: answers.troubleshooted.value >= 0.8
? "review previous attempts"
: "suggest initial troubleshooting"
});
const duration = Date.now() - startTime;
console.log("Duration: " + duration + "ms"); // 170ms
```
```TypeScript
import {
getLlama,
resolveModelFile,
LlamaChatSession
} from "node-llama-cpp";
const modelUri = "hf:giladgd/gemma-4-E2B-it-GGUF:Q8_0";
const llama = await getLlama();
const model = await llama.loadModel({
modelPath: await resolveModelFile(modelUri)
});
const context = await model.createContext();
const session = new LlamaChatSession({
contextSequence: context.getSequence(),
systemPrompt:
"You help customers resolve account access issues"
});
await session.prompt(
"I can't sign in after resetting my password.",
{maxTokens: 150}
);
await session.prompt(
"That fixed it, thanks!",
{maxTokens: 100}
);
const startTime = Date.now();
const answers = await session.decide({
resolved: {
type: "noul",
instruction:
"Has the customer confirmed " +
"that their issue is resolved?"
},
needsHuman: {
type: "noul",
instruction:
"Has the customer asked to speak to a person?"
}
});
if (answers.needsHuman.value >= 0.8)
console.log("Hand off to a support agent");
else if (answers.resolved.value >= 0.8)
console.log("Mark the ticket as resolved");
else
console.log("Continue the conversation");
const duration = Date.now() - startTime;
console.log("Duration: " + duration + "ms"); // 123ms
```