{"as_of":"2026-10-12","license":"CC-BY-4.0","license_url":"https://creativecommons.org/licenses/by/4.0/","cite_as":"The Local Bench (thelocalbench.com), data as of 2026-10-12, CC BY 4.0","source_url":"https://thelocalbench.com","guide":{"slug":"private-document-search-rag","title":"Set up private document search (RAG) with Ollama and Open WebUI","short":"Private document search","dek":"Ask questions about your own PDFs and notes with a model that never leaves your machine. Uses a local embedding model, an Open WebUI knowledge base, and a chat model with enough context to read what it finds.","summary":"Pull a local embedding model and a tool-capable chat model in Ollama, raise Ollama's context length, point Open WebUI's document embedding at Ollama, create a knowledge base, upload files, and query it with","difficulty":"Intermediate","time_minutes":40,"cost":{"usd":0,"note":"free software; a GPU with 12 GB+ VRAM or 32 GB+ unified memory recommended"},"hardware":["GPU with 12 GB+ VRAM, or a Mac / mini PC with 32 GB+ unified memory","CPU-only works for small collections, slowly"],"os":["Linux","macOS","Windows"],"prerequisites":["Ollama and Open WebUI installed (see \"Run your first local model\")","Some text-based PDFs, Word files or notes to test with"],"parts":[{"name":"Ollama","note":"runs the chat and embedding models","href":"https://ollama.com/download"},{"name":"Open WebUI","note":"knowledge bases and chat","href":"https://docs.openwebui.com/features/workspace/knowledge/"},{"name":"nomic-embed-text","note":"embedding model, 274 MB","href":"https://ollama.com/library/nomic-embed-text"},{"name":"Qwen3 8B or 14B","note":"chat model with native tool calling","href":"https://ollama.com/library/qwen3"}],"tested_on":[],"last_tested":null,"flag":"untested","flag_note":"Commands and menu paths come from the Ollama and Open WebUI documentation (checked 2026-10-13). We have not yet run this guide end to end. Open WebUI menu labels change between releases, so expect small differences.","untested":["The whole guide end to end","The exact labels of the Embedding Model Engine and Embedding Model fields in Settings > Admin > Documents","Retrieval quality with nomic-embed-text on non-English documents"],"reviewed":"2026-10-13","tools":["ollama","open-webui"],"explainers":["rag-and-embeddings","context-length","tool-calling-and-mcp"],"sources":[{"label":"Open WebUI docs: Knowledge bases and document chat","url":"https://docs.openwebui.com/features/workspace/knowledge/"},{"label":"Open WebUI docs: RAG troubleshooting","url":"https://docs.openwebui.com/troubleshooting/rag/"},{"label":"Ollama blog: embedding models","url":"https://ollama.com/blog/embedding-models"},{"label":"Ollama API: POST /api/embed","url":"https://github.com/ollama/ollama/blob/main/docs/api.md"},{"label":"Ollama docs: context length","url":"https://docs.ollama.com/context-length"}],"steps":[{"n":1,"title":"Pull an embedding model and a chat model"},{"n":2,"title":"Check the embedding model answers"},{"n":3,"title":"Give the chat model enough context"},{"n":4,"title":"Point Open WebUI's document embedding at Ollama"},{"n":5,"title":"Create a knowledge base and add files"},{"n":6,"title":"Verify it works"}],"url":"https://thelocalbench.com/how-to/private-document-search-rag","markdown":"https://thelocalbench.com/how-to/private-document-search-rag.md","json":"https://thelocalbench.com/api/v1/how-to/private-document-search-rag.json"},"json_ld":{"@context":"https://schema.org","@type":"HowTo","name":"Set up private document search (RAG) with Ollama and Open WebUI","description":"Pull a local embedding model and a tool-capable chat model in Ollama, raise Ollama's context length, point Open WebUI's document embedding at Ollama, create a knowledge base, upload files, and query it with","url":"https://thelocalbench.com/how-to/private-document-search-rag","totalTime":"PT40M","dateModified":"2026-10-13","estimatedCost":{"@type":"MonetaryAmount","currency":"USD","value":0},"supply":[{"@type":"HowToSupply","name":"Ollama"},{"@type":"HowToSupply","name":"Open WebUI"},{"@type":"HowToSupply","name":"nomic-embed-text"},{"@type":"HowToSupply","name":"Qwen3 8B or 14B"}],"tool":[{"@type":"HowToTool","name":"Ollama and Open WebUI installed (see \"Run your first local model\")"},{"@type":"HowToTool","name":"Some text-based PDFs, Word files or notes to test with"}],"step":[{"@type":"HowToStep","position":1,"name":"Pull an embedding model and a chat model","url":"https://thelocalbench.com/how-to/private-document-search-rag#step-1"},{"@type":"HowToStep","position":2,"name":"Check the embedding model answers","url":"https://thelocalbench.com/how-to/private-document-search-rag#step-2"},{"@type":"HowToStep","position":3,"name":"Give the chat model enough context","url":"https://thelocalbench.com/how-to/private-document-search-rag#step-3"},{"@type":"HowToStep","position":4,"name":"Point Open WebUI's document embedding at Ollama","url":"https://thelocalbench.com/how-to/private-document-search-rag#step-4"},{"@type":"HowToStep","position":5,"name":"Create a knowledge base and add files","url":"https://thelocalbench.com/how-to/private-document-search-rag#step-5"},{"@type":"HowToStep","position":6,"name":"Verify it works","url":"https://thelocalbench.com/how-to/private-document-search-rag#step-6"}]}}