feat: improve server robustness, context manager support, and fix windows dll handling

CHANGELOG.md

@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+- feat: add context manager support to Llama
+- feat(server): add get model by id endpoint and container methods to llamaproxy
+- feat(grammar): expose grammar and root properties on LlamaGrammar
+- fix(windows): safely check directory existence before calling add_dll_directory
+- fix(server): safely handle non-json and get requests in route error handler
+- fix(types): allow tool role in chat request message and deduplicate union
+- fix(cache): remove leftover debug print statements from disk cache
+- fix(server): fix mutually exclusive argument branch in cli parser
+
+
 ## [0.3.36]
 
 - feat: update llama.cpp to ggml-org/llama.cpp@0c1e57098

docs/server.md

@@ -46,6 +46,19 @@ CLI arguments and environment variables are available for all of the fields defi
 
 Additionally the server supports configuration check out the [configuration section](#configuration-and-multi-model-support) for more information and examples.
 
+### Supported Endpoints
+
+The server exposes standard OpenAI-compatible and helper endpoints:
+
+- `GET /v1/models` - List all configured models.
+- `GET /v1/models/{model}` - Retrieve metadata for a specific model by alias/ID.
+- `POST /v1/completions` - Text completions (supports streaming).
+- `POST /v1/chat/completions` - Chat completions (supports streaming, tools, and JSON mode).
+- `POST /v1/embeddings` - Embedding creation.
+- `POST /extras/tokenize` - Tokenize text into token IDs.
+- `POST /extras/tokenize/count` - Count tokens for input.
+- `POST /extras/detokenize` - Detokenize token IDs to text.
+
 
 ## Guides
 

examples/high_level_api/high_level_api_context_manager.py

@@ -0,0 +1,27 @@
+import json
+import argparse
+
+from llama_cpp import Llama
+
+parser = argparse.ArgumentParser(
+    description="Run inference using Llama as a context manager."
+)
+parser.add_argument("-m", "--model", type=str, required=True, help="Path to GGUF model")
+parser.add_argument(
+    "-p",
+    "--prompt",
+    type=str,
+    default="Question: What is the speed of light? Answer:",
+    help="Prompt for completion",
+)
+args = parser.parse_args()
+
+# Using Llama in a context manager automatically closes the model and frees resources
+with Llama(model_path=args.model, verbose=False) as llm:
+    output = llm(
+        args.prompt,
+        max_tokens=64,
+        stop=["\n", "Question:"],
+        echo=True,
+    )
+    print(json.dumps(output, indent=2))

llama_cpp/_ctypes_extensions.py

@@ -55,18 +55,19 @@ def load_shared_library(lib_base_name: str, base_path: pathlib.Path):
 
     # Add the library directory to the DLL search path on Windows (if needed)
     if sys.platform == "win32":
-        os.add_dll_directory(str(base_path))
-        os.environ["PATH"] = str(base_path) + os.pathsep + os.environ["PATH"]
-
-    if sys.platform == "win32" and sys.version_info >= (3, 8):
-        os.add_dll_directory(str(base_path))
-        if "CUDA_PATH" in os.environ:
-            os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "bin"))
-            os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "lib"))
-        if "HIP_PATH" in os.environ:
-            os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "bin"))
-            os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "lib"))
-        cdll_args["winmode"] = ctypes.RTLD_GLOBAL
+        if base_path.exists() and base_path.is_dir():
+            if sys.version_info >= (3, 8):
+                os.add_dll_directory(str(base_path))
+            os.environ["PATH"] = str(base_path) + os.pathsep + os.environ["PATH"]
+
+        if sys.version_info >= (3, 8):
+            for env_var in ("CUDA_PATH", "HIP_PATH"):
+                if env_var in os.environ:
+                    for sub in ("bin", "lib"):
+                        sub_path = os.path.join(os.environ[env_var], sub)
+                        if os.path.isdir(sub_path):
+                            os.add_dll_directory(sub_path)
+            cdll_args["winmode"] = ctypes.RTLD_GLOBAL
 
     if sys.platform == "emscripten":
         cdll_args["mode"] = ctypes.RTLD_GLOBAL

llama_cpp/llama.py

@@ -2296,6 +2296,12 @@ def close(self) -> None:
         """Explicitly free the model from memory."""
         self._stack.close()
 
+    def __enter__(self) -> Llama:
+        return self
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        self.close()
+
     def __del__(self) -> None:
         self.close()
 

llama_cpp/llama_cache.py

@@ -142,14 +142,10 @@ def __contains__(self, key: Sequence[int]) -> bool:
         return self._find_longest_prefix_key(tuple(key)) is not None
 
     def __setitem__(self, key: Sequence[int], value: "llama_cpp.llama.LlamaState"):
-        print("LlamaDiskCache.__setitem__: called", file=sys.stderr)
         key = tuple(key)
         if key in self.cache:
-            print("LlamaDiskCache.__setitem__: delete", file=sys.stderr)
             del self.cache[key]
         self.cache[key] = value
-        print("LlamaDiskCache.__setitem__: set", file=sys.stderr)
         while self.cache_size > self.capacity_bytes and len(self.cache) > 0:
             key_to_remove = next(iter(self.cache))
             del self.cache[key_to_remove]
-        print("LlamaDiskCache.__setitem__: trim", file=sys.stderr)

llama_cpp/llama_grammar.py

@@ -21,6 +21,16 @@ def __init__(self, *args, _grammar: str, **kwargs):
         self._grammar = _grammar
         self._root = LLAMA_GRAMMAR_DEFAULT_ROOT
 
+    @property
+    def grammar(self) -> str:
+        """Return the GBNF grammar string."""
+        return self._grammar
+
+    @property
+    def root(self) -> str:
+        """Return the root rule name."""
+        return self._root
+
     @classmethod
     def from_string(cls, grammar: str, verbose: bool = True) -> "LlamaGrammar":
         return cls(_grammar=grammar)
@@ -39,7 +49,7 @@ def from_file(cls, file: Union[str, Path], verbose: bool = True) -> "LlamaGramma
             return cls.from_string(grammar, verbose=verbose)
 
         raise ValueError(
-            f"{cls.from_file.__name__}: error parsing grammar file: params_grammer is empty"
+            f"{cls.from_file.__name__}: error parsing grammar file: params_grammar is empty"
         )
 
     @classmethod

llama_cpp/llama_tokenizer.py

@@ -8,7 +8,6 @@
 )
 
 import llama_cpp
-from llama_cpp.llama_types import List
 
 
 class BaseLlamaTokenizer(abc.ABC):

llama_cpp/llama_types.py

@@ -255,7 +255,6 @@ class ChatCompletionRequestFunctionMessage(TypedDict):
     ChatCompletionRequestSystemMessage,
     ChatCompletionRequestUserMessage,
     ChatCompletionRequestAssistantMessage,
-    ChatCompletionRequestUserMessage,
     ChatCompletionRequestToolMessage,
     ChatCompletionRequestFunctionMessage,
 ]

llama_cpp/server/app.py

@@ -36,6 +36,7 @@
     CreateEmbeddingRequest,
     CreateChatCompletionRequest,
     ModelList,
+    ModelData,
     TokenizeInputRequest,
     TokenizeInputResponse,
     TokenizeInputCountResponse,
@@ -549,6 +550,29 @@ async def get_models(
     }
 
 
+@router.get(
+    "/v1/models/{model:path}",
+    summary="Get Model",
+    dependencies=[Depends(authenticate)],
+    tags=[openai_v1_tag],
+)
+async def get_model(
+    model: str,
+    llama_proxy: LlamaProxy = Depends(get_llama_proxy),
+) -> ModelData:
+    if model not in llama_proxy:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Model '{model}' not found",
+        )
+    return {
+        "id": model,
+        "object": "model",
+        "owned_by": "me",
+        "permissions": [],
+    }
+
+
 extras_tag = "Extras"
 
 

llama_cpp/server/cli.py

@@ -98,20 +98,20 @@ def add_args_from_model(parser: argparse.ArgumentParser, model: Type[BaseModel])
                 type=_parse_json_object_arg,
                 help=description,
             )
-        elif base_type is not bool:
+        elif base_type is bool:
             parser.add_argument(
                 f"--{name}",
                 dest=name,
-                nargs="*" if list_type else None,
-                type=base_type,
-                help=description,
+                type=_parse_bool_arg,
+                help=f"{description}",
             )
-        if base_type is bool:
+        else:
             parser.add_argument(
                 f"--{name}",
                 dest=name,
-                type=_parse_bool_arg,
-                help=f"{description}",
+                nargs="*" if list_type else None,
+                type=base_type,
+                help=description,
             )
 
 

llama_cpp/server/errors.py

@@ -178,25 +178,27 @@ async def custom_route_handler(request: Request) -> Response:
                 # api key check failed
                 raise unauthorized
             except Exception as exc:
-                json_body = await request.json()
+                body: Optional[
+                    Union[
+                        CreateChatCompletionRequest,
+                        CreateCompletionRequest,
+                        CreateEmbeddingRequest,
+                    ]
+                ] = None
                 try:
-                    if "messages" in json_body:
-                        # Chat completion
-                        body: Optional[
-                            Union[
-                                CreateChatCompletionRequest,
-                                CreateCompletionRequest,
-                                CreateEmbeddingRequest,
-                            ]
-                        ] = CreateChatCompletionRequest(**json_body)
-                    elif "prompt" in json_body:
-                        # Text completion
-                        body = CreateCompletionRequest(**json_body)
-                    else:
-                        # Embedding
-                        body = CreateEmbeddingRequest(**json_body)
+                    json_body = await request.json()
+                    if isinstance(json_body, dict):
+                        if "messages" in json_body:
+                            # Chat completion
+                            body = CreateChatCompletionRequest(**json_body)
+                        elif "prompt" in json_body:
+                            # Text completion
+                            body = CreateCompletionRequest(**json_body)
+                        else:
+                            # Embedding
+                            body = CreateEmbeddingRequest(**json_body)
                 except Exception:
-                    # Invalid request body
+                    # Invalid or unreadable request body (e.g. GET request or non-JSON)
                     body = None
 
                 # Get proper error message from the exception

llama_cpp/server/model.py

@@ -2,7 +2,7 @@
 
 import json
 
-from typing import Dict, Optional, Union, List
+from typing import Dict, Optional, Union, List, Any
 
 import llama_cpp
 import llama_cpp.llama_speculative as llama_speculative
@@ -65,6 +65,17 @@ def __iter__(self):
         for model in self._model_settings_dict:
             yield model
 
+    def __contains__(self, model: str) -> bool:
+        return model in self._model_settings_dict
+
+    def __len__(self) -> int:
+        return len(self._model_settings_dict)
+
+    def get(self, model: str, default: Any = None) -> Any:
+        if model in self._model_settings_dict:
+            return self[model]
+        return default
+
     def free(self):
         if self._current_model:
             self._current_model.close()

llama_cpp/server/types.py

@@ -181,7 +181,7 @@ class CreateEmbeddingRequest(BaseModel):
 
 
 class ChatCompletionRequestMessage(BaseModel):
-    role: Literal["system", "user", "assistant", "function"] = Field(
+    role: Literal["system", "user", "assistant", "tool", "function"] = Field(
         default="user", description="The role of the message."
     )
     content: Optional[str] = Field(
@@ -303,7 +303,7 @@ class TokenizeInputCountResponse(BaseModel):
 
 class DetokenizeInputRequest(BaseModel):
     model: Optional[str] = model_field
-    tokens: List[int] = Field(description="A list of toekns to detokenize.")
+    tokens: List[int] = Field(description="A list of tokens to detokenize.")
 
     model_config = {"json_schema_extra": {"example": [{"tokens": [123, 321, 222]}]}}
 

tests/test_llama.py

@@ -20,6 +20,16 @@ def test_llama_cpp_version():
     assert llama_cpp.__version__
 
 
+def test_llama_context_manager():
+    from unittest.mock import MagicMock
+
+    mock_llama = llama_cpp.Llama.__new__(llama_cpp.Llama)
+    mock_llama._stack = MagicMock()
+    with mock_llama as llm:
+        assert llm is mock_llama
+    mock_llama._stack.close.assert_called_once()
+
+
 def test_llama_cpp_tokenization():
     llama = llama_cpp.Llama(model_path=MODEL, vocab_only=True, verbose=False)