From cad638bea3f20fb29f545c4023723fbce062b855 Mon Sep 17 00:00:00 2001 From: dajiaohuang Date: Thu, 22 Oct 2026 01:23:48 +0800 Subject: [PATCH 1/3] fix(tokenizer): honor add_bos in HF tokenization --- llama_cpp/llama_tokenizer.py | 19 +++++++++++++++--- tests/test_llama_tokenizer.py | 36 +++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 tests/test_llama_tokenizer.py diff --git a/llama_cpp/llama_tokenizer.py b/llama_cpp/llama_tokenizer.py index 1375e1392d..fc81b30be1 100644 --- a/llama_cpp/llama_tokenizer.py +++ b/llama_cpp/llama_tokenizer.py @@ -81,9 +81,22 @@ def __init__(self, hf_tokenizer: Any): def tokenize( self, text: bytes, add_bos: bool = True, special: bool = True ) -> List[int]: - return self.hf_tokenizer.encode( - text.decode("utf-8", errors="ignore"), add_special_tokens=special - ) + text = text.decode("utf-8", errors="ignore") + tokens = self.hf_tokenizer.encode(text, add_special_tokens=special) + bos_token_id = getattr(self.hf_tokenizer, "bos_token_id", None) + if bos_token_id is None: + return tokens + + if add_bos: + if not tokens or tokens[0] != bos_token_id: + tokens.insert(0, bos_token_id) + elif special and tokens and tokens[0] == bos_token_id: + raw_tokens = self.hf_tokenizer.encode(text, add_special_tokens=False) + # Preserve a BOS token from the input; remove only a prefixed one. + if tokens[1 : len(raw_tokens) + 1] == raw_tokens: + tokens = tokens[1:] + + return tokens def detokenize( self, diff --git a/tests/test_llama_tokenizer.py b/tests/test_llama_tokenizer.py new file mode 100644 index 0000000000..dfd423f894 --- /dev/null +++ b/tests/test_llama_tokenizer.py @@ -0,0 +1,36 @@ +from llama_cpp.llama_tokenizer import LlamaHFTokenizer + + +class SyntheticHFTokenizer: + bos_token_id = 1 + eos_token_id = 2 + + def encode(self, text, add_special_tokens=True): + tokens = [self.bos_token_id, 3] if text.startswith("") else [3] + if add_special_tokens: + return [self.bos_token_id] + tokens + [self.eos_token_id] + return tokens + + +def test_hf_tokenizer_respects_add_bos_false(): + tokenizer = LlamaHFTokenizer(SyntheticHFTokenizer()) + + assert tokenizer.tokenize(b"prompt", add_bos=False, special=True) == [3, 2] + + +def test_hf_tokenizer_adds_bos_when_special_tokens_are_disabled(): + tokenizer = LlamaHFTokenizer(SyntheticHFTokenizer()) + + assert tokenizer.tokenize(b"prompt", add_bos=True, special=False) == [1, 3] + + +def test_hf_tokenizer_preserves_explicit_bos_when_add_bos_is_false(): + tokenizer = LlamaHFTokenizer(SyntheticHFTokenizer()) + + assert tokenizer.tokenize(b"prompt", add_bos=False, special=True) == [1, 3, 2] + + +def test_hf_tokenizer_does_not_duplicate_bos_when_special_tokens_add_it(): + tokenizer = LlamaHFTokenizer(SyntheticHFTokenizer()) + + assert tokenizer.tokenize(b"prompt", add_bos=True, special=True) == [1, 3, 2] From 71f81fde5a7609ea749c621e799713020424d404 Mon Sep 17 00:00:00 2001 From: dajiaohuang Date: Thu, 22 Oct 2026 01:24:48 +0800 Subject: [PATCH 2/3] docs: add HF tokenizer BOS changelog entry --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c34e2729e..3bcdc3f511 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix(tokenizer): honor add_bos in HF tokenization by @dajiaohuang in #2389 + ## [0.3.36] - feat: update llama.cpp to ggml-org/llama.cpp@0c1e57098 From 445e31bde3920e2adec0ea68b942331451ef5ccf Mon Sep 17 00:00:00 2001 From: dajiaohuang Date: Thu, 22 Oct 2026 01:26:52 +0800 Subject: [PATCH 3/3] docs: defer overlapping changelog entry --- CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bcdc3f511..5c34e2729e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -- fix(tokenizer): honor add_bos in HF tokenization by @dajiaohuang in #2389 - ## [0.3.36] - feat: update llama.cpp to ggml-org/llama.cpp@0c1e57098