diff --git a/.github/validate-models.bat b/.github/validate-models.bat index dccf529e..1f374cc9 100644 --- a/.github/validate-models.bat +++ b/.github/validate-models.bat @@ -4,7 +4,7 @@ REM GGUF files start with magic bytes: 0x47 0x47 0x55 0x46 ("GGUF") setlocal enabledelayedexpansion -set "MODELS=models\codellama-7b.Q2_K.gguf" "models\jina-reranker-v1-tiny-en-Q4_0.gguf" "models\AMD-Llama-135m-code.Q2_K.gguf" +set "MODELS=models\codellama-7b.Q2_K.gguf" "models\jina-reranker-v1-tiny-en-Q4_0.gguf" "models\AMD-Llama-135m-code.Q2_K.gguf" "models\Qwen3-0.6B-Q4_K_M.gguf" echo Validating model files... for %%M in (%MODELS%) do ( diff --git a/.github/validate-models.sh b/.github/validate-models.sh index d63b0571..fa53d8b8 100755 --- a/.github/validate-models.sh +++ b/.github/validate-models.sh @@ -8,6 +8,7 @@ MODELS=( "models/codellama-7b.Q2_K.gguf" "models/jina-reranker-v1-tiny-en-Q4_0.gguf" "models/AMD-Llama-135m-code.Q2_K.gguf" + "models/Qwen3-0.6B-Q4_K_M.gguf" ) echo "Validating model files..." diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ff0bcf43..0eb5bee2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -21,6 +21,8 @@ env: RERANKING_MODEL_NAME: "jina-reranker-v1-tiny-en-Q4_0.gguf" DRAFT_MODEL_URL: "https://huggingface.co/QuantFactory/AMD-Llama-135m-code-GGUF/resolve/main/AMD-Llama-135m-code.Q2_K.gguf" DRAFT_MODEL_NAME: "AMD-Llama-135m-code.Q2_K.gguf" + REASONING_MODEL_URL: "https://huggingface.co/unsloth/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q4_K_M.gguf" + REASONING_MODEL_NAME: "Qwen3-0.6B-Q4_K_M.gguf" permissions: contents: read jobs: @@ -308,6 +310,8 @@ jobs: run: curl -L --fail ${RERANKING_MODEL_URL} --create-dirs -o models/${RERANKING_MODEL_NAME} - name: Download draft model run: curl -L --fail ${DRAFT_MODEL_URL} --create-dirs -o models/${DRAFT_MODEL_NAME} + - name: Download reasoning model + run: curl -L --fail ${REASONING_MODEL_URL} --create-dirs -o models/${REASONING_MODEL_NAME} - name: List files in models directory run: ls -l models/ - name: Validate model files @@ -357,6 +361,8 @@ jobs: run: curl -L --fail ${RERANKING_MODEL_URL} --create-dirs -o models/${RERANKING_MODEL_NAME} - name: Download draft model run: curl -L --fail ${DRAFT_MODEL_URL} --create-dirs -o models/${DRAFT_MODEL_NAME} + - name: Download reasoning model + run: curl -L --fail ${REASONING_MODEL_URL} --create-dirs -o models/${REASONING_MODEL_NAME} - name: List files in models directory run: ls -l models/ - name: Validate model files @@ -417,6 +423,8 @@ jobs: run: curl -L --fail ${RERANKING_MODEL_URL} --create-dirs -o models/${RERANKING_MODEL_NAME} - name: Download draft model run: curl -L --fail ${DRAFT_MODEL_URL} --create-dirs -o models/${DRAFT_MODEL_NAME} + - name: Download reasoning model + run: curl -L --fail ${REASONING_MODEL_URL} --create-dirs -o models/${REASONING_MODEL_NAME} - name: List files in models directory run: ls -l models/ - name: Validate model files @@ -468,6 +476,8 @@ jobs: run: curl -L --fail ${RERANKING_MODEL_URL} --create-dirs -o models/${RERANKING_MODEL_NAME} - name: Download draft model run: curl -L --fail ${DRAFT_MODEL_URL} --create-dirs -o models/${DRAFT_MODEL_NAME} + - name: Download reasoning model + run: curl -L --fail ${REASONING_MODEL_URL} --create-dirs -o models/${REASONING_MODEL_NAME} - name: List files in models directory run: ls -l models/ - name: Validate model files @@ -522,6 +532,8 @@ jobs: run: curl -L --fail $env:RERANKING_MODEL_URL --create-dirs -o models/$env:RERANKING_MODEL_NAME - name: Download draft model run: curl -L --fail $env:DRAFT_MODEL_URL --create-dirs -o models/$env:DRAFT_MODEL_NAME + - name: Download reasoning model + run: curl -L --fail $env:REASONING_MODEL_URL --create-dirs -o models/$env:REASONING_MODEL_NAME - name: List files in models directory run: ls -l models/ - name: Validate model files diff --git a/src/test/java/net/ladenthin/llama/ReasoningBudgetTest.java b/src/test/java/net/ladenthin/llama/ReasoningBudgetTest.java new file mode 100644 index 00000000..84a12343 --- /dev/null +++ b/src/test/java/net/ladenthin/llama/ReasoningBudgetTest.java @@ -0,0 +1,225 @@ +package net.ladenthin.llama; + +import java.io.File; +import java.util.Collections; + +import net.ladenthin.llama.args.ReasoningFormat; +import net.ladenthin.llama.json.ChatResponseParser; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; + +/** + * Integration tests for thinking/reasoning mode using Qwen3-0.6B-Q4_K_M. + * + *

These tests require the Qwen3-0.6B-Q4_K_M model (downloaded by CI). The entire + * class is skipped when the model file is absent, matching the pattern used by all + * other model-dependent test classes. + * + *

Confirmed behaviour (Qwen3-0.6B, llama.cpp b9151)

+ *
    + *
  1. Thinking is active by default. Qwen3's built-in chat template injects + * {@code } into the prompt before generation starts. No extra kwarg is + * required; the model reasons on every request.
    2. + *
  2. DEEPSEEK reasoning format correctly extracts thinking tokens. Setting + * {@code --reasoning-format deepseek} at model load time causes the server to + * strip the {@code …} block from the response body and surface it + * in {@code reasoning_content}.
    3. + *
  3. {@code reasoning_budget_tokens} is NOT enforced for any model when set + * per-request. The root cause is a bug in + * {@code tools/server/server-common.cpp}, function + * {@code oaicompat_chat_params_parse}: the reasoning-budget block writes + * the model-level default ({@code opt.reasoning_budget}, typically −1) + * into {@code llama_params["reasoning_budget_tokens"]} before the generic + * copy loop runs. The copy loop then skips the per-request value from the + * request body because the key already exists + * ({@code !llama_params.contains(item.key())} is false). Result: the + * reasoning-budget sampler is never created (it requires + * {@code reasoning_budget_tokens ≥ 0}), and any per-request budget + * has no effect. Parameter serialisation itself is correct — see + * {@code InferenceParametersTest} and the C++ unit tests.
    4. + *
+ */ +@ClaudeGenerated( + purpose = "Integration tests for Qwen3 thinking-mode extraction and reasoning_budget_tokens " + + "parameter acceptance. Documents the known llama.cpp limitation that budget " + + "enforcement does not work for prompt-injected thinking models." +) +public class ReasoningBudgetTest { + + /** + * Generous token budget: Qwen3-0.6B spends up to ~200 tokens thinking before answering. + * 500 is enough for thinking + a short answer on all tested platforms. + */ + private static final int N_PREDICT = 500; + + private static LlamaModel model; + private final ChatResponseParser parser = new ChatResponseParser(); + + @BeforeClass + public static void setup() { + Assume.assumeTrue("Reasoning model not found, skipping ReasoningBudgetTest", + new File(TestConstants.REASONING_MODEL_PATH).exists()); + int gpuLayers = Integer.getInteger(TestConstants.PROP_TEST_NGL, TestConstants.DEFAULT_TEST_NGL); + model = new LlamaModel( + new ModelParameters() + .setModel(TestConstants.REASONING_MODEL_PATH) + .setCtxSize(2048) + .setGpuLayers(gpuLayers) + .setFit(false) + .setReasoningFormat(ReasoningFormat.DEEPSEEK) + .enableLogTimestamps().enableLogPrefix() + ); + } + + @AfterClass + public static void tearDown() { + if (model != null) { + model.close(); + } + } + + /** + * Qwen3 enters thinking mode by default. With {@code reasoning_format=deepseek} set + * at model level, the thinking tokens must appear in {@code reasoning_content} and + * the final answer must appear in {@code content}. + */ + @Test + public void testThinkingDefault_reasoningContentAndAnswerPresent() { + InferenceParameters params = new InferenceParameters("") + .setMessages(null, Collections.singletonList(new Pair<>("user", "What is 2+2?"))) + .setNPredict(N_PREDICT); + + String json = model.chatComplete(params); + String reasoningContent = parser.extractChoiceReasoningContent(json); + String content = parser.extractChoiceContent(json); + + Assert.assertFalse( + "reasoning_content should be non-empty (Qwen3 thinks by default)", + reasoningContent == null || reasoningContent.trim().isEmpty()); + Assert.assertFalse( + "content must not be empty (model must produce an answer after thinking)", + content == null || content.trim().isEmpty()); + } + + /** + * {@code reasoning_budget_tokens=0} is accepted by the API and the response + * completes without error, but the budget is NOT enforced. + * + *

Documents current (broken) behaviour. The per-request value is + * silently discarded by a bug in {@code tools/server/server-common.cpp} + * ({@code oaicompat_chat_params_parse}): the reasoning-budget block writes the + * model-level default (−1) to {@code llama_params["reasoning_budget_tokens"]} + * before the generic copy loop runs, and the copy loop then skips the user value + * because the key already exists. The reasoning-budget sampler is therefore never + * created, and {@code reasoning_content} remains non-empty. + * + *

This assertion will start failing once the llama.cpp bug is fixed — + * that is the signal to remove this test and enable + * {@link #testReasoningBudgetZero_expectedBehavior_suppressesThinking}. + */ + @Test + public void testReasoningBudgetZero_parameterAccepted_thinkingNotSuppressed() { + InferenceParameters params = new InferenceParameters("") + .setMessages(null, Collections.singletonList(new Pair<>("user", "What is 2+2?"))) + .setReasoningBudgetTokens(0) + .setNPredict(N_PREDICT); + + String json = model.chatComplete(params); + + Assert.assertNotNull("Response JSON must not be null", json); + + String reasoningContent = parser.extractChoiceReasoningContent(json); + Assert.assertFalse( + "reasoning_content is expected to be present because the per-request " + + "budget is not applied (llama.cpp server-common.cpp copy-loop bug). " + + "If this assertion fails, the bug has been fixed — remove this test and " + + "enable testReasoningBudgetZero_expectedBehavior_suppressesThinking.", + reasoningContent == null || reasoningContent.trim().isEmpty()); + } + + /** + * Expected correct behaviour after the llama.cpp bug is fixed. + * + *

Bug: In {@code tools/server/server-common.cpp}, + * {@code oaicompat_chat_params_parse} sets + * {@code llama_params["reasoning_budget_tokens"]} to the model-level default + * ({@code opt.reasoning_budget}, typically −1) before the generic copy + * loop runs. The copy loop then skips the per-request value from the request + * body because the key already exists. Result: the sampler is never created + * ({@code reasoning_budget_tokens ≥ 0} is required), and budget=0 + * has no effect. + * + *

Fix (server-common.cpp, reasoning budget block): + * Read {@code reasoning_budget_tokens} from the request body before + * writing to {@code llama_params}: + * 

+     * int reasoning_budget = opt.reasoning_budget;
+     * if (body.contains("reasoning_budget_tokens")) {
+     *     reasoning_budget = json_value(body, "reasoning_budget_tokens", reasoning_budget);
+     * }
+     * if (reasoning_budget == -1 && body.contains("thinking_budget_tokens")) {
+     *     reasoning_budget = json_value(body, "thinking_budget_tokens", -1);
+     * }
+     *
+ * + *

Once this fix is applied: remove {@code @Ignore}, confirm this test passes, + * and remove + * {@link #testReasoningBudgetZero_parameterAccepted_thinkingNotSuppressed}. + */ + @Ignore("llama.cpp bug: per-request reasoning_budget_tokens is overwritten by model default " + + "in oaicompat_chat_params_parse (server-common.cpp). " + + "See Javadoc for exact fix location and code.") + @Test + public void testReasoningBudgetZero_expectedBehavior_suppressesThinking() { + InferenceParameters params = new InferenceParameters("") + .setMessages(null, Collections.singletonList(new Pair<>("user", "What is 2+2?"))) + .setReasoningBudgetTokens(0) + .setNPredict(N_PREDICT); + + String json = model.chatComplete(params); + Assert.assertNotNull("Response JSON must not be null", json); + + String reasoningContent = parser.extractChoiceReasoningContent(json); + Assert.assertTrue( + "reasoning_content should be empty when budget=0 suppresses thinking, " + + "but was: " + reasoningContent, + reasoningContent == null || reasoningContent.trim().isEmpty()); + } + + /** + * A positive {@code reasoning_budget_tokens} value is accepted and the call completes + * without error. + * + *

The assertion checks that the model produced a non-empty response — either in + * {@code reasoning_content} or {@code content}. On slow or constrained hardware the + * model may exhaust the token budget inside the thinking block and emit an empty + * {@code content}; checking both fields makes the test robust to that behaviour. + * + *

See {@link #testReasoningBudgetZero_parameterAccepted_thinkingNotSuppressed} for + * the note on why the budget count itself is not asserted. + */ + @Test + public void testReasoningBudgetPositive_parameterAccepted() { + InferenceParameters params = new InferenceParameters("") + .setMessages(null, Collections.singletonList( + new Pair<>("user", "Think step by step: what is 3 times 7?"))) + .setReasoningBudgetTokens(100) + .setNPredict(N_PREDICT); + + String json = model.chatComplete(params); + Assert.assertNotNull("Response JSON must not be null", json); + + String reasoningContent = parser.extractChoiceReasoningContent(json); + String content = parser.extractChoiceContent(json); + boolean hasReasoning = reasoningContent != null && !reasoningContent.trim().isEmpty(); + boolean hasContent = content != null && !content.trim().isEmpty(); + Assert.assertTrue( + "model must produce at least some output in reasoning_content or content, " + + "but both were empty", + hasReasoning || hasContent); + } +} diff --git a/src/test/java/net/ladenthin/llama/TestConstants.java b/src/test/java/net/ladenthin/llama/TestConstants.java index d589d4fa..c289ffe5 100644 --- a/src/test/java/net/ladenthin/llama/TestConstants.java +++ b/src/test/java/net/ladenthin/llama/TestConstants.java @@ -13,4 +13,7 @@ class TestConstants { /** Path to the draft model used for speculative decoding tests. */ static final String DRAFT_MODEL_PATH = "models/AMD-Llama-135m-code.Q2_K.gguf"; + /** Path to the Qwen3 thinking model used for reasoning budget tests. */ + static final String REASONING_MODEL_PATH = "models/Qwen3-0.6B-Q4_K_M.gguf"; + }