From 3c74b9111278bd231ce8e2151a8a76ab8e15fc1b Mon Sep 17 00:00:00 2001
From: Martin Evans <martindevans@gmail.com>
Date: Sat, 1 Jun 2024 21:22:37 +0100
Subject: [PATCH 1/2] Added support for decoding "special" tokens in the
 streaming text decoder

---
 LLama/StreamingTokenDecoder.cs | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/LLama/StreamingTokenDecoder.cs b/LLama/StreamingTokenDecoder.cs
index 9252e532d..d56fa3f57 100644
--- a/LLama/StreamingTokenDecoder.cs
+++ b/LLama/StreamingTokenDecoder.cs
@@ -1,9 +1,8 @@
-using System.Buffers;
+using System.Buffers;
 using System.Diagnostics;
 using System;
 using System.Collections.Generic;
 using System.Text;
-using LLama.Extensions;
 using LLama.Native;
 
 namespace LLama
@@ -23,6 +22,11 @@ public sealed class StreamingTokenDecoder
         /// </summary>
         public int AvailableCharacters => _characters.Count;
 
+        /// <summary>
+        /// If true, special characters will be converted to text. If false they will be invisible.
+        /// </summary>
+        public bool Special { get; set; }
+
         #region constructors
         /// <summary>
         /// Create a new decoder
@@ -76,7 +80,7 @@ public void Add(LLamaToken token)
             try
             {
                 // Convert this token into bytes
-                var bytesAvailable = TokenToBytes(ref bytesArr, token, _weights).Length;
+                var bytesAvailable = TokenToBytes(ref bytesArr, token, _weights, Special).Length;
 
                 // Convert those bytes into characters
                 var bytesOffset = 0;
@@ -108,10 +112,10 @@ public void Add(LLamaToken token)
 
             // Converts a single token into bytes, using the `bytes` array as temporary storage.
             // If the `bytes` array is too small it will get a larger one from the ArrayPool.
-            static Span<byte> TokenToBytes(ref byte[] bytes, LLamaToken token, SafeLlamaModelHandle model)
+            static Span<byte> TokenToBytes(ref byte[] bytes, LLamaToken token, SafeLlamaModelHandle model, bool special)
             {
                 // Try to get bytes
-                var l = model.TokenToSpan(token, bytes);
+                var l = model.TokenToSpan(token, bytes, special);
 
                 // Check if the length was larger than the buffer. If so expand the buffer and try again
                 if (l > bytes.Length)

From 1378ee002bdeeb819a36e93b03f35c55f3ccdc65 Mon Sep 17 00:00:00 2001
From: Martin Evans <martindevans@gmail.com>
Date: Sun, 2 Jun 2024 15:17:56 +0100
Subject: [PATCH 2/2] Renamed `Special` to `DecodeSpecialTokens`

---
 LLama/StreamingTokenDecoder.cs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/LLama/StreamingTokenDecoder.cs b/LLama/StreamingTokenDecoder.cs
index d56fa3f57..60de20769 100644
--- a/LLama/StreamingTokenDecoder.cs
+++ b/LLama/StreamingTokenDecoder.cs
@@ -25,7 +25,7 @@ public sealed class StreamingTokenDecoder
         /// <summary>
         /// If true, special characters will be converted to text. If false they will be invisible.
         /// </summary>
-        public bool Special { get; set; }
+        public bool DecodeSpecialTokens { get; set; }
 
         #region constructors
         /// <summary>
@@ -80,7 +80,7 @@ public void Add(LLamaToken token)
             try
             {
                 // Convert this token into bytes
-                var bytesAvailable = TokenToBytes(ref bytesArr, token, _weights, Special).Length;
+                var bytesAvailable = TokenToBytes(ref bytesArr, token, _weights, DecodeSpecialTokens).Length;
 
                 // Convert those bytes into characters
                 var bytesOffset = 0;