From 6f28ff293132c27901e28c5a8d861929549b6212 Mon Sep 17 00:00:00 2001
From: xbotter <xbotter@live.cn>
Date: Sun, 3 Dec 2023 15:31:02 +0800
Subject: [PATCH 01/12] =?UTF-8?q?=F0=9F=93=9D=20Add=20SemanticKernelTextGe?=
 =?UTF-8?q?neration=20for=20Semantic=20Kernel=20Text=20Completion?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add a new file SemanticKernelTextGeneration.cs to implement the ITextGeneration interface for Semantic Kernel Text Completion. This class provides the functionality to generate text asynchronously based on a prompt and text generation options. The AIRequestSettings are converted from the TextGenerationOptions to set parameters such as temperature, top-p, presence penalty, frequency penalty, stop sequences, max tokens, results per prompt, and token selection biases. The SemanticKernelTextGeneration class is added to the KernelMemoryBuilderExtensions as an extension method to configure the Semantic Kernel Text Completion service.
---
 KernelMemory.sln                              |   7 +++
 ...otnet-SemanticKernel-TextCompletion.csproj |  26 +++++++++++
 .../Program.cs                                |  32 ++++++++++++++
 .../sample-SK-Readme.pdf                      | Bin 0 -> 186802 bytes
 .../Core/AI/SemanticKernelTextGeneration.cs   |  41 ++++++++++++++++++
 service/Core/KernelMemoryBuilderExtensions.cs |  10 +++++
 6 files changed, 116 insertions(+)
 create mode 100644 examples/105-dotnet-SemanticKernel-TextCompletion/105-dotnet-SemanticKernel-TextCompletion.csproj
 create mode 100644 examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs
 create mode 100644 examples/105-dotnet-SemanticKernel-TextCompletion/sample-SK-Readme.pdf
 create mode 100644 service/Core/AI/SemanticKernelTextGeneration.cs

diff --git a/KernelMemory.sln b/KernelMemory.sln
index 491c4695d..87979ffa1 100644
--- a/KernelMemory.sln
+++ b/KernelMemory.sln
@@ -179,6 +179,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "204-dotnet-ASP.NET-MVC-inte
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "205-dotnet-extract-text-from-docs", "examples\205-dotnet-extract-text-from-docs\205-dotnet-extract-text-from-docs.csproj", "{9202380E-D793-4C3C-89B3-3BE790925029}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "105-dotnet-SemanticKernel-TextCompletion", "examples\105-dotnet-SemanticKernel-TextCompletion\105-dotnet-SemanticKernel-TextCompletion.csproj", "{F866C3E9-1DD4-4D5A-A30F-079FA195EC3C}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -221,6 +223,7 @@ Global
 		{1B41A9A1-0B57-4297-B579-D04FC0BA3227} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841}
 		{10A6C940-AECC-4CEF-963E-9908DDA816B2} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841}
 		{9202380E-D793-4C3C-89B3-3BE790925029} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841}
+		{F866C3E9-1DD4-4D5A-A30F-079FA195EC3C} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841}
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{8A9FA587-7EBA-4D43-BE47-38D798B1C74C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
@@ -327,5 +330,9 @@ Global
 		{9202380E-D793-4C3C-89B3-3BE790925029}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{9202380E-D793-4C3C-89B3-3BE790925029}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{9202380E-D793-4C3C-89B3-3BE790925029}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F866C3E9-1DD4-4D5A-A30F-079FA195EC3C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F866C3E9-1DD4-4D5A-A30F-079FA195EC3C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F866C3E9-1DD4-4D5A-A30F-079FA195EC3C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F866C3E9-1DD4-4D5A-A30F-079FA195EC3C}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal
diff --git a/examples/105-dotnet-SemanticKernel-TextCompletion/105-dotnet-SemanticKernel-TextCompletion.csproj b/examples/105-dotnet-SemanticKernel-TextCompletion/105-dotnet-SemanticKernel-TextCompletion.csproj
new file mode 100644
index 000000000..8d5c47377
--- /dev/null
+++ b/examples/105-dotnet-SemanticKernel-TextCompletion/105-dotnet-SemanticKernel-TextCompletion.csproj
@@ -0,0 +1,26 @@
+﻿<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net8.0</TargetFramework>
+        <RootNamespace>_105_dotnet_SemanticKernel_TextCompletion</RootNamespace>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+        <UserSecretsId>5ee045b0-aea3-4f08-8d31-32d1a6f8fed0</UserSecretsId>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <None Remove="sample-SK-Readme.pdf" />
+    </ItemGroup>
+
+    <ItemGroup>
+        <Content Include="sample-SK-Readme.pdf">
+            <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+        </Content>
+    </ItemGroup>
+
+    <ItemGroup>
+        <ProjectReference Include="..\..\service\Core\Core.csproj" />
+    </ItemGroup>
+
+</Project>
diff --git a/examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs b/examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs
new file mode 100644
index 000000000..89d4d38e5
--- /dev/null
+++ b/examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs
@@ -0,0 +1,32 @@
+﻿// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.KernelMemory;
+using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletion;
+using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextEmbedding;
+
+var endpoint = Env.Var("AOAI_ENDPOINT");
+var apiKey = Env.Var("AOAI_API_KEY");
+var chatDeployment = Env.Var("AOAI_DEPLOYMENT_CHAT");
+var embeddingDeployment = Env.Var("AOAI_DEPLOYMENT_EMBEDDING");
+
+var memory = new KernelMemoryBuilder()
+    .WithSemanticKernelTextCompletion(new AzureOpenAIChatCompletion(chatDeployment, endpoint, apiKey))
+    .WithCustomEmbeddingGeneration(new AzureOpenAITextEmbeddingGeneration(embeddingDeployment, endpoint, apiKey))
+    .Build<MemoryServerless>();
+
+await memory.ImportDocumentAsync("sample-SK-Readme.pdf", documentId: "doc001");
+
+var question = "What's Semantic Kernel?";
+
+Console.WriteLine($"\n\nQuestion: {question}");
+
+var answer = await memory.AskAsync(question);
+
+Console.WriteLine($"\nAnswer: {answer.Result}");
+
+Console.WriteLine("\n\n  Sources:\n");
+
+foreach (var x in answer.RelevantSources)
+{
+    Console.WriteLine($"  - {x.SourceName}  - {x.Link} [{x.Partitions.First().LastUpdate:D}]");
+}
diff --git a/examples/105-dotnet-SemanticKernel-TextCompletion/sample-SK-Readme.pdf b/examples/105-dotnet-SemanticKernel-TextCompletion/sample-SK-Readme.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..c4a87c0ef87c1e6ce39f8a516c8e2d07cae65f85
GIT binary patch
diff --git a/service/Core/AI/SemanticKernelTextGeneration.cs b/service/Core/AI/SemanticKernelTextGeneration.cs
new file mode 100644
index 000000000..3c857d397
--- /dev/null
+++ b/service/Core/AI/SemanticKernelTextGeneration.cs
@@ -0,0 +1,41 @@
+﻿// Copyright (c) Microsoft. All rights reserved.
+
+using System.Collections.Generic;
+using System.Threading;
+using Microsoft.SemanticKernel.AI;
+using Microsoft.SemanticKernel.AI.TextCompletion;
+
+namespace Microsoft.KernelMemory.AI;
+internal class SemanticKernelTextGeneration : ITextGeneration
+{
+    private readonly ITextCompletion _completion;
+
+    public SemanticKernelTextGeneration(ITextCompletion completion)
+    {
+        this._completion = completion;
+    }
+
+    public IAsyncEnumerable<string> GenerateTextAsync(string prompt, TextGenerationOptions options, CancellationToken cancellationToken = default)
+    {
+        return this._completion.CompleteStreamAsync(prompt, this.ToAIRequestSettings(options), cancellationToken);
+    }
+
+    private AIRequestSettings ToAIRequestSettings(TextGenerationOptions options)
+    {
+        var settings = new AIRequestSettings();
+        settings.ExtensionData[nameof(options.Temperature)] = options.Temperature;
+        settings.ExtensionData[nameof(options.TopP)] = options.TopP;
+        settings.ExtensionData[nameof(options.PresencePenalty)] = options.PresencePenalty;
+        settings.ExtensionData[nameof(options.FrequencyPenalty)] = options.FrequencyPenalty;
+        settings.ExtensionData[nameof(options.StopSequences)] = options.StopSequences;
+
+        if (options.MaxTokens != null)
+        {
+            settings.ExtensionData[nameof(options.MaxTokens)] = options.MaxTokens;
+        }
+
+        settings.ExtensionData[nameof(options.ResultsPerPrompt)] = options.ResultsPerPrompt;
+        settings.ExtensionData[nameof(options.TokenSelectionBiases)] = options.TokenSelectionBiases;
+        return settings;
+    }
+}
diff --git a/service/Core/KernelMemoryBuilderExtensions.cs b/service/Core/KernelMemoryBuilderExtensions.cs
index 2ff887172..c9fe5f57c 100644
--- a/service/Core/KernelMemoryBuilderExtensions.cs
+++ b/service/Core/KernelMemoryBuilderExtensions.cs
@@ -9,6 +9,7 @@
 using Microsoft.KernelMemory.Pipeline.Queue;
 using Microsoft.KernelMemory.Prompts;
 using Microsoft.SemanticKernel.AI.Embeddings;
+using Microsoft.SemanticKernel.AI.TextCompletion;
 
 namespace Microsoft.KernelMemory;
 
@@ -136,4 +137,13 @@ public static IKernelMemoryBuilder WithCustomPromptProvider(
         builder.AddSingleton<IPromptProvider>(service);
         return builder;
     }
+
+    public static IKernelMemoryBuilder WithSemanticKernelTextCompletion(
+        this IKernelMemoryBuilder builder,
+        ITextCompletion service)
+    {
+        service = service ?? throw new ConfigurationException("The semantic kernel text completion instance is NULL");
+        builder.AddSingleton<ITextGeneration>(new SemanticKernelTextGeneration(service));
+        return builder;
+    }
 }

From 0294d8998501357ade3831f2461f27ab465d7fd4 Mon Sep 17 00:00:00 2001
From: xbotter <xbotter@live.cn>
Date: Sun, 3 Dec 2023 15:46:06 +0800
Subject: [PATCH 02/12] Remove duplicate pdf file.

---
 ...dotnet-SemanticKernel-TextCompletion.csproj |  10 ----------
 .../Program.cs                                 |   2 ++
 .../sample-SK-Readme.pdf                       | Bin 186802 -> 0 bytes
 3 files changed, 2 insertions(+), 10 deletions(-)
 delete mode 100644 examples/105-dotnet-SemanticKernel-TextCompletion/sample-SK-Readme.pdf

diff --git a/examples/105-dotnet-SemanticKernel-TextCompletion/105-dotnet-SemanticKernel-TextCompletion.csproj b/examples/105-dotnet-SemanticKernel-TextCompletion/105-dotnet-SemanticKernel-TextCompletion.csproj
index 8d5c47377..c67e3d7ac 100644
--- a/examples/105-dotnet-SemanticKernel-TextCompletion/105-dotnet-SemanticKernel-TextCompletion.csproj
+++ b/examples/105-dotnet-SemanticKernel-TextCompletion/105-dotnet-SemanticKernel-TextCompletion.csproj
@@ -9,16 +9,6 @@
         <UserSecretsId>5ee045b0-aea3-4f08-8d31-32d1a6f8fed0</UserSecretsId>
     </PropertyGroup>
 
-    <ItemGroup>
-        <None Remove="sample-SK-Readme.pdf" />
-    </ItemGroup>
-
-    <ItemGroup>
-        <Content Include="sample-SK-Readme.pdf">
-            <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-        </Content>
-    </ItemGroup>
-
     <ItemGroup>
         <ProjectReference Include="..\..\service\Core\Core.csproj" />
     </ItemGroup>
diff --git a/examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs b/examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs
index 89d4d38e5..08d53aa33 100644
--- a/examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs
+++ b/examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs
@@ -14,6 +14,8 @@
     .WithCustomEmbeddingGeneration(new AzureOpenAITextEmbeddingGeneration(embeddingDeployment, endpoint, apiKey))
     .Build<MemoryServerless>();
 
+// using document form 203-dotnet-using-core-nuget
+
 await memory.ImportDocumentAsync("sample-SK-Readme.pdf", documentId: "doc001");
 
 var question = "What's Semantic Kernel?";
diff --git a/examples/105-dotnet-SemanticKernel-TextCompletion/sample-SK-Readme.pdf b/examples/105-dotnet-SemanticKernel-TextCompletion/sample-SK-Readme.pdf
deleted file mode 100644
index c4a87c0ef87c1e6ce39f8a516c8e2d07cae65f85..0000000000000000000000000000000000000000
GIT binary patch

From cd85e5265feffde321d1db2dacd5d8dafbaedb7c Mon Sep 17 00:00:00 2001
From: xbotter <xbotter@live.cn>
Date: Wed, 6 Dec 2023 13:55:42 +0800
Subject: [PATCH 03/12] update sk rc2

---
 ...otnet-SemanticKernel-TextCompletion.csproj |  1 +
 .../Program.cs                                |  4 ++--
 .../Core/AI/SemanticKernelTextGeneration.cs   | 22 +++++++++++++------
 service/Core/KernelMemoryBuilderExtensions.cs |  6 ++---
 4 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/examples/105-dotnet-SemanticKernel-TextCompletion/105-dotnet-SemanticKernel-TextCompletion.csproj b/examples/105-dotnet-SemanticKernel-TextCompletion/105-dotnet-SemanticKernel-TextCompletion.csproj
index c67e3d7ac..3967e5064 100644
--- a/examples/105-dotnet-SemanticKernel-TextCompletion/105-dotnet-SemanticKernel-TextCompletion.csproj
+++ b/examples/105-dotnet-SemanticKernel-TextCompletion/105-dotnet-SemanticKernel-TextCompletion.csproj
@@ -7,6 +7,7 @@
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
         <UserSecretsId>5ee045b0-aea3-4f08-8d31-32d1a6f8fed0</UserSecretsId>
+        <NoWarn>CA1711,CA1724,CA1308,CA1812,CA1859,CA2208,CS1591,NU5104,SKEXP0001,SKEXP0011</NoWarn>
     </PropertyGroup>
 
     <ItemGroup>
diff --git a/examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs b/examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs
index 08d53aa33..c3c32b493 100644
--- a/examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs
+++ b/examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs
@@ -10,8 +10,8 @@
 var embeddingDeployment = Env.Var("AOAI_DEPLOYMENT_EMBEDDING");
 
 var memory = new KernelMemoryBuilder()
-    .WithSemanticKernelTextCompletion(new AzureOpenAIChatCompletion(chatDeployment, endpoint, apiKey))
-    .WithCustomEmbeddingGeneration(new AzureOpenAITextEmbeddingGeneration(embeddingDeployment, endpoint, apiKey))
+    .WithSemanticKernelTextCompletion(new AzureOpenAIChatCompletionService(chatDeployment, "", endpoint, apiKey))
+    .WithCustomEmbeddingGeneration(new AzureOpenAITextEmbeddingGeneration(embeddingDeployment, "", endpoint, apiKey))
     .Build<MemoryServerless>();
 
 // using document form 203-dotnet-using-core-nuget
diff --git a/service/Core/AI/SemanticKernelTextGeneration.cs b/service/Core/AI/SemanticKernelTextGeneration.cs
index 3c857d397..2a9ce806a 100644
--- a/service/Core/AI/SemanticKernelTextGeneration.cs
+++ b/service/Core/AI/SemanticKernelTextGeneration.cs
@@ -1,28 +1,36 @@
 ﻿// Copyright (c) Microsoft. All rights reserved.
 
 using System.Collections.Generic;
+using System.Runtime.CompilerServices;
 using System.Threading;
 using Microsoft.SemanticKernel.AI;
-using Microsoft.SemanticKernel.AI.TextCompletion;
+using Microsoft.SemanticKernel.AI.TextGeneration;
 
 namespace Microsoft.KernelMemory.AI;
 internal class SemanticKernelTextGeneration : ITextGeneration
 {
-    private readonly ITextCompletion _completion;
+    private readonly ITextGenerationService _completion;
 
-    public SemanticKernelTextGeneration(ITextCompletion completion)
+    public SemanticKernelTextGeneration(ITextGenerationService completion)
     {
         this._completion = completion;
     }
 
-    public IAsyncEnumerable<string> GenerateTextAsync(string prompt, TextGenerationOptions options, CancellationToken cancellationToken = default)
+    public async IAsyncEnumerable<string> GenerateTextAsync(string prompt, TextGenerationOptions options, [EnumeratorCancellation] CancellationToken cancellationToken = default)
     {
-        return this._completion.CompleteStreamAsync(prompt, this.ToAIRequestSettings(options), cancellationToken);
+        var contents = this._completion.GetStreamingTextContentsAsync(prompt, this.ToAIRequestSettings(options), null, cancellationToken);
+        await foreach (var content in contents)
+        {
+            if (content != null)
+            {
+                yield return content.ToString();
+            }
+        }
     }
 
-    private AIRequestSettings ToAIRequestSettings(TextGenerationOptions options)
+    private PromptExecutionSettings ToAIRequestSettings(TextGenerationOptions options)
     {
-        var settings = new AIRequestSettings();
+        var settings = new PromptExecutionSettings();
         settings.ExtensionData[nameof(options.Temperature)] = options.Temperature;
         settings.ExtensionData[nameof(options.TopP)] = options.TopP;
         settings.ExtensionData[nameof(options.PresencePenalty)] = options.PresencePenalty;
diff --git a/service/Core/KernelMemoryBuilderExtensions.cs b/service/Core/KernelMemoryBuilderExtensions.cs
index c9fe5f57c..dae2ac09a 100644
--- a/service/Core/KernelMemoryBuilderExtensions.cs
+++ b/service/Core/KernelMemoryBuilderExtensions.cs
@@ -9,7 +9,7 @@
 using Microsoft.KernelMemory.Pipeline.Queue;
 using Microsoft.KernelMemory.Prompts;
 using Microsoft.SemanticKernel.AI.Embeddings;
-using Microsoft.SemanticKernel.AI.TextCompletion;
+using Microsoft.SemanticKernel.AI.TextGeneration;
 
 namespace Microsoft.KernelMemory;
 
@@ -140,9 +140,9 @@ public static IKernelMemoryBuilder WithCustomPromptProvider(
 
     public static IKernelMemoryBuilder WithSemanticKernelTextCompletion(
         this IKernelMemoryBuilder builder,
-        ITextCompletion service)
+        ITextGenerationService service)
     {
-        service = service ?? throw new ConfigurationException("The semantic kernel text completion instance is NULL");
+        service = service ?? throw new ConfigurationException("The semantic kernel text generation service instance is NULL");
         builder.AddSingleton<ITextGeneration>(new SemanticKernelTextGeneration(service));
         return builder;
     }

From 5a4cb4e8adb128cb329a3f6d03c3ae24cc82f8c3 Mon Sep 17 00:00:00 2001
From: xbotter <xbotter@live.cn>
Date: Wed, 6 Dec 2023 14:02:44 +0800
Subject: [PATCH 04/12] rename WithSemanticKernelTextGenerationService

---
 examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs | 2 +-
 service/Core/KernelMemoryBuilderExtensions.cs                | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs b/examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs
index c3c32b493..28048cb64 100644
--- a/examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs
+++ b/examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs
@@ -10,7 +10,7 @@
 var embeddingDeployment = Env.Var("AOAI_DEPLOYMENT_EMBEDDING");
 
 var memory = new KernelMemoryBuilder()
-    .WithSemanticKernelTextCompletion(new AzureOpenAIChatCompletionService(chatDeployment, "", endpoint, apiKey))
+    .WithSemanticKernelTextGenerationService(new AzureOpenAIChatCompletionService(chatDeployment, "", endpoint, apiKey))
     .WithCustomEmbeddingGeneration(new AzureOpenAITextEmbeddingGeneration(embeddingDeployment, "", endpoint, apiKey))
     .Build<MemoryServerless>();
 
diff --git a/service/Core/KernelMemoryBuilderExtensions.cs b/service/Core/KernelMemoryBuilderExtensions.cs
index dae2ac09a..b299411f6 100644
--- a/service/Core/KernelMemoryBuilderExtensions.cs
+++ b/service/Core/KernelMemoryBuilderExtensions.cs
@@ -138,7 +138,7 @@ public static IKernelMemoryBuilder WithCustomPromptProvider(
         return builder;
     }
 
-    public static IKernelMemoryBuilder WithSemanticKernelTextCompletion(
+    public static IKernelMemoryBuilder WithSemanticKernelTextGenerationService(
         this IKernelMemoryBuilder builder,
         ITextGenerationService service)
     {

From 64489f1130549073a8eb32a2ed7b94de9db16706 Mon Sep 17 00:00:00 2001
From: Devis Lucato <devis@microsoft.com>
Date: Tue, 5 Dec 2023 22:43:17 -0800
Subject: [PATCH 05/12] move code to Abstractions plus some refactoring

---
 .../KernelMemoryBuilderExtensions.cs          | 28 +++++++++
 .../SemanticKernelTextGeneration.cs           | 63 +++++++++++++++++++
 .../Core/AI/SemanticKernelTextGeneration.cs   | 49 ---------------
 service/Core/KernelMemoryBuilderExtensions.cs | 10 ---
 4 files changed, 91 insertions(+), 59 deletions(-)
 create mode 100644 service/Abstractions/SemanticKernel/KernelMemoryBuilderExtensions.cs
 create mode 100644 service/Abstractions/SemanticKernel/SemanticKernelTextGeneration.cs
 delete mode 100644 service/Core/AI/SemanticKernelTextGeneration.cs

diff --git a/service/Abstractions/SemanticKernel/KernelMemoryBuilderExtensions.cs b/service/Abstractions/SemanticKernel/KernelMemoryBuilderExtensions.cs
new file mode 100644
index 000000000..39cff2023
--- /dev/null
+++ b/service/Abstractions/SemanticKernel/KernelMemoryBuilderExtensions.cs
@@ -0,0 +1,28 @@
+﻿// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.KernelMemory.AI;
+using Microsoft.KernelMemory.Configuration;
+using Microsoft.SemanticKernel.AI.TextGeneration;
+
+namespace Microsoft.KernelMemory;
+
+/// <summary>
+/// Kernel Memory builder extensions.
+/// </summary>
+public static partial class KernelMemoryBuilderExtensions
+{
+    /// <summary>
+    /// Inject an implementation of <see cref="ITextGenerationService">SK text generation service</see>
+    /// for local dependencies on <see cref="ITextGeneration"/>
+    /// </summary>
+    /// <param name="builder">KM builder</param>
+    /// <param name="service">SK text generation service instance</param>
+    /// <returns>KM builder</returns>
+    public static IKernelMemoryBuilder WithSemanticKernelTextGenerationService(
+        this IKernelMemoryBuilder builder,
+        ITextGenerationService service)
+    {
+        service = service ?? throw new ConfigurationException("The semantic kernel text generation service instance is NULL");
+        return builder.AddSingleton<ITextGeneration>(new SemanticKernelTextGeneration(service));
+    }
+}
diff --git a/service/Abstractions/SemanticKernel/SemanticKernelTextGeneration.cs b/service/Abstractions/SemanticKernel/SemanticKernelTextGeneration.cs
new file mode 100644
index 000000000..249ea5412
--- /dev/null
+++ b/service/Abstractions/SemanticKernel/SemanticKernelTextGeneration.cs
@@ -0,0 +1,63 @@
+﻿// Copyright (c) Microsoft. All rights reserved.
+
+using System.Collections.Generic;
+using System.Runtime.CompilerServices;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.SemanticKernel.AI;
+using Microsoft.SemanticKernel.AI.TextGeneration;
+
+namespace Microsoft.KernelMemory.AI;
+
+internal class SemanticKernelTextGeneration : ITextGeneration
+{
+    private readonly ITextGenerationService _service;
+
+    public SemanticKernelTextGeneration(ITextGenerationService service)
+    {
+        this._service = service;
+    }
+
+    /// <inheritdoc />
+    public async IAsyncEnumerable<string> GenerateTextAsync(
+        string prompt,
+        TextGenerationOptions options,
+        [EnumeratorCancellation] CancellationToken cancellationToken = default)
+    {
+        var contents = this._service.GetStreamingTextContentsAsync(
+            prompt, ToPromptExecutionSettings(options), null, cancellationToken).ConfigureAwait(false);
+
+        await foreach (StreamingTextContent? content in contents)
+        {
+            if (content != null)
+            {
+                yield return content.ToString();
+            }
+        }
+    }
+
+    private static PromptExecutionSettings ToPromptExecutionSettings(TextGenerationOptions options)
+    {
+        var settings = new PromptExecutionSettings
+        {
+            ExtensionData =
+            {
+                [nameof(options.Temperature)] = options.Temperature,
+                [nameof(options.TopP)] = options.TopP,
+                [nameof(options.PresencePenalty)] = options.PresencePenalty,
+                [nameof(options.FrequencyPenalty)] = options.FrequencyPenalty,
+                [nameof(options.StopSequences)] = options.StopSequences
+            }
+        };
+
+        if (options.MaxTokens != null)
+        {
+            settings.ExtensionData[nameof(options.MaxTokens)] = options.MaxTokens;
+        }
+
+        settings.ExtensionData[nameof(options.ResultsPerPrompt)] = options.ResultsPerPrompt;
+        settings.ExtensionData[nameof(options.TokenSelectionBiases)] = options.TokenSelectionBiases;
+
+        return settings;
+    }
+}
diff --git a/service/Core/AI/SemanticKernelTextGeneration.cs b/service/Core/AI/SemanticKernelTextGeneration.cs
deleted file mode 100644
index 2a9ce806a..000000000
--- a/service/Core/AI/SemanticKernelTextGeneration.cs
+++ /dev/null
@@ -1,49 +0,0 @@
-﻿// Copyright (c) Microsoft. All rights reserved.
-
-using System.Collections.Generic;
-using System.Runtime.CompilerServices;
-using System.Threading;
-using Microsoft.SemanticKernel.AI;
-using Microsoft.SemanticKernel.AI.TextGeneration;
-
-namespace Microsoft.KernelMemory.AI;
-internal class SemanticKernelTextGeneration : ITextGeneration
-{
-    private readonly ITextGenerationService _completion;
-
-    public SemanticKernelTextGeneration(ITextGenerationService completion)
-    {
-        this._completion = completion;
-    }
-
-    public async IAsyncEnumerable<string> GenerateTextAsync(string prompt, TextGenerationOptions options, [EnumeratorCancellation] CancellationToken cancellationToken = default)
-    {
-        var contents = this._completion.GetStreamingTextContentsAsync(prompt, this.ToAIRequestSettings(options), null, cancellationToken);
-        await foreach (var content in contents)
-        {
-            if (content != null)
-            {
-                yield return content.ToString();
-            }
-        }
-    }
-
-    private PromptExecutionSettings ToAIRequestSettings(TextGenerationOptions options)
-    {
-        var settings = new PromptExecutionSettings();
-        settings.ExtensionData[nameof(options.Temperature)] = options.Temperature;
-        settings.ExtensionData[nameof(options.TopP)] = options.TopP;
-        settings.ExtensionData[nameof(options.PresencePenalty)] = options.PresencePenalty;
-        settings.ExtensionData[nameof(options.FrequencyPenalty)] = options.FrequencyPenalty;
-        settings.ExtensionData[nameof(options.StopSequences)] = options.StopSequences;
-
-        if (options.MaxTokens != null)
-        {
-            settings.ExtensionData[nameof(options.MaxTokens)] = options.MaxTokens;
-        }
-
-        settings.ExtensionData[nameof(options.ResultsPerPrompt)] = options.ResultsPerPrompt;
-        settings.ExtensionData[nameof(options.TokenSelectionBiases)] = options.TokenSelectionBiases;
-        return settings;
-    }
-}
diff --git a/service/Core/KernelMemoryBuilderExtensions.cs b/service/Core/KernelMemoryBuilderExtensions.cs
index b299411f6..2ff887172 100644
--- a/service/Core/KernelMemoryBuilderExtensions.cs
+++ b/service/Core/KernelMemoryBuilderExtensions.cs
@@ -9,7 +9,6 @@
 using Microsoft.KernelMemory.Pipeline.Queue;
 using Microsoft.KernelMemory.Prompts;
 using Microsoft.SemanticKernel.AI.Embeddings;
-using Microsoft.SemanticKernel.AI.TextGeneration;
 
 namespace Microsoft.KernelMemory;
 
@@ -137,13 +136,4 @@ public static IKernelMemoryBuilder WithCustomPromptProvider(
         builder.AddSingleton<IPromptProvider>(service);
         return builder;
     }
-
-    public static IKernelMemoryBuilder WithSemanticKernelTextGenerationService(
-        this IKernelMemoryBuilder builder,
-        ITextGenerationService service)
-    {
-        service = service ?? throw new ConfigurationException("The semantic kernel text generation service instance is NULL");
-        builder.AddSingleton<ITextGeneration>(new SemanticKernelTextGeneration(service));
-        return builder;
-    }
 }

From 7fb90ee2500e933fd189ecfc28a96224a27a828b Mon Sep 17 00:00:00 2001
From: xbotter <xbotter@live.cn>
Date: Wed, 6 Dec 2023 15:00:39 +0800
Subject: [PATCH 06/12] fix: example 105 include pdf file

---
 .../105-dotnet-SemanticKernel-TextCompletion.csproj         | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/examples/105-dotnet-SemanticKernel-TextCompletion/105-dotnet-SemanticKernel-TextCompletion.csproj b/examples/105-dotnet-SemanticKernel-TextCompletion/105-dotnet-SemanticKernel-TextCompletion.csproj
index 3967e5064..42b2c4fa6 100644
--- a/examples/105-dotnet-SemanticKernel-TextCompletion/105-dotnet-SemanticKernel-TextCompletion.csproj
+++ b/examples/105-dotnet-SemanticKernel-TextCompletion/105-dotnet-SemanticKernel-TextCompletion.csproj
@@ -10,6 +10,12 @@
         <NoWarn>CA1711,CA1724,CA1308,CA1812,CA1859,CA2208,CS1591,NU5104,SKEXP0001,SKEXP0011</NoWarn>
     </PropertyGroup>
 
+    <ItemGroup>
+      <None Include="..\203-dotnet-using-core-nuget\sample-SK-Readme.pdf" Link="sample-SK-Readme.pdf">
+        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      </None>
+    </ItemGroup>
+
     <ItemGroup>
         <ProjectReference Include="..\..\service\Core\Core.csproj" />
     </ItemGroup>

From 8f15615171c1edacc87519d2f6df6672c05d5746 Mon Sep 17 00:00:00 2001
From: xbotter <xbotter@live.cn>
Date: Fri, 8 Dec 2023 11:12:23 +0800
Subject: [PATCH 07/12] =?UTF-8?q?=F0=9F=94=A7=20Refactor=20Semantic=20Kern?=
 =?UTF-8?q?el=20Text=20Generation=20and=20Embedding=20Generation?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Refactored the Semantic Kernel Text Generation and Embedding Generation classes to improve code organization and readability.
- Added a new class, SemanticKernelConfig, to store configuration settings for the Text Generator and Embedding Generator.
- Updated the KernelMemoryBuilderExtensions class to include methods for injecting the new Text Generator and Embedding Generator implementations.
- The Text Generator and Embedding Generator now use the provided configuration and tokenizer instances.
- Renamed the SemanticKernelTextGeneration class to SemanticKernelTextGenerator to better reflect its purpose.
- The SemanticKernelTextEmbeddingGenerator class now implements the ITextEmbeddingGenerator interface and uses the provided Text Tokenizer to count tokens.
- The GenerateEmbeddingAsync method in the SemanticKernelTextEmbeddingGenerator class now calls the GenerateEmbeddingAsync method in the ITextEmbeddingGeneration instance.
- The CountTokens method in the SemanticKernelTextEmbeddingGenerator class now calls the CountTokens method in the provided Text Tokenizer.
---
 .../Program.cs                                | 12 ++++-
 .../KernelMemoryBuilderExtensions.cs          | 45 +++++++++++++++++--
 .../SemanticKernel/SemanticKernelConfig.cs    | 10 +++++
 .../SemanticKernelTextEmbeddingGenerator.cs   | 34 ++++++++++++++
 ...tion.cs => SemanticKernelTextGenerator.cs} | 21 ++++++---
 5 files changed, 111 insertions(+), 11 deletions(-)
 create mode 100644 service/Abstractions/SemanticKernel/SemanticKernelConfig.cs
 create mode 100644 service/Abstractions/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs
 rename service/Abstractions/SemanticKernel/{SemanticKernelTextGeneration.cs => SemanticKernelTextGenerator.cs} (71%)

diff --git a/examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs b/examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs
index 28048cb64..6c50a50ea 100644
--- a/examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs
+++ b/examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs
@@ -1,6 +1,7 @@
 ﻿// Copyright (c) Microsoft. All rights reserved.
 
 using Microsoft.KernelMemory;
+using Microsoft.KernelMemory.AI.Tokenizers;
 using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletion;
 using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextEmbedding;
 
@@ -9,9 +10,16 @@
 var chatDeployment = Env.Var("AOAI_DEPLOYMENT_CHAT");
 var embeddingDeployment = Env.Var("AOAI_DEPLOYMENT_EMBEDDING");
 
+var config = new SemanticKernelConfig();
+var tokenizer = new DefaultGPTTokenizer();
+
 var memory = new KernelMemoryBuilder()
-    .WithSemanticKernelTextGenerationService(new AzureOpenAIChatCompletionService(chatDeployment, "", endpoint, apiKey))
-    .WithCustomEmbeddingGeneration(new AzureOpenAITextEmbeddingGeneration(embeddingDeployment, "", endpoint, apiKey))
+    .WithSemanticKernelTextGenerationService(new AzureOpenAIChatCompletionService(chatDeployment, "", endpoint, apiKey)
+                                            , config
+                                            , tokenizer)
+    .WithSemanticKernelTextEmbeddingGeneration(new AzureOpenAITextEmbeddingGeneration(embeddingDeployment, "", endpoint, apiKey)
+                                            , config
+                                            , tokenizer)
     .Build<MemoryServerless>();
 
 // using document form 203-dotnet-using-core-nuget
diff --git a/service/Abstractions/SemanticKernel/KernelMemoryBuilderExtensions.cs b/service/Abstractions/SemanticKernel/KernelMemoryBuilderExtensions.cs
index 39cff2023..526f90ac7 100644
--- a/service/Abstractions/SemanticKernel/KernelMemoryBuilderExtensions.cs
+++ b/service/Abstractions/SemanticKernel/KernelMemoryBuilderExtensions.cs
@@ -2,6 +2,7 @@
 
 using Microsoft.KernelMemory.AI;
 using Microsoft.KernelMemory.Configuration;
+using Microsoft.SemanticKernel.AI.Embeddings;
 using Microsoft.SemanticKernel.AI.TextGeneration;
 
 namespace Microsoft.KernelMemory;
@@ -13,16 +14,54 @@ public static partial class KernelMemoryBuilderExtensions
 {
     /// <summary>
     /// Inject an implementation of <see cref="ITextGenerationService">SK text generation service</see>
-    /// for local dependencies on <see cref="ITextGeneration"/>
+    /// for local dependencies on <see cref="ITextGenerator"/>
     /// </summary>
     /// <param name="builder">KM builder</param>
     /// <param name="service">SK text generation service instance</param>
+    /// <param name="config">SK text generator settings</param>
+    /// <param name="tokenizer">Tokenizer used to count tokens used by prompts</param>
     /// <returns>KM builder</returns>
     public static IKernelMemoryBuilder WithSemanticKernelTextGenerationService(
         this IKernelMemoryBuilder builder,
-        ITextGenerationService service)
+        ITextGenerationService service,
+        SemanticKernelConfig config,
+        ITextTokenizer tokenizer)
     {
         service = service ?? throw new ConfigurationException("The semantic kernel text generation service instance is NULL");
-        return builder.AddSingleton<ITextGeneration>(new SemanticKernelTextGeneration(service));
+        tokenizer = tokenizer ?? throw new ConfigurationException("The text tokenizer instance is NULL");
+
+        return builder.AddSingleton<ITextGenerator>(new SemanticKernelTextGenerator(service, config, tokenizer));
+    }
+
+    /// <summary>
+    ///Inject an implementation of<see cref="ITextEmbeddingGeneration">SK text embedding generation</see>
+    /// for local dependencies on <see cref="ITextEmbeddingGenerator"/>
+    /// </summary>
+    /// <param name="builder">KM builder</param>
+    /// <param name="service">SK text embedding generation instance</param>
+    /// <param name="config">SK text embedding generator settings</param>
+    /// <param name="tokenizer">Tokenizer used to count tokens sent to the embedding generator</param>
+    /// <param name="onlyForRetrieval">Whether to use this embedding generator only during data ingestion, and not for retrieval (search and ask API)</param>
+    /// <returns>KM builder</returns>
+    public static IKernelMemoryBuilder WithSemanticKernelTextEmbeddingGeneration(
+        this IKernelMemoryBuilder builder,
+        ITextEmbeddingGeneration service,
+        SemanticKernelConfig config,
+        ITextTokenizer tokenizer,
+        bool onlyForRetrieval = false)
+    {
+        service = service ?? throw new ConfigurationException("The semantic kernel text embedding generation service instance is NULL");
+        tokenizer = tokenizer ?? throw new ConfigurationException("The text tokenizer instance is NULL");
+
+        var generator = new SemanticKernelTextEmbeddingGenerator(service, config, tokenizer);
+
+        builder.AddSingleton<ITextEmbeddingGenerator>(generator);
+
+        if (!onlyForRetrieval)
+        {
+            builder.AddIngestionEmbeddingGenerator(generator);
+        }
+
+        return builder;
     }
 }
diff --git a/service/Abstractions/SemanticKernel/SemanticKernelConfig.cs b/service/Abstractions/SemanticKernel/SemanticKernelConfig.cs
new file mode 100644
index 000000000..7731f17eb
--- /dev/null
+++ b/service/Abstractions/SemanticKernel/SemanticKernelConfig.cs
@@ -0,0 +1,10 @@
+﻿// Copyright (c) Microsoft. All rights reserved.
+
+namespace Microsoft.KernelMemory;
+/// <summary>
+/// Semantic Kernel TextGenerator And TextEmbeddingGenerator Config
+/// </summary>
+public class SemanticKernelConfig
+{
+    public int MaxTokenTotal { get; set; } = 8191;
+}
diff --git a/service/Abstractions/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs b/service/Abstractions/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs
new file mode 100644
index 000000000..cf864c921
--- /dev/null
+++ b/service/Abstractions/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs
@@ -0,0 +1,34 @@
+﻿// Copyright (c) Microsoft. All rights reserved.
+
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.KernelMemory.AI;
+using Microsoft.SemanticKernel.AI.Embeddings;
+
+namespace Microsoft.KernelMemory;
+internal class SemanticKernelTextEmbeddingGenerator : ITextEmbeddingGenerator
+{
+    private readonly ITextTokenizer _textTokenizer;
+    private readonly ITextEmbeddingGeneration _generation;
+    private readonly SemanticKernelConfig _config;
+
+    public int MaxTokens { get; }
+
+    public SemanticKernelTextEmbeddingGenerator(ITextEmbeddingGeneration generation,
+                                                SemanticKernelConfig config,
+                                                ITextTokenizer tokenizer)
+    {
+        this._generation = generation;
+        this._config = config;
+        this._textTokenizer = tokenizer;
+
+        this.MaxTokens = config.MaxTokenTotal;
+    }
+
+    /// <inheritdoc />
+    public int CountTokens(string text) => this._textTokenizer.CountTokens(text);
+
+    /// <inheritdoc />
+    public Task<Embedding> GenerateEmbeddingAsync(string text, CancellationToken cancellationToken = default)
+        => this._generation.GenerateEmbeddingAsync(text, cancellationToken);
+}
diff --git a/service/Abstractions/SemanticKernel/SemanticKernelTextGeneration.cs b/service/Abstractions/SemanticKernel/SemanticKernelTextGenerator.cs
similarity index 71%
rename from service/Abstractions/SemanticKernel/SemanticKernelTextGeneration.cs
rename to service/Abstractions/SemanticKernel/SemanticKernelTextGenerator.cs
index 249ea5412..ad97dcca0 100644
--- a/service/Abstractions/SemanticKernel/SemanticKernelTextGeneration.cs
+++ b/service/Abstractions/SemanticKernel/SemanticKernelTextGenerator.cs
@@ -9,13 +9,20 @@
 
 namespace Microsoft.KernelMemory.AI;
 
-internal class SemanticKernelTextGeneration : ITextGeneration
+internal class SemanticKernelTextGenerator : ITextGenerator
 {
     private readonly ITextGenerationService _service;
+    private readonly ITextTokenizer _tokenizer;
+    private readonly SemanticKernelConfig _config;
 
-    public SemanticKernelTextGeneration(ITextGenerationService service)
+    public int MaxTokenTotal { get; }
+
+    public SemanticKernelTextGenerator(ITextGenerationService service, SemanticKernelConfig config, ITextTokenizer tokenizer)
     {
         this._service = service;
+        this._config = config;
+        this._tokenizer = tokenizer;
+        this.MaxTokenTotal = config.MaxTokenTotal;
     }
 
     /// <inheritdoc />
@@ -46,7 +53,9 @@ private static PromptExecutionSettings ToPromptExecutionSettings(TextGenerationO
                 [nameof(options.TopP)] = options.TopP,
                 [nameof(options.PresencePenalty)] = options.PresencePenalty,
                 [nameof(options.FrequencyPenalty)] = options.FrequencyPenalty,
-                [nameof(options.StopSequences)] = options.StopSequences
+                [nameof(options.StopSequences)] = options.StopSequences,
+                [nameof(options.ResultsPerPrompt)] = options.ResultsPerPrompt,
+                [nameof(options.TokenSelectionBiases)] = options.TokenSelectionBiases
             }
         };
 
@@ -55,9 +64,9 @@ private static PromptExecutionSettings ToPromptExecutionSettings(TextGenerationO
             settings.ExtensionData[nameof(options.MaxTokens)] = options.MaxTokens;
         }
 
-        settings.ExtensionData[nameof(options.ResultsPerPrompt)] = options.ResultsPerPrompt;
-        settings.ExtensionData[nameof(options.TokenSelectionBiases)] = options.TokenSelectionBiases;
-
         return settings;
     }
+
+    /// <inheritdoc />
+    public int CountTokens(string text) => this._tokenizer.CountTokens(text);
 }

From f6a432196b030965115e0136201fd37d1bfb1559 Mon Sep 17 00:00:00 2001
From: xbotter <xbotter@live.cn>
Date: Fri, 8 Dec 2023 11:20:36 +0800
Subject: [PATCH 08/12] =?UTF-8?q?=F0=9F=94=A7=20Update=20SemanticKernelCon?=
 =?UTF-8?q?fig=20and=20SemanticKernelTextEmbeddingGenerator?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Update SemanticKernelTextEmbeddingGenerator.cs to add a using statement and modify the constructor to handle null arguments.
- Update SemanticKernelTextGenerator.cs to add a using statement and modify the constructor to handle null arguments.
---
 .../SemanticKernel/SemanticKernelConfig.cs           |  1 +
 .../SemanticKernelTextEmbeddingGenerator.cs          | 12 +++++++-----
 .../SemanticKernel/SemanticKernelTextGenerator.cs    |  5 +++--
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/service/Abstractions/SemanticKernel/SemanticKernelConfig.cs b/service/Abstractions/SemanticKernel/SemanticKernelConfig.cs
index 7731f17eb..8be7c0175 100644
--- a/service/Abstractions/SemanticKernel/SemanticKernelConfig.cs
+++ b/service/Abstractions/SemanticKernel/SemanticKernelConfig.cs
@@ -1,6 +1,7 @@
 ﻿// Copyright (c) Microsoft. All rights reserved.
 
 namespace Microsoft.KernelMemory;
+
 /// <summary>
 /// Semantic Kernel TextGenerator And TextEmbeddingGenerator Config
 /// </summary>
diff --git a/service/Abstractions/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs b/service/Abstractions/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs
index cf864c921..836ac1862 100644
--- a/service/Abstractions/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs
+++ b/service/Abstractions/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs
@@ -1,5 +1,6 @@
 ﻿// Copyright (c) Microsoft. All rights reserved.
 
+using System;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.KernelMemory.AI;
@@ -8,7 +9,7 @@
 namespace Microsoft.KernelMemory;
 internal class SemanticKernelTextEmbeddingGenerator : ITextEmbeddingGenerator
 {
-    private readonly ITextTokenizer _textTokenizer;
+    private readonly ITextTokenizer _tokenizer;
     private readonly ITextEmbeddingGeneration _generation;
     private readonly SemanticKernelConfig _config;
 
@@ -18,15 +19,16 @@ public SemanticKernelTextEmbeddingGenerator(ITextEmbeddingGeneration generation,
                                                 SemanticKernelConfig config,
                                                 ITextTokenizer tokenizer)
     {
-        this._generation = generation;
-        this._config = config;
-        this._textTokenizer = tokenizer;
+        this._generation = generation ?? throw new ArgumentNullException(nameof(generation));
+        this._tokenizer = tokenizer ?? throw new ArgumentNullException(nameof(tokenizer), "Tokenizer not specified. The token count might be incorrect, causing unexpected errors");
 
+        this._tokenizer = tokenizer;
+        this._config = config;
         this.MaxTokens = config.MaxTokenTotal;
     }
 
     /// <inheritdoc />
-    public int CountTokens(string text) => this._textTokenizer.CountTokens(text);
+    public int CountTokens(string text) => this._tokenizer.CountTokens(text);
 
     /// <inheritdoc />
     public Task<Embedding> GenerateEmbeddingAsync(string text, CancellationToken cancellationToken = default)
diff --git a/service/Abstractions/SemanticKernel/SemanticKernelTextGenerator.cs b/service/Abstractions/SemanticKernel/SemanticKernelTextGenerator.cs
index ad97dcca0..fe4fb2d88 100644
--- a/service/Abstractions/SemanticKernel/SemanticKernelTextGenerator.cs
+++ b/service/Abstractions/SemanticKernel/SemanticKernelTextGenerator.cs
@@ -1,5 +1,6 @@
 ﻿// Copyright (c) Microsoft. All rights reserved.
 
+using System;
 using System.Collections.Generic;
 using System.Runtime.CompilerServices;
 using System.Threading;
@@ -19,9 +20,9 @@ internal class SemanticKernelTextGenerator : ITextGenerator
 
     public SemanticKernelTextGenerator(ITextGenerationService service, SemanticKernelConfig config, ITextTokenizer tokenizer)
     {
-        this._service = service;
+        this._service = service ?? throw new ArgumentNullException(nameof(service));
+        this._tokenizer = tokenizer ?? throw new ArgumentNullException(nameof(tokenizer), "Tokenizer not specified. The token count might be incorrect, causing unexpected errors");
         this._config = config;
-        this._tokenizer = tokenizer;
         this.MaxTokenTotal = config.MaxTokenTotal;
     }
 

From 8831cb2b41f6b7774da11d4a925b98782f039b21 Mon Sep 17 00:00:00 2001
From: xbotter <xbotter@live.cn>
Date: Tue, 12 Dec 2023 12:05:25 +0800
Subject: [PATCH 09/12] rename 107-dotnet-SemanticKernel-TextCompletion project

---
 KernelMemory.sln                                     | 12 ++++++++----
 .../107-dotnet-SemanticKernel-TextCompletion.csproj} |  2 +-
 .../Program.cs                                       |  0
 3 files changed, 9 insertions(+), 5 deletions(-)
 rename examples/{105-dotnet-SemanticKernel-TextCompletion/105-dotnet-SemanticKernel-TextCompletion.csproj => 107-dotnet-SemanticKernel-TextCompletion/107-dotnet-SemanticKernel-TextCompletion.csproj} (92%)
 rename examples/{105-dotnet-SemanticKernel-TextCompletion => 107-dotnet-SemanticKernel-TextCompletion}/Program.cs (100%)

diff --git a/KernelMemory.sln b/KernelMemory.sln
index 4813ef993..1b036dea2 100644
--- a/KernelMemory.sln
+++ b/KernelMemory.sln
@@ -1,6 +1,6 @@
 ﻿
 Microsoft Visual Studio Solution File, Format Version 12.00
-
+# 
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "root", "root", "{6EF76FD8-4C35-4370-8539-5DDF45357A50}"
 	ProjectSection(SolutionItems) = preProject
 		README.md = README.md
@@ -56,7 +56,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{CA49F1A1
 		tools\create-azure-webapp-publish-artifacts.sh = tools\create-azure-webapp-publish-artifacts.sh
 	EndProjectSection
 EndProject
-
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Abstractions", "service\Abstractions\Abstractions.csproj", "{8A9FA587-7EBA-4D43-BE47-38D798B1C74C}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Core", "service\Core\Core.csproj", "{27910ADC-5A28-4EB4-A16E-974B91940758}"
@@ -77,7 +76,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SemanticKernelPlugin", "cli
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WebClient", "clients\dotnet\WebClient\WebClient.csproj", "{D04A01C0-EF1B-49B2-B6AB-8AC635566E6A}"
 EndProject
-
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "000-notebooks", "000-notebooks", "{C93FCED9-808A-4B03-9B4C-0DBAE46D5BDD}"
 	ProjectSection(SolutionItems) = preProject
 		examples\000-notebooks\001-upload-and-ask.ipynb = examples\000-notebooks\001-upload-and-ask.ipynb
@@ -185,6 +183,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "105-dotnet-serverless-llama
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "106-dotnet-retrieve-synthetics", "examples\106-dotnet-retrieve-synthetics\106-dotnet-retrieve-synthetics.csproj", "{EDFDA12E-BA10-4A00-BBE7-1C10A0B0F1A6}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "107-dotnet-SemanticKernel-TextCompletion", "examples\107-dotnet-SemanticKernel-TextCompletion\107-dotnet-SemanticKernel-TextCompletion.csproj", "{D9FEFBA3-51B0-4E9B-8E89-3BDF239EF5C3}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -230,7 +230,7 @@ Global
 		{F866C3E9-1DD4-4D5A-A30F-079FA195EC3C} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841}
 		{E762B1C6-4B5B-487A-BF8C-C6870D6EACB7} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841}
 		{EDFDA12E-BA10-4A00-BBE7-1C10A0B0F1A6} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841}
-
+		{D9FEFBA3-51B0-4E9B-8E89-3BDF239EF5C3} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841}
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{8A9FA587-7EBA-4D43-BE47-38D798B1C74C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
@@ -349,5 +349,9 @@ Global
 		{EDFDA12E-BA10-4A00-BBE7-1C10A0B0F1A6}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{EDFDA12E-BA10-4A00-BBE7-1C10A0B0F1A6}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{EDFDA12E-BA10-4A00-BBE7-1C10A0B0F1A6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D9FEFBA3-51B0-4E9B-8E89-3BDF239EF5C3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D9FEFBA3-51B0-4E9B-8E89-3BDF239EF5C3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D9FEFBA3-51B0-4E9B-8E89-3BDF239EF5C3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D9FEFBA3-51B0-4E9B-8E89-3BDF239EF5C3}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal
diff --git a/examples/105-dotnet-SemanticKernel-TextCompletion/105-dotnet-SemanticKernel-TextCompletion.csproj b/examples/107-dotnet-SemanticKernel-TextCompletion/107-dotnet-SemanticKernel-TextCompletion.csproj
similarity index 92%
rename from examples/105-dotnet-SemanticKernel-TextCompletion/105-dotnet-SemanticKernel-TextCompletion.csproj
rename to examples/107-dotnet-SemanticKernel-TextCompletion/107-dotnet-SemanticKernel-TextCompletion.csproj
index 42b2c4fa6..ba302eb1b 100644
--- a/examples/105-dotnet-SemanticKernel-TextCompletion/105-dotnet-SemanticKernel-TextCompletion.csproj
+++ b/examples/107-dotnet-SemanticKernel-TextCompletion/107-dotnet-SemanticKernel-TextCompletion.csproj
@@ -3,7 +3,7 @@
     <PropertyGroup>
         <OutputType>Exe</OutputType>
         <TargetFramework>net8.0</TargetFramework>
-        <RootNamespace>_105_dotnet_SemanticKernel_TextCompletion</RootNamespace>
+        <RootNamespace>_107_dotnet_SemanticKernel_TextCompletion</RootNamespace>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
         <UserSecretsId>5ee045b0-aea3-4f08-8d31-32d1a6f8fed0</UserSecretsId>
diff --git a/examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs b/examples/107-dotnet-SemanticKernel-TextCompletion/Program.cs
similarity index 100%
rename from examples/105-dotnet-SemanticKernel-TextCompletion/Program.cs
rename to examples/107-dotnet-SemanticKernel-TextCompletion/Program.cs

From 3bb5ee8e1a623dad15f6bf86f0bde51597058c1e Mon Sep 17 00:00:00 2001
From: xbotter <xbotter@live.cn>
Date: Tue, 19 Dec 2023 09:09:10 +0800
Subject: [PATCH 10/12] sk 1.0.1

---
 .../107-dotnet-SemanticKernel-TextCompletion/Program.cs  | 7 +++----
 .../SemanticKernel/KernelMemoryBuilderExtensions.cs      | 9 +++++----
 .../SemanticKernelTextEmbeddingGenerator.cs              | 5 +++--
 .../SemanticKernel/SemanticKernelTextGenerator.cs        | 7 +++----
 4 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/examples/107-dotnet-SemanticKernel-TextCompletion/Program.cs b/examples/107-dotnet-SemanticKernel-TextCompletion/Program.cs
index 6c50a50ea..8de54b98b 100644
--- a/examples/107-dotnet-SemanticKernel-TextCompletion/Program.cs
+++ b/examples/107-dotnet-SemanticKernel-TextCompletion/Program.cs
@@ -2,8 +2,7 @@
 
 using Microsoft.KernelMemory;
 using Microsoft.KernelMemory.AI.Tokenizers;
-using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletion;
-using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextEmbedding;
+using Microsoft.SemanticKernel.Connectors.OpenAI;
 
 var endpoint = Env.Var("AOAI_ENDPOINT");
 var apiKey = Env.Var("AOAI_API_KEY");
@@ -14,10 +13,10 @@
 var tokenizer = new DefaultGPTTokenizer();
 
 var memory = new KernelMemoryBuilder()
-    .WithSemanticKernelTextGenerationService(new AzureOpenAIChatCompletionService(chatDeployment, "", endpoint, apiKey)
+    .WithSemanticKernelTextGenerationService(new AzureOpenAIChatCompletionService(chatDeployment, endpoint, apiKey)
                                             , config
                                             , tokenizer)
-    .WithSemanticKernelTextEmbeddingGeneration(new AzureOpenAITextEmbeddingGeneration(embeddingDeployment, "", endpoint, apiKey)
+    .WithSemanticKernelTextEmbeddingGenerationService(new AzureOpenAITextEmbeddingGenerationService(embeddingDeployment, endpoint, apiKey)
                                             , config
                                             , tokenizer)
     .Build<MemoryServerless>();
diff --git a/service/Abstractions/SemanticKernel/KernelMemoryBuilderExtensions.cs b/service/Abstractions/SemanticKernel/KernelMemoryBuilderExtensions.cs
index 526f90ac7..95ea0119b 100644
--- a/service/Abstractions/SemanticKernel/KernelMemoryBuilderExtensions.cs
+++ b/service/Abstractions/SemanticKernel/KernelMemoryBuilderExtensions.cs
@@ -2,8 +2,9 @@
 
 using Microsoft.KernelMemory.AI;
 using Microsoft.KernelMemory.Configuration;
+using Microsoft.SemanticKernel.TextGeneration;
 using Microsoft.SemanticKernel.AI.Embeddings;
-using Microsoft.SemanticKernel.AI.TextGeneration;
+using Microsoft.SemanticKernel.Embeddings;
 
 namespace Microsoft.KernelMemory;
 
@@ -34,7 +35,7 @@ public static IKernelMemoryBuilder WithSemanticKernelTextGenerationService(
     }
 
     /// <summary>
-    ///Inject an implementation of<see cref="ITextEmbeddingGeneration">SK text embedding generation</see>
+    ///Inject an implementation of<see cref="ITextEmbeddingGenerationService">SK text embedding generation service</see>
     /// for local dependencies on <see cref="ITextEmbeddingGenerator"/>
     /// </summary>
     /// <param name="builder">KM builder</param>
@@ -43,9 +44,9 @@ public static IKernelMemoryBuilder WithSemanticKernelTextGenerationService(
     /// <param name="tokenizer">Tokenizer used to count tokens sent to the embedding generator</param>
     /// <param name="onlyForRetrieval">Whether to use this embedding generator only during data ingestion, and not for retrieval (search and ask API)</param>
     /// <returns>KM builder</returns>
-    public static IKernelMemoryBuilder WithSemanticKernelTextEmbeddingGeneration(
+    public static IKernelMemoryBuilder WithSemanticKernelTextEmbeddingGenerationService(
         this IKernelMemoryBuilder builder,
-        ITextEmbeddingGeneration service,
+        ITextEmbeddingGenerationService service,
         SemanticKernelConfig config,
         ITextTokenizer tokenizer,
         bool onlyForRetrieval = false)
diff --git a/service/Abstractions/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs b/service/Abstractions/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs
index 836ac1862..ac06363ae 100644
--- a/service/Abstractions/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs
+++ b/service/Abstractions/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs
@@ -5,17 +5,18 @@
 using System.Threading.Tasks;
 using Microsoft.KernelMemory.AI;
 using Microsoft.SemanticKernel.AI.Embeddings;
+using Microsoft.SemanticKernel.Embeddings;
 
 namespace Microsoft.KernelMemory;
 internal class SemanticKernelTextEmbeddingGenerator : ITextEmbeddingGenerator
 {
     private readonly ITextTokenizer _tokenizer;
-    private readonly ITextEmbeddingGeneration _generation;
+    private readonly ITextEmbeddingGenerationService _generation;
     private readonly SemanticKernelConfig _config;
 
     public int MaxTokens { get; }
 
-    public SemanticKernelTextEmbeddingGenerator(ITextEmbeddingGeneration generation,
+    public SemanticKernelTextEmbeddingGenerator(ITextEmbeddingGenerationService generation,
                                                 SemanticKernelConfig config,
                                                 ITextTokenizer tokenizer)
     {
diff --git a/service/Abstractions/SemanticKernel/SemanticKernelTextGenerator.cs b/service/Abstractions/SemanticKernel/SemanticKernelTextGenerator.cs
index fe4fb2d88..2833d2200 100644
--- a/service/Abstractions/SemanticKernel/SemanticKernelTextGenerator.cs
+++ b/service/Abstractions/SemanticKernel/SemanticKernelTextGenerator.cs
@@ -5,9 +5,8 @@
 using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.SemanticKernel.AI;
-using Microsoft.SemanticKernel.AI.TextGeneration;
-
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.TextGeneration;
 namespace Microsoft.KernelMemory.AI;
 
 internal class SemanticKernelTextGenerator : ITextGenerator
@@ -48,7 +47,7 @@ private static PromptExecutionSettings ToPromptExecutionSettings(TextGenerationO
     {
         var settings = new PromptExecutionSettings
         {
-            ExtensionData =
+            ExtensionData = new()
             {
                 [nameof(options.Temperature)] = options.Temperature,
                 [nameof(options.TopP)] = options.TopP,

From 61dac3b078f3260078d2f0d4bf55ab37cfbb041c Mon Sep 17 00:00:00 2001
From: Devis Lucato <devis@microsoft.com>
Date: Mon, 18 Mar 2024 11:06:31 +0100
Subject: [PATCH 11/12] Update PR

---
 KernelMemory.sln                              |   6 +++++
 ...otnet-SemanticKernel-TextCompletion.csproj |  12 +++++----
 .../Program.cs                                |  24 ++++++++----------
 .../story.docx                                | Bin 0 -> 28247 bytes
 .../KernelMemoryBuilderExtensions.cs          |   3 +--
 .../SemanticKernelTextEmbeddingGenerator.cs   |  12 +++++----
 .../SemanticKernelTextGenerator.cs            |  10 +++++---
 service/Service/Service.csproj                |   1 +
 8 files changed, 38 insertions(+), 30 deletions(-)
 create mode 100644 examples/107-dotnet-SemanticKernel-TextCompletion/story.docx

diff --git a/KernelMemory.sln b/KernelMemory.sln
index d11ab1451..59c2ae5bf 100644
--- a/KernelMemory.sln
+++ b/KernelMemory.sln
@@ -230,6 +230,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MongoDbAtlas", "extensions\
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MongoDbAtlas.FunctionalTests", "extensions\MongoDbAtlas\MongoDbAtlas.FunctionalTests\MongoDbAtlas.FunctionalTests.csproj", "{8A602227-B291-4F1B-ACB8-237F49501B6A}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "107-dotnet-SemanticKernel-TextCompletion", "examples\107-dotnet-SemanticKernel-TextCompletion\107-dotnet-SemanticKernel-TextCompletion.csproj", "{494B8590-F0B2-4D40-A895-F9D7BDF26250}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -298,6 +300,7 @@ Global
 		{A5CE3CB2-F746-4832-B239-1A3226F9BD48} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841}
 		{4A539320-EB49-4688-82E1-4FA0ADBB3810} = {155DA079-E267-49AF-973A-D1D44681970F}
 		{8A602227-B291-4F1B-ACB8-237F49501B6A} = {3C17F42B-CFC8-4900-8CFB-88936311E919}
+		{494B8590-F0B2-4D40-A895-F9D7BDF26250} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841}
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{8A9FA587-7EBA-4D43-BE47-38D798B1C74C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
@@ -486,5 +489,8 @@ Global
 		{8A602227-B291-4F1B-ACB8-237F49501B6A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{8A602227-B291-4F1B-ACB8-237F49501B6A}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{8A602227-B291-4F1B-ACB8-237F49501B6A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{494B8590-F0B2-4D40-A895-F9D7BDF26250}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{494B8590-F0B2-4D40-A895-F9D7BDF26250}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{494B8590-F0B2-4D40-A895-F9D7BDF26250}.Release|Any CPU.ActiveCfg = Release|Any CPU
 	EndGlobalSection
 EndGlobal
diff --git a/examples/107-dotnet-SemanticKernel-TextCompletion/107-dotnet-SemanticKernel-TextCompletion.csproj b/examples/107-dotnet-SemanticKernel-TextCompletion/107-dotnet-SemanticKernel-TextCompletion.csproj
index ba302eb1b..882fd7c5a 100644
--- a/examples/107-dotnet-SemanticKernel-TextCompletion/107-dotnet-SemanticKernel-TextCompletion.csproj
+++ b/examples/107-dotnet-SemanticKernel-TextCompletion/107-dotnet-SemanticKernel-TextCompletion.csproj
@@ -7,17 +7,19 @@
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
         <UserSecretsId>5ee045b0-aea3-4f08-8d31-32d1a6f8fed0</UserSecretsId>
-        <NoWarn>CA1711,CA1724,CA1308,CA1812,CA1859,CA2208,CS1591,NU5104,SKEXP0001,SKEXP0011</NoWarn>
+        <NoWarn>$(NoWarn);SKEXP0010;</NoWarn>
+        <DefineConstants Condition="'$(SolutionName)' == 'KernelMemoryDev'">$(DefineConstants);KernelMemoryDev</DefineConstants>
     </PropertyGroup>
 
     <ItemGroup>
-      <None Include="..\203-dotnet-using-core-nuget\sample-SK-Readme.pdf" Link="sample-SK-Readme.pdf">
-        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-      </None>
+        <ProjectReference Include="..\..\service\Core\Core.csproj" />
     </ItemGroup>
 
     <ItemGroup>
-        <ProjectReference Include="..\..\service\Core\Core.csproj" />
+        <None Remove="story.docx" />
+        <Content Include="story.docx">
+          <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+        </Content>
     </ItemGroup>
 
 </Project>
diff --git a/examples/107-dotnet-SemanticKernel-TextCompletion/Program.cs b/examples/107-dotnet-SemanticKernel-TextCompletion/Program.cs
index 8de54b98b..ec2ff8866 100644
--- a/examples/107-dotnet-SemanticKernel-TextCompletion/Program.cs
+++ b/examples/107-dotnet-SemanticKernel-TextCompletion/Program.cs
@@ -1,7 +1,8 @@
 ﻿// Copyright (c) Microsoft. All rights reserved.
 
+#if KernelMemoryDev // This code requires the next release (use KernelMemoryDev.sln)
 using Microsoft.KernelMemory;
-using Microsoft.KernelMemory.AI.Tokenizers;
+using Microsoft.KernelMemory.AI.OpenAI;
 using Microsoft.SemanticKernel.Connectors.OpenAI;
 
 var endpoint = Env.Var("AOAI_ENDPOINT");
@@ -13,29 +14,24 @@
 var tokenizer = new DefaultGPTTokenizer();
 
 var memory = new KernelMemoryBuilder()
-    .WithSemanticKernelTextGenerationService(new AzureOpenAIChatCompletionService(chatDeployment, endpoint, apiKey)
-                                            , config
-                                            , tokenizer)
-    .WithSemanticKernelTextEmbeddingGenerationService(new AzureOpenAITextEmbeddingGenerationService(embeddingDeployment, endpoint, apiKey)
-                                            , config
-                                            , tokenizer)
+    .WithSemanticKernelTextGenerationService(
+        new AzureOpenAIChatCompletionService(chatDeployment, endpoint, apiKey), config, tokenizer)
+    .WithSemanticKernelTextEmbeddingGenerationService(
+        new AzureOpenAITextEmbeddingGenerationService(embeddingDeployment, endpoint, apiKey), config, tokenizer)
     .Build<MemoryServerless>();
 
-// using document form 203-dotnet-using-core-nuget
-
-await memory.ImportDocumentAsync("sample-SK-Readme.pdf", documentId: "doc001");
-
-var question = "What's Semantic Kernel?";
+await memory.ImportDocumentAsync("story.docx", documentId: "doc001");
 
+var question = "What's radiant mold?";
 Console.WriteLine($"\n\nQuestion: {question}");
 
 var answer = await memory.AskAsync(question);
-
 Console.WriteLine($"\nAnswer: {answer.Result}");
 
 Console.WriteLine("\n\n  Sources:\n");
-
 foreach (var x in answer.RelevantSources)
 {
     Console.WriteLine($"  - {x.SourceName}  - {x.Link} [{x.Partitions.First().LastUpdate:D}]");
 }
+
+#endif
diff --git a/examples/107-dotnet-SemanticKernel-TextCompletion/story.docx b/examples/107-dotnet-SemanticKernel-TextCompletion/story.docx
new file mode 100644
index 0000000000000000000000000000000000000000..c3dbde1d98bb653cbece13527d2e680c132e9faa
GIT binary patch
diff --git a/service/Abstractions/SemanticKernel/KernelMemoryBuilderExtensions.cs b/service/Abstractions/SemanticKernel/KernelMemoryBuilderExtensions.cs
index 95ea0119b..314575818 100644
--- a/service/Abstractions/SemanticKernel/KernelMemoryBuilderExtensions.cs
+++ b/service/Abstractions/SemanticKernel/KernelMemoryBuilderExtensions.cs
@@ -2,9 +2,8 @@
 
 using Microsoft.KernelMemory.AI;
 using Microsoft.KernelMemory.Configuration;
-using Microsoft.SemanticKernel.TextGeneration;
-using Microsoft.SemanticKernel.AI.Embeddings;
 using Microsoft.SemanticKernel.Embeddings;
+using Microsoft.SemanticKernel.TextGeneration;
 
 namespace Microsoft.KernelMemory;
 
diff --git a/service/Abstractions/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs b/service/Abstractions/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs
index ac06363ae..fc24db856 100644
--- a/service/Abstractions/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs
+++ b/service/Abstractions/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs
@@ -8,17 +8,22 @@
 using Microsoft.SemanticKernel.Embeddings;
 
 namespace Microsoft.KernelMemory;
+
 internal class SemanticKernelTextEmbeddingGenerator : ITextEmbeddingGenerator
 {
     private readonly ITextTokenizer _tokenizer;
     private readonly ITextEmbeddingGenerationService _generation;
     private readonly SemanticKernelConfig _config;
 
+    /// <inheritdoc />
     public int MaxTokens { get; }
 
+    /// <inheritdoc />
+    public int CountTokens(string text) => this._tokenizer.CountTokens(text);
+
     public SemanticKernelTextEmbeddingGenerator(ITextEmbeddingGenerationService generation,
-                                                SemanticKernelConfig config,
-                                                ITextTokenizer tokenizer)
+        SemanticKernelConfig config,
+        ITextTokenizer tokenizer)
     {
         this._generation = generation ?? throw new ArgumentNullException(nameof(generation));
         this._tokenizer = tokenizer ?? throw new ArgumentNullException(nameof(tokenizer), "Tokenizer not specified. The token count might be incorrect, causing unexpected errors");
@@ -28,9 +33,6 @@ public SemanticKernelTextEmbeddingGenerator(ITextEmbeddingGenerationService gene
         this.MaxTokens = config.MaxTokenTotal;
     }
 
-    /// <inheritdoc />
-    public int CountTokens(string text) => this._tokenizer.CountTokens(text);
-
     /// <inheritdoc />
     public Task<Embedding> GenerateEmbeddingAsync(string text, CancellationToken cancellationToken = default)
         => this._generation.GenerateEmbeddingAsync(text, cancellationToken);
diff --git a/service/Abstractions/SemanticKernel/SemanticKernelTextGenerator.cs b/service/Abstractions/SemanticKernel/SemanticKernelTextGenerator.cs
index 2833d2200..7c267a71c 100644
--- a/service/Abstractions/SemanticKernel/SemanticKernelTextGenerator.cs
+++ b/service/Abstractions/SemanticKernel/SemanticKernelTextGenerator.cs
@@ -7,6 +7,7 @@
 using System.Threading.Tasks;
 using Microsoft.SemanticKernel;
 using Microsoft.SemanticKernel.TextGeneration;
+
 namespace Microsoft.KernelMemory.AI;
 
 internal class SemanticKernelTextGenerator : ITextGenerator
@@ -15,8 +16,12 @@ internal class SemanticKernelTextGenerator : ITextGenerator
     private readonly ITextTokenizer _tokenizer;
     private readonly SemanticKernelConfig _config;
 
+    /// <inheritdoc />
     public int MaxTokenTotal { get; }
 
+    /// <inheritdoc />
+    public int CountTokens(string text) => this._tokenizer.CountTokens(text);
+
     public SemanticKernelTextGenerator(ITextGenerationService service, SemanticKernelConfig config, ITextTokenizer tokenizer)
     {
         this._service = service ?? throw new ArgumentNullException(nameof(service));
@@ -47,7 +52,7 @@ private static PromptExecutionSettings ToPromptExecutionSettings(TextGenerationO
     {
         var settings = new PromptExecutionSettings
         {
-            ExtensionData = new()
+            ExtensionData = new Dictionary<string, object>()
             {
                 [nameof(options.Temperature)] = options.Temperature,
                 [nameof(options.TopP)] = options.TopP,
@@ -66,7 +71,4 @@ private static PromptExecutionSettings ToPromptExecutionSettings(TextGenerationO
 
         return settings;
     }
-
-    /// <inheritdoc />
-    public int CountTokens(string text) => this._tokenizer.CountTokens(text);
 }
diff --git a/service/Service/Service.csproj b/service/Service/Service.csproj
index 1e965c999..f1f6b5667 100644
--- a/service/Service/Service.csproj
+++ b/service/Service/Service.csproj
@@ -7,6 +7,7 @@
         <ManagePackageVersionsCentrally>false</ManagePackageVersionsCentrally>
         <UserSecretsId>5ee045b0-aea3-4f08-8d31-32d1a6f8fed0</UserSecretsId>
         <NoWarn>$(NoWarn);CA2007,CA1724,CA2254,CA1031,CS1591,CA1861,CA1303,SKEXP0001</NoWarn>
+        <DefineConstants Condition="'$(SolutionName)' == 'KernelMemoryDev'">$(DefineConstants);KernelMemoryDev</DefineConstants>
     </PropertyGroup>
 
     <ItemGroup>

From e54fdead1348028a40ea4efde36f6f7aff3983a2 Mon Sep 17 00:00:00 2001
From: Devis Lucato <devis@microsoft.com>
Date: Mon, 18 Mar 2024 11:33:43 +0100
Subject: [PATCH 12/12] Move code to Service and make tokenizer optional

---
 ...otnet-SemanticKernel-TextCompletion.csproj |  10 +--
 .../Program.cs                                |   8 +--
 .../story.docx                                | Bin 28247 -> 0 bytes
 .../SemanticKernelTextEmbeddingGenerator.cs   |  39 ------------
 service/Core/Core.csproj                      |   2 +-
 .../KernelMemoryBuilderExtensions.cs          |  43 +++++++------
 .../SemanticKernel/SemanticKernelConfig.cs    |   5 +-
 .../SemanticKernelTextEmbeddingGenerator.cs   |  57 ++++++++++++++++++
 .../SemanticKernelTextGenerator.cs            |  33 ++++++++--
 9 files changed, 117 insertions(+), 80 deletions(-)
 delete mode 100644 examples/107-dotnet-SemanticKernel-TextCompletion/story.docx
 delete mode 100644 service/Abstractions/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs
 rename service/{Abstractions => Core}/SemanticKernel/KernelMemoryBuilderExtensions.cs (50%)
 rename service/{Abstractions => Core}/SemanticKernel/SemanticKernelConfig.cs (58%)
 create mode 100644 service/Core/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs
 rename service/{Abstractions => Core}/SemanticKernel/SemanticKernelTextGenerator.cs (65%)

diff --git a/examples/107-dotnet-SemanticKernel-TextCompletion/107-dotnet-SemanticKernel-TextCompletion.csproj b/examples/107-dotnet-SemanticKernel-TextCompletion/107-dotnet-SemanticKernel-TextCompletion.csproj
index 882fd7c5a..cfa574100 100644
--- a/examples/107-dotnet-SemanticKernel-TextCompletion/107-dotnet-SemanticKernel-TextCompletion.csproj
+++ b/examples/107-dotnet-SemanticKernel-TextCompletion/107-dotnet-SemanticKernel-TextCompletion.csproj
@@ -7,19 +7,11 @@
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
         <UserSecretsId>5ee045b0-aea3-4f08-8d31-32d1a6f8fed0</UserSecretsId>
-        <NoWarn>$(NoWarn);SKEXP0010;</NoWarn>
-        <DefineConstants Condition="'$(SolutionName)' == 'KernelMemoryDev'">$(DefineConstants);KernelMemoryDev</DefineConstants>
+        <NoWarn>$(NoWarn);SKEXP0001;SKEXP0010;</NoWarn>
     </PropertyGroup>
 
     <ItemGroup>
         <ProjectReference Include="..\..\service\Core\Core.csproj" />
     </ItemGroup>
 
-    <ItemGroup>
-        <None Remove="story.docx" />
-        <Content Include="story.docx">
-          <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-        </Content>
-    </ItemGroup>
-
 </Project>
diff --git a/examples/107-dotnet-SemanticKernel-TextCompletion/Program.cs b/examples/107-dotnet-SemanticKernel-TextCompletion/Program.cs
index ec2ff8866..a05c5b7e3 100644
--- a/examples/107-dotnet-SemanticKernel-TextCompletion/Program.cs
+++ b/examples/107-dotnet-SemanticKernel-TextCompletion/Program.cs
@@ -1,8 +1,8 @@
 ﻿// Copyright (c) Microsoft. All rights reserved.
 
-#if KernelMemoryDev // This code requires the next release (use KernelMemoryDev.sln)
 using Microsoft.KernelMemory;
 using Microsoft.KernelMemory.AI.OpenAI;
+using Microsoft.KernelMemory.SemanticKernel;
 using Microsoft.SemanticKernel.Connectors.OpenAI;
 
 var endpoint = Env.Var("AOAI_ENDPOINT");
@@ -20,9 +20,9 @@
         new AzureOpenAITextEmbeddingGenerationService(embeddingDeployment, endpoint, apiKey), config, tokenizer)
     .Build<MemoryServerless>();
 
-await memory.ImportDocumentAsync("story.docx", documentId: "doc001");
+await memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/kernel-memory/main/COMMUNITY.md", documentId: "doc001");
 
-var question = "What's radiant mold?";
+var question = "How can I join Kernel Memory's Discord?";
 Console.WriteLine($"\n\nQuestion: {question}");
 
 var answer = await memory.AskAsync(question);
@@ -33,5 +33,3 @@
 {
     Console.WriteLine($"  - {x.SourceName}  - {x.Link} [{x.Partitions.First().LastUpdate:D}]");
 }
-
-#endif
diff --git a/examples/107-dotnet-SemanticKernel-TextCompletion/story.docx b/examples/107-dotnet-SemanticKernel-TextCompletion/story.docx
deleted file mode 100644
index c3dbde1d98bb653cbece13527d2e680c132e9faa..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/service/Abstractions/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs b/service/Abstractions/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs
deleted file mode 100644
index fc24db856..000000000
--- a/service/Abstractions/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs
+++ /dev/null
@@ -1,39 +0,0 @@
-﻿// Copyright (c) Microsoft. All rights reserved.
-
-using System;
-using System.Threading;
-using System.Threading.Tasks;
-using Microsoft.KernelMemory.AI;
-using Microsoft.SemanticKernel.AI.Embeddings;
-using Microsoft.SemanticKernel.Embeddings;
-
-namespace Microsoft.KernelMemory;
-
-internal class SemanticKernelTextEmbeddingGenerator : ITextEmbeddingGenerator
-{
-    private readonly ITextTokenizer _tokenizer;
-    private readonly ITextEmbeddingGenerationService _generation;
-    private readonly SemanticKernelConfig _config;
-
-    /// <inheritdoc />
-    public int MaxTokens { get; }
-
-    /// <inheritdoc />
-    public int CountTokens(string text) => this._tokenizer.CountTokens(text);
-
-    public SemanticKernelTextEmbeddingGenerator(ITextEmbeddingGenerationService generation,
-        SemanticKernelConfig config,
-        ITextTokenizer tokenizer)
-    {
-        this._generation = generation ?? throw new ArgumentNullException(nameof(generation));
-        this._tokenizer = tokenizer ?? throw new ArgumentNullException(nameof(tokenizer), "Tokenizer not specified. The token count might be incorrect, causing unexpected errors");
-
-        this._tokenizer = tokenizer;
-        this._config = config;
-        this.MaxTokens = config.MaxTokenTotal;
-    }
-
-    /// <inheritdoc />
-    public Task<Embedding> GenerateEmbeddingAsync(string text, CancellationToken cancellationToken = default)
-        => this._generation.GenerateEmbeddingAsync(text, cancellationToken);
-}
diff --git a/service/Core/Core.csproj b/service/Core/Core.csproj
index cee218b80..8b9259efd 100644
--- a/service/Core/Core.csproj
+++ b/service/Core/Core.csproj
@@ -5,7 +5,7 @@
         <RollForward>LatestMajor</RollForward>
         <AssemblyName>Microsoft.KernelMemory.Core</AssemblyName>
         <RootNamespace>Microsoft.KernelMemory</RootNamespace>
-        <NoWarn>$(NoWarn);SKEXP0011;CA2208;CA1308;CA1724;CS1591;</NoWarn>
+        <NoWarn>$(NoWarn);SKEXP0001;SKEXP0011;CA2208;CA1308;CA1724;CS1591;</NoWarn>
         <DefineConstants Condition="'$(SolutionName)' == 'KernelMemoryDev'">$(DefineConstants);KernelMemoryDev</DefineConstants>
     </PropertyGroup>
 
diff --git a/service/Abstractions/SemanticKernel/KernelMemoryBuilderExtensions.cs b/service/Core/SemanticKernel/KernelMemoryBuilderExtensions.cs
similarity index 50%
rename from service/Abstractions/SemanticKernel/KernelMemoryBuilderExtensions.cs
rename to service/Core/SemanticKernel/KernelMemoryBuilderExtensions.cs
index 314575818..0695b2df9 100644
--- a/service/Abstractions/SemanticKernel/KernelMemoryBuilderExtensions.cs
+++ b/service/Core/SemanticKernel/KernelMemoryBuilderExtensions.cs
@@ -1,10 +1,14 @@
 ﻿// Copyright (c) Microsoft. All rights reserved.
 
+using Microsoft.Extensions.Logging;
 using Microsoft.KernelMemory.AI;
 using Microsoft.KernelMemory.Configuration;
+using Microsoft.KernelMemory.SemanticKernel;
 using Microsoft.SemanticKernel.Embeddings;
 using Microsoft.SemanticKernel.TextGeneration;
 
+#pragma warning disable IDE0130 // reduce number of "using" statements
+// ReSharper disable once CheckNamespace - reduce number of "using" statements
 namespace Microsoft.KernelMemory;
 
 /// <summary>
@@ -20,41 +24,42 @@ public static partial class KernelMemoryBuilderExtensions
     /// <param name="service">SK text generation service instance</param>
     /// <param name="config">SK text generator settings</param>
     /// <param name="tokenizer">Tokenizer used to count tokens used by prompts</param>
+    ///  <param name="loggerFactory">.NET logger factory</param>
     /// <returns>KM builder</returns>
     public static IKernelMemoryBuilder WithSemanticKernelTextGenerationService(
         this IKernelMemoryBuilder builder,
         ITextGenerationService service,
         SemanticKernelConfig config,
-        ITextTokenizer tokenizer)
+        ITextTokenizer? tokenizer = null,
+        ILoggerFactory? loggerFactory = null)
     {
-        service = service ?? throw new ConfigurationException("The semantic kernel text generation service instance is NULL");
-        tokenizer = tokenizer ?? throw new ConfigurationException("The text tokenizer instance is NULL");
+        if (service == null) { throw new ConfigurationException("The semantic kernel text generation service instance is NULL"); }
 
-        return builder.AddSingleton<ITextGenerator>(new SemanticKernelTextGenerator(service, config, tokenizer));
+        return builder.AddSingleton<ITextGenerator>(new SemanticKernelTextGenerator(service, config, tokenizer, loggerFactory));
     }
 
-    /// <summary>
-    ///Inject an implementation of<see cref="ITextEmbeddingGenerationService">SK text embedding generation service</see>
-    /// for local dependencies on <see cref="ITextEmbeddingGenerator"/>
-    /// </summary>
-    /// <param name="builder">KM builder</param>
-    /// <param name="service">SK text embedding generation instance</param>
-    /// <param name="config">SK text embedding generator settings</param>
-    /// <param name="tokenizer">Tokenizer used to count tokens sent to the embedding generator</param>
-    /// <param name="onlyForRetrieval">Whether to use this embedding generator only during data ingestion, and not for retrieval (search and ask API)</param>
-    /// <returns>KM builder</returns>
+    ///  <summary>
+    /// Inject an implementation of<see cref="ITextEmbeddingGenerationService">SK text embedding generation service</see>
+    ///  for local dependencies on <see cref="ITextEmbeddingGenerator"/>
+    ///  </summary>
+    ///  <param name="builder">KM builder</param>
+    ///  <param name="service">SK text embedding generation instance</param>
+    ///  <param name="config">SK text embedding generator settings</param>
+    ///  <param name="tokenizer">Tokenizer used to count tokens sent to the embedding generator</param>
+    ///  <param name="loggerFactory">.NET logger factory</param>
+    ///  <param name="onlyForRetrieval">Whether to use this embedding generator only during data ingestion, and not for retrieval (search and ask API)</param>
+    ///  <returns>KM builder</returns>
     public static IKernelMemoryBuilder WithSemanticKernelTextEmbeddingGenerationService(
         this IKernelMemoryBuilder builder,
         ITextEmbeddingGenerationService service,
         SemanticKernelConfig config,
-        ITextTokenizer tokenizer,
+        ITextTokenizer? tokenizer = null,
+        ILoggerFactory? loggerFactory = null,
         bool onlyForRetrieval = false)
     {
-        service = service ?? throw new ConfigurationException("The semantic kernel text embedding generation service instance is NULL");
-        tokenizer = tokenizer ?? throw new ConfigurationException("The text tokenizer instance is NULL");
-
-        var generator = new SemanticKernelTextEmbeddingGenerator(service, config, tokenizer);
+        if (service == null) { throw new ConfigurationException("The semantic kernel text embedding generation service instance is NULL"); }
 
+        var generator = new SemanticKernelTextEmbeddingGenerator(service, config, tokenizer, loggerFactory);
         builder.AddSingleton<ITextEmbeddingGenerator>(generator);
 
         if (!onlyForRetrieval)
diff --git a/service/Abstractions/SemanticKernel/SemanticKernelConfig.cs b/service/Core/SemanticKernel/SemanticKernelConfig.cs
similarity index 58%
rename from service/Abstractions/SemanticKernel/SemanticKernelConfig.cs
rename to service/Core/SemanticKernel/SemanticKernelConfig.cs
index 8be7c0175..b20d90059 100644
--- a/service/Abstractions/SemanticKernel/SemanticKernelConfig.cs
+++ b/service/Core/SemanticKernel/SemanticKernelConfig.cs
@@ -1,11 +1,14 @@
 ﻿// Copyright (c) Microsoft. All rights reserved.
 
-namespace Microsoft.KernelMemory;
+namespace Microsoft.KernelMemory.SemanticKernel;
 
 /// <summary>
 /// Semantic Kernel TextGenerator And TextEmbeddingGenerator Config
 /// </summary>
 public class SemanticKernelConfig
 {
+    /// <summary>
+    /// Max size of the LLM attention window, ie max tokens that can be processed.
+    /// </summary>
     public int MaxTokenTotal { get; set; } = 8191;
 }
diff --git a/service/Core/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs b/service/Core/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs
new file mode 100644
index 000000000..561181017
--- /dev/null
+++ b/service/Core/SemanticKernel/SemanticKernelTextEmbeddingGenerator.cs
@@ -0,0 +1,57 @@
+﻿// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.KernelMemory.AI;
+using Microsoft.KernelMemory.AI.OpenAI;
+using Microsoft.KernelMemory.Diagnostics;
+using Microsoft.SemanticKernel.AI.Embeddings;
+using Microsoft.SemanticKernel.Embeddings;
+
+namespace Microsoft.KernelMemory.SemanticKernel;
+
+internal class SemanticKernelTextEmbeddingGenerator : ITextEmbeddingGenerator
+{
+    private readonly ITextEmbeddingGenerationService _service;
+    private readonly ITextTokenizer _tokenizer;
+    private readonly ILogger<SemanticKernelTextEmbeddingGenerator> _log;
+
+    /// <inheritdoc />
+    public int MaxTokens { get; }
+
+    /// <inheritdoc />
+    public int CountTokens(string text) => this._tokenizer.CountTokens(text);
+
+    public SemanticKernelTextEmbeddingGenerator(
+        ITextEmbeddingGenerationService textEmbeddingGenerationService,
+        SemanticKernelConfig config,
+        ITextTokenizer? textTokenizer = null,
+        ILoggerFactory? loggerFactory = null)
+    {
+        this._service = textEmbeddingGenerationService ?? throw new ArgumentNullException(nameof(textEmbeddingGenerationService));
+        this.MaxTokens = config.MaxTokenTotal;
+
+        var log = loggerFactory?.CreateLogger<SemanticKernelTextEmbeddingGenerator>();
+        this._log = log ?? DefaultLogger<SemanticKernelTextEmbeddingGenerator>.Instance;
+
+        if (textTokenizer == null)
+        {
+            this._log.LogWarning(
+                "Tokenizer not specified, will use {0}. The token count might be incorrect, causing unexpected errors",
+                nameof(DefaultGPTTokenizer));
+            textTokenizer = new DefaultGPTTokenizer();
+        }
+
+        this._tokenizer = textTokenizer;
+    }
+
+    /// <inheritdoc />
+    public Task<Embedding> GenerateEmbeddingAsync(string text, CancellationToken cancellationToken = default)
+    {
+        this._log.LogTrace("Generating embedding with SK embedding generator service");
+
+        return this._service.GenerateEmbeddingAsync(text, cancellationToken);
+    }
+}
diff --git a/service/Abstractions/SemanticKernel/SemanticKernelTextGenerator.cs b/service/Core/SemanticKernel/SemanticKernelTextGenerator.cs
similarity index 65%
rename from service/Abstractions/SemanticKernel/SemanticKernelTextGenerator.cs
rename to service/Core/SemanticKernel/SemanticKernelTextGenerator.cs
index 7c267a71c..1699705b4 100644
--- a/service/Abstractions/SemanticKernel/SemanticKernelTextGenerator.cs
+++ b/service/Core/SemanticKernel/SemanticKernelTextGenerator.cs
@@ -5,16 +5,20 @@
 using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.KernelMemory.AI;
+using Microsoft.KernelMemory.AI.OpenAI;
+using Microsoft.KernelMemory.Diagnostics;
 using Microsoft.SemanticKernel;
 using Microsoft.SemanticKernel.TextGeneration;
 
-namespace Microsoft.KernelMemory.AI;
+namespace Microsoft.KernelMemory.SemanticKernel;
 
 internal class SemanticKernelTextGenerator : ITextGenerator
 {
     private readonly ITextGenerationService _service;
     private readonly ITextTokenizer _tokenizer;
-    private readonly SemanticKernelConfig _config;
+    private readonly ILogger<SemanticKernelTextGenerator> _log;
 
     /// <inheritdoc />
     public int MaxTokenTotal { get; }
@@ -22,12 +26,27 @@ internal class SemanticKernelTextGenerator : ITextGenerator
     /// <inheritdoc />
     public int CountTokens(string text) => this._tokenizer.CountTokens(text);
 
-    public SemanticKernelTextGenerator(ITextGenerationService service, SemanticKernelConfig config, ITextTokenizer tokenizer)
+    public SemanticKernelTextGenerator(
+        ITextGenerationService textGenerationService,
+        SemanticKernelConfig config,
+        ITextTokenizer? textTokenizer = null,
+        ILoggerFactory? loggerFactory = null)
     {
-        this._service = service ?? throw new ArgumentNullException(nameof(service));
-        this._tokenizer = tokenizer ?? throw new ArgumentNullException(nameof(tokenizer), "Tokenizer not specified. The token count might be incorrect, causing unexpected errors");
-        this._config = config;
+        this._service = textGenerationService ?? throw new ArgumentNullException(nameof(textGenerationService));
         this.MaxTokenTotal = config.MaxTokenTotal;
+
+        var log = loggerFactory?.CreateLogger<SemanticKernelTextGenerator>();
+        this._log = log ?? DefaultLogger<SemanticKernelTextGenerator>.Instance;
+
+        if (textTokenizer == null)
+        {
+            this._log.LogWarning(
+                "Tokenizer not specified, will use {0}. The token count might be incorrect, causing unexpected errors",
+                nameof(DefaultGPTTokenizer));
+            textTokenizer = new DefaultGPTTokenizer();
+        }
+
+        this._tokenizer = textTokenizer;
     }
 
     /// <inheritdoc />
@@ -36,6 +55,8 @@ public async IAsyncEnumerable<string> GenerateTextAsync(
         TextGenerationOptions options,
         [EnumeratorCancellation] CancellationToken cancellationToken = default)
     {
+        this._log.LogTrace("Generating text with SK text generator service");
+
         var contents = this._service.GetStreamingTextContentsAsync(
             prompt, ToPromptExecutionSettings(options), null, cancellationToken).ConfigureAwait(false);
 
