From 24b9e88a55087f317c5611f3cc13af53c150663a Mon Sep 17 00:00:00 2001 From: Kris Borowinski Date: Tue, 19 Nov 2024 12:14:30 +0100 Subject: [PATCH 01/15] Implement settings file for Ollama agent --- .../AIShell.Ollama.Agent/OllamaAgent.cs | 59 ++++++++++++++++++- .../AIShell.Ollama.Agent/OllamaChatService.cs | 13 ++-- shell/agents/AIShell.Ollama.Agent/Settings.cs | 35 +++++++++++ 3 files changed, 97 insertions(+), 10 deletions(-) create mode 100644 shell/agents/AIShell.Ollama.Agent/Settings.cs diff --git a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs index 36aa6140..08886226 100644 --- a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs +++ b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs @@ -1,10 +1,20 @@ using System.Diagnostics; +using System.Text; +using System.Text.Json; using AIShell.Abstraction; namespace AIShell.Ollama.Agent; public sealed class OllamaAgent : ILLMAgent { + private const string SettingFileName = "ollama.config.json"; + private Settings _settings; + + /// + /// Gets the settings. + /// + internal Settings Settings => _settings; + /// /// The name of the agent /// @@ -51,7 +61,17 @@ public void Dispose() /// Agent configuration for any configuration file and other settings public void Initialize(AgentConfig config) { - _chatService = new OllamaChatService(); + SettingFile = Path.Combine(config.ConfigurationRoot, SettingFileName); + _settings = ReadSettings(); + + if (_settings is null) + { + // Create the setting file with examples to serve as a template for user to update. + NewExampleSettingFile(); + _settings = ReadSettings(); + } + + _chatService = new OllamaChatService(_settings); LegalLinks = new(StringComparer.OrdinalIgnoreCase) { @@ -68,7 +88,7 @@ public void Initialize(AgentConfig config) /// /// Gets the path to the setting file of the agent. /// - public string SettingFile { private set; get; } = null; + public string SettingFile { private set; get; } /// /// Gets a value indicating whether the agent accepts a specific user action feedback. @@ -122,4 +142,39 @@ public async Task ChatAsync(string input, IShell shell) return true; } + + private Settings ReadSettings() + { + Settings settings = null; + FileInfo file = new(SettingFile); + + if (file.Exists) + { + try + { + using var stream = file.OpenRead(); + var data = JsonSerializer.Deserialize(stream, SourceGenerationContext.Default.ConfigData); + settings = new Settings(data); + } + catch (Exception e) + { + throw new InvalidDataException($"Parsing settings from '{SettingFile}' failed with the following error: {e.Message}", e); + } + } + + return settings; + } + + private void NewExampleSettingFile() + { + string SampleContent = $$""" + { + // Declare Ollama model name. + "Model": "phi3", + // Declare Ollama endpoint. + "Endpoint": "http://localhost:11434/api/generate" + } + """; + File.WriteAllText(SettingFile, SampleContent, Encoding.UTF8); + } } diff --git a/shell/agents/AIShell.Ollama.Agent/OllamaChatService.cs b/shell/agents/AIShell.Ollama.Agent/OllamaChatService.cs index 8809eff3..63314dd7 100644 --- a/shell/agents/AIShell.Ollama.Agent/OllamaChatService.cs +++ b/shell/agents/AIShell.Ollama.Agent/OllamaChatService.cs @@ -7,11 +7,7 @@ namespace AIShell.Ollama.Agent; internal class OllamaChatService : IDisposable { - /// - /// Ollama endpoint to call to generate a response - /// - internal const string Endpoint = "http://localhost:11434/api/generate"; - + private Settings _settings; /// /// Http client /// @@ -20,8 +16,9 @@ internal class OllamaChatService : IDisposable /// /// Initialization method to initialize the http client /// - internal OllamaChatService() + internal OllamaChatService(Settings settings) { + _settings = settings; _client = new HttpClient(); } @@ -43,7 +40,7 @@ private HttpRequestMessage PrepareForChat(string input) // Main data to send to the endpoint var requestData = new Query { - model = "phi3", + model = _settings.Model, prompt = input, stream = false }; @@ -51,7 +48,7 @@ private HttpRequestMessage PrepareForChat(string input) var json = JsonSerializer.Serialize(requestData); var data = new StringContent(json, Encoding.UTF8, "application/json"); - var request = new HttpRequestMessage(HttpMethod.Post, Endpoint) { Content = data }; + var request = new HttpRequestMessage(HttpMethod.Post, _settings.Endpoint) { Content = data }; return request; } diff --git a/shell/agents/AIShell.Ollama.Agent/Settings.cs b/shell/agents/AIShell.Ollama.Agent/Settings.cs new file mode 100644 index 00000000..7ac3555b --- /dev/null +++ b/shell/agents/AIShell.Ollama.Agent/Settings.cs @@ -0,0 +1,35 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace AIShell.Ollama.Agent; + +internal class Settings +{ + public string Model { get; set; } + public string Endpoint { get; set; } + + public Settings(ConfigData configData) + { + Model = configData.Model; + Endpoint = configData.Endpoint; + } +} + +internal class ConfigData +{ + public string Model { get; set; } + public string Endpoint { get; set; } +} + +/// +/// Use source generation to serialize and deserialize the setting file. +/// Both metadata-based and serialization-optimization modes are used to gain the best performance. +/// +[JsonSourceGenerationOptions( + WriteIndented = true, + AllowTrailingCommas = true, + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + UseStringEnumConverter = true)] +[JsonSerializable(typeof(ConfigData))] +internal partial class SourceGenerationContext : JsonSerializerContext { } From cdcc98e43adccd99fdee7d35feb6be54d919b078 Mon Sep 17 00:00:00 2001 From: Kris Borowinski Date: Tue, 19 Nov 2024 17:02:59 +0100 Subject: [PATCH 02/15] Update readme and links to the documentation; Handle http exception due to Ollama misconfiguration --- .../AIShell.Ollama.Agent/OllamaAgent.cs | 51 +++++++++++++------ shell/agents/AIShell.Ollama.Agent/README.md | 28 +++++++--- 2 files changed, 58 insertions(+), 21 deletions(-) diff --git a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs index 08886226..28e421f3 100644 --- a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs +++ b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs @@ -23,7 +23,7 @@ public sealed class OllamaAgent : ILLMAgent /// /// The description of the agent to be shown at start up /// - public string Description => "This is an AI assistant to interact with a language model running locally by utilizing the Ollama CLI tool. Be sure to follow all prerequisites in aka.ms/aish/ollama"; + public string Description => "This is an AI assistant to interact with a language model running locally by utilizing the Ollama CLI tool. Be sure to follow all prerequisites in https://github.com/PowerShell/AIShell/tree/main/shell/agents/AIShell.Ollama.Agent"; /// /// This is the company added to /like and /dislike verbiage for who the telemetry helps. @@ -76,7 +76,7 @@ public void Initialize(AgentConfig config) LegalLinks = new(StringComparer.OrdinalIgnoreCase) { ["Ollama Docs"] = "https://github.com/ollama/ollama", - ["Prerequisites"] = "https://aka.ms/ollama/readme" + ["Prerequisites"] = "https://github.com/PowerShell/AIShell/tree/main/shell/agents/AIShell.Ollama.Agent" }; } @@ -125,21 +125,29 @@ public async Task ChatAsync(string input, IShell shell) if (Process.GetProcessesByName("ollama").Length is 0) { - host.RenderFullResponse("Please be sure the Ollama is installed and server is running. Check all the prerequisites in the README of this agent are met."); + host.MarkupWarningLine($"[[{Name}]]: Please be sure the Ollama is installed and server is running. Check all the prerequisites in the README of this agent are met."); return false; } - ResponseData ollamaResponse = await host.RunWithSpinnerAsync( - status: "Thinking ...", - func: async context => await _chatService.GetChatResponseAsync(context, input, token) - ).ConfigureAwait(false); + try + { + ResponseData ollamaResponse = await host.RunWithSpinnerAsync( + status: "Thinking ...", + func: async context => await _chatService.GetChatResponseAsync(context, input, token) + ).ConfigureAwait(false); - if (ollamaResponse is not null) + if (ollamaResponse is not null) + { + // render the content + host.RenderFullResponse(ollamaResponse.response); + } + } + catch (HttpRequestException) { - // render the content - host.RenderFullResponse(ollamaResponse.response); + host.MarkupWarningLine($"[[{Name}]]: Cannot serve the query due to the incorrect configuration. Please properly update the setting file."); + return false; } - + return true; } @@ -169,10 +177,23 @@ private void NewExampleSettingFile() { string SampleContent = $$""" { - // Declare Ollama model name. - "Model": "phi3", - // Declare Ollama endpoint. - "Endpoint": "http://localhost:11434/api/generate" + /* + To use Ollama API service: + + 1. Install Ollama: + winget install Ollama.Ollama + + 2. Start Ollama API server: + ollama serve + + 3. Install Ollama model: + ollama pull phi3 + */ + + // Declare Ollama model + "Model": "phi3", + // Declare Ollama endpoint + "Endpoint": "http://localhost:11434/api/generate" } """; File.WriteAllText(SettingFile, SampleContent, Encoding.UTF8); diff --git a/shell/agents/AIShell.Ollama.Agent/README.md b/shell/agents/AIShell.Ollama.Agent/README.md index 559d504a..baff9048 100644 --- a/shell/agents/AIShell.Ollama.Agent/README.md +++ b/shell/agents/AIShell.Ollama.Agent/README.md @@ -12,11 +12,29 @@ this agent you need to have Ollama installed and running. ## Configuration -Currently to change the model you will need to modify the query in the code in the -`OllamaChatService` class. The default model is `phi3`. +To configure the agent, run `/agent config ollama` to open up the setting file in your default editor, and then update the file based on the following example. -The default endpoint is `http://localhost:11434/api/generate` with `11434` being the default port. This can be changed in the code -and eventually will be added to a configuration file. +```json +{ + /* + To use Ollama API service: + + 1. Install Ollama: + winget install Ollama.Ollama + + 2. Start Ollama API server: + ollama serve + + 3. Install Ollama model: + ollama pull phi3 + */ + + // Declare Ollama model + "Model": "phi3", + // Declare Ollama endpoint + "Endpoint": "http://localhost:11434/api/generate" +} +``` ## Known Limitations @@ -24,5 +42,3 @@ and eventually will be added to a configuration file. queries - Streaming is currently not supported if you change the stream value to `true` in the data to send to the API it will not work -- Configuration is currently hard coded in the code and will be moved to a configuration file in the - future \ No newline at end of file From c0f83892b75d655ab423e765ce77c35939ecac2b Mon Sep 17 00:00:00 2001 From: Kris Borowinski Date: Tue, 19 Nov 2024 17:25:52 +0100 Subject: [PATCH 03/15] Implement config file watcher to reload settings on change --- .../AIShell.Ollama.Agent/OllamaAgent.cs | 67 ++++++++++++++++++- .../AIShell.Ollama.Agent/OllamaChatService.cs | 9 +++ 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs index 28e421f3..7c59960e 100644 --- a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs +++ b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs @@ -7,8 +7,16 @@ namespace AIShell.Ollama.Agent; public sealed class OllamaAgent : ILLMAgent { - private const string SettingFileName = "ollama.config.json"; + private bool _reloadSettings; + private bool _isDisposed; + private string _configRoot; private Settings _settings; + private FileSystemWatcher _watcher; + + /// + /// The name of setting file + /// + private const string SettingFileName = "ollama.config.json"; /// /// Gets the settings. @@ -52,7 +60,15 @@ public sealed class OllamaAgent : ILLMAgent /// public void Dispose() { + if (_isDisposed) + { + return; + } + + GC.SuppressFinalize(this); _chatService?.Dispose(); + _watcher.Dispose(); + _isDisposed = true; } /// @@ -61,7 +77,9 @@ public void Dispose() /// Agent configuration for any configuration file and other settings public void Initialize(AgentConfig config) { - SettingFile = Path.Combine(config.ConfigurationRoot, SettingFileName); + _configRoot = config.ConfigurationRoot; + + SettingFile = Path.Combine(_configRoot, SettingFileName); _settings = ReadSettings(); if (_settings is null) @@ -73,6 +91,13 @@ public void Initialize(AgentConfig config) _chatService = new OllamaChatService(_settings); + _watcher = new FileSystemWatcher(_configRoot, SettingFileName) + { + NotifyFilter = NotifyFilters.LastWrite, + EnableRaisingEvents = true, + }; + _watcher.Changed += OnSettingFileChange; + LegalLinks = new(StringComparer.OrdinalIgnoreCase) { ["Ollama Docs"] = "https://github.com/ollama/ollama", @@ -107,7 +132,16 @@ public void OnUserAction(UserActionPayload actionPayload) {} /// Refresh the current chat by starting a new chat session. /// This method allows an agent to reset chat states, interact with user for authentication, print welcome message, and more. /// - public Task RefreshChatAsync(IShell shell, bool force) => Task.CompletedTask; + public Task RefreshChatAsync(IShell shell, bool force) + { + if (force) + { + // Reload the setting file if needed. + ReloadSettings(); + } + + return Task.CompletedTask; + } /// /// Main chat function that takes the users input and passes it to the LLM and renders it. @@ -123,6 +157,9 @@ public async Task ChatAsync(string input, IShell shell) // get the cancellation token CancellationToken token = shell.CancellationToken; + // Reload the setting file if needed. + ReloadSettings(); + if (Process.GetProcessesByName("ollama").Length is 0) { host.MarkupWarningLine($"[[{Name}]]: Please be sure the Ollama is installed and server is running. Check all the prerequisites in the README of this agent are met."); @@ -151,6 +188,22 @@ public async Task ChatAsync(string input, IShell shell) return true; } + internal void ReloadSettings() + { + if (_reloadSettings) + { + _reloadSettings = false; + var settings = ReadSettings(); + if (settings is null) + { + return; + } + + _settings = settings; + _chatService.RefreshSettings(_settings); + } + } + private Settings ReadSettings() { Settings settings = null; @@ -173,6 +226,14 @@ private Settings ReadSettings() return settings; } + private void OnSettingFileChange(object sender, FileSystemEventArgs e) + { + if (e.ChangeType is WatcherChangeTypes.Changed) + { + _reloadSettings = true; + } + } + private void NewExampleSettingFile() { string SampleContent = $$""" diff --git a/shell/agents/AIShell.Ollama.Agent/OllamaChatService.cs b/shell/agents/AIShell.Ollama.Agent/OllamaChatService.cs index 63314dd7..b01a94f8 100644 --- a/shell/agents/AIShell.Ollama.Agent/OllamaChatService.cs +++ b/shell/agents/AIShell.Ollama.Agent/OllamaChatService.cs @@ -22,6 +22,15 @@ internal OllamaChatService(Settings settings) _client = new HttpClient(); } + /// + /// Refresh settings + /// + /// + internal void RefreshSettings(Settings settings) + { + _settings = settings; + } + /// /// Dispose of the http client /// From 0899ddbb11bd081f0ab621cec4613c23c3987986 Mon Sep 17 00:00:00 2001 From: Kris Borowinski Date: Tue, 19 Nov 2024 19:15:16 +0100 Subject: [PATCH 04/15] Do not force user to specify complete endpoint URL, just IP address and port; Precise warning message --- shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs | 4 ++-- .../agents/AIShell.Ollama.Agent/OllamaChatService.cs | 2 +- shell/agents/AIShell.Ollama.Agent/README.md | 2 +- shell/agents/AIShell.Ollama.Agent/Settings.cs | 11 +++++++---- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs index 7c59960e..9fa5aa70 100644 --- a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs +++ b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs @@ -181,7 +181,7 @@ public async Task ChatAsync(string input, IShell shell) } catch (HttpRequestException) { - host.MarkupWarningLine($"[[{Name}]]: Cannot serve the query due to the incorrect configuration. Please properly update the setting file."); + host.MarkupWarningLine($"[[{Name}]]: Cannot serve the query due to the endpoint or model misconfiguration. Please properly update the setting file."); return false; } @@ -254,7 +254,7 @@ ollama pull phi3 // Declare Ollama model "Model": "phi3", // Declare Ollama endpoint - "Endpoint": "http://localhost:11434/api/generate" + "Endpoint": "http://localhost:11434" } """; File.WriteAllText(SettingFile, SampleContent, Encoding.UTF8); diff --git a/shell/agents/AIShell.Ollama.Agent/OllamaChatService.cs b/shell/agents/AIShell.Ollama.Agent/OllamaChatService.cs index b01a94f8..746e5e4d 100644 --- a/shell/agents/AIShell.Ollama.Agent/OllamaChatService.cs +++ b/shell/agents/AIShell.Ollama.Agent/OllamaChatService.cs @@ -57,7 +57,7 @@ private HttpRequestMessage PrepareForChat(string input) var json = JsonSerializer.Serialize(requestData); var data = new StringContent(json, Encoding.UTF8, "application/json"); - var request = new HttpRequestMessage(HttpMethod.Post, _settings.Endpoint) { Content = data }; + var request = new HttpRequestMessage(HttpMethod.Post, $"{_settings.Endpoint}/api/generate") { Content = data }; return request; } diff --git a/shell/agents/AIShell.Ollama.Agent/README.md b/shell/agents/AIShell.Ollama.Agent/README.md index baff9048..bb7b3931 100644 --- a/shell/agents/AIShell.Ollama.Agent/README.md +++ b/shell/agents/AIShell.Ollama.Agent/README.md @@ -32,7 +32,7 @@ To configure the agent, run `/agent config ollama` to open up the setting file i // Declare Ollama model "Model": "phi3", // Declare Ollama endpoint - "Endpoint": "http://localhost:11434/api/generate" + "Endpoint": "http://localhost:11434" } ``` diff --git a/shell/agents/AIShell.Ollama.Agent/Settings.cs b/shell/agents/AIShell.Ollama.Agent/Settings.cs index 7ac3555b..fc6bcb28 100644 --- a/shell/agents/AIShell.Ollama.Agent/Settings.cs +++ b/shell/agents/AIShell.Ollama.Agent/Settings.cs @@ -5,13 +5,16 @@ namespace AIShell.Ollama.Agent; internal class Settings { - public string Model { get; set; } - public string Endpoint { get; set; } + private string _model; + private string _endpoint; + + public string Model => _model; + public string Endpoint => _endpoint; public Settings(ConfigData configData) { - Model = configData.Model; - Endpoint = configData.Endpoint; + _model = configData?.Model; + _endpoint = configData?.Endpoint.TrimEnd('/'); } } From bcc5d1e3e23b8995a7f45629667dad89381cbe33 Mon Sep 17 00:00:00 2001 From: Kris Borowinski Date: Tue, 19 Nov 2024 19:17:52 +0100 Subject: [PATCH 05/15] Change json to config so comments are not rendered on GitHub in red --- shell/agents/AIShell.Ollama.Agent/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/agents/AIShell.Ollama.Agent/README.md b/shell/agents/AIShell.Ollama.Agent/README.md index bb7b3931..2401b645 100644 --- a/shell/agents/AIShell.Ollama.Agent/README.md +++ b/shell/agents/AIShell.Ollama.Agent/README.md @@ -14,7 +14,7 @@ this agent you need to have Ollama installed and running. To configure the agent, run `/agent config ollama` to open up the setting file in your default editor, and then update the file based on the following example. -```json +```config { /* To use Ollama API service: From 9c358b861c4d94088e24498440617492d226702a Mon Sep 17 00:00:00 2001 From: Kris Borowinski Date: Tue, 19 Nov 2024 19:32:49 +0100 Subject: [PATCH 06/15] Revert to json and update the comments in the config file --- .../AIShell.Ollama.Agent/OllamaAgent.cs | 19 ++++------ shell/agents/AIShell.Ollama.Agent/README.md | 36 ++++++++----------- 2 files changed, 22 insertions(+), 33 deletions(-) diff --git a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs index 9fa5aa70..3ac2aa7c 100644 --- a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs +++ b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs @@ -238,18 +238,13 @@ private void NewExampleSettingFile() { string SampleContent = $$""" { - /* - To use Ollama API service: - - 1. Install Ollama: - winget install Ollama.Ollama - - 2. Start Ollama API server: - ollama serve - - 3. Install Ollama model: - ollama pull phi3 - */ + // To use Ollama API service: + // 1. Install Ollama: + // winget install Ollama.Ollama + // 2. Start Ollama API server: + // ollama serve + // 3. Install Ollama model: + // ollama pull phi3 // Declare Ollama model "Model": "phi3", diff --git a/shell/agents/AIShell.Ollama.Agent/README.md b/shell/agents/AIShell.Ollama.Agent/README.md index 2401b645..87375aec 100644 --- a/shell/agents/AIShell.Ollama.Agent/README.md +++ b/shell/agents/AIShell.Ollama.Agent/README.md @@ -5,34 +5,28 @@ this agent you need to have Ollama installed and running. ## Pre-requisites to using the agent -- Install [Ollama](https://github.com/ollama/ollama) -- Install a [Ollama model](https://github.com/ollama/ollama?tab=readme-ov-file#model-library), we - suggest using the `phi3` model as it is set as the default model in the code +- Install [Ollama](https://github.com/ollama/ollama) +- Install a [Ollama model](https://github.com/ollama/ollama?tab=readme-ov-file#model-library), we suggest using the `phi3` model as it is set as the default model in the code - [Start the Ollama API server](https://github.com/ollama/ollama?tab=readme-ov-file#start-ollama) ## Configuration To configure the agent, run `/agent config ollama` to open up the setting file in your default editor, and then update the file based on the following example. -```config +```json { - /* - To use Ollama API service: - - 1. Install Ollama: - winget install Ollama.Ollama - - 2. Start Ollama API server: - ollama serve - - 3. Install Ollama model: - ollama pull phi3 - */ - - // Declare Ollama model - "Model": "phi3", - // Declare Ollama endpoint - "Endpoint": "http://localhost:11434" + // To use Ollama API service: + // 1. Install Ollama: + // winget install Ollama.Ollama + // 2. Start Ollama API server: + // ollama serve + // 3. Install Ollama model: + // ollama pull phi3 + + // Declare Ollama model + "Model": "phi3", + // Declare Ollama endpoint + "Endpoint": "http://localhost:11434" } ``` From 9b57754a1e7cedd8682a8dd2d1bc011b6e126203 Mon Sep 17 00:00:00 2001 From: Kris Borowinski Date: Wed, 20 Nov 2024 17:03:29 +0100 Subject: [PATCH 07/15] Add model and endpoint self check; Change warnings to errors; Fix possible null exception --- .../AIShell.Ollama.Agent/OllamaAgent.cs | 27 ++++++++++++++++++- shell/agents/AIShell.Ollama.Agent/Settings.cs | 2 +- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs index 3ac2aa7c..d3a4dcba 100644 --- a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs +++ b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs @@ -166,6 +166,11 @@ public async Task ChatAsync(string input, IShell shell) return false; } + if (!SelfCheck(host)) + { + return false; + } + try { ResponseData ollamaResponse = await host.RunWithSpinnerAsync( @@ -181,7 +186,7 @@ public async Task ChatAsync(string input, IShell shell) } catch (HttpRequestException) { - host.MarkupWarningLine($"[[{Name}]]: Cannot serve the query due to the endpoint or model misconfiguration. Please properly update the setting file."); + host.WriteErrorLine($"[{Name}]: Cannot serve the query due to the Endpoint or Model misconfiguration. Please properly update the setting file."); return false; } @@ -234,6 +239,26 @@ private void OnSettingFileChange(object sender, FileSystemEventArgs e) } } + internal bool SelfCheck(IHost host) + { + var settings = new (string settingValue, string settingName)[] + { + (_settings?.Model, "Model"), + (_settings?.Endpoint, "Endpoint") + }; + + foreach (var (settingValue, settingName) in settings) + { + if (string.IsNullOrWhiteSpace(settingValue)) + { + host.WriteErrorLine($"[{Name}]: {settingName} is undefined. Please declare it in the setting file."); + return false; + } + } + + return true; + } + private void NewExampleSettingFile() { string SampleContent = $$""" diff --git a/shell/agents/AIShell.Ollama.Agent/Settings.cs b/shell/agents/AIShell.Ollama.Agent/Settings.cs index fc6bcb28..ad7dccef 100644 --- a/shell/agents/AIShell.Ollama.Agent/Settings.cs +++ b/shell/agents/AIShell.Ollama.Agent/Settings.cs @@ -14,7 +14,7 @@ internal class Settings public Settings(ConfigData configData) { _model = configData?.Model; - _endpoint = configData?.Endpoint.TrimEnd('/'); + _endpoint = configData?.Endpoint?.TrimEnd('/'); } } From d9d6facf07d2e389cb8ecbc1222941e6790cfaf1 Mon Sep 17 00:00:00 2001 From: Kris Borowinski Date: Thu, 21 Nov 2024 10:06:10 +0100 Subject: [PATCH 08/15] Switch to OllamaSharp nuget package to simplify agent code and support streaming --- .../AIShell.Ollama.Agent.csproj | 7 +- .../AIShell.Ollama.Agent/OllamaAgent.cs | 91 ++++++++++++------- .../AIShell.Ollama.Agent/OllamaChatService.cs | 91 ------------------- .../AIShell.Ollama.Agent/OllamaSchema.cs | 34 ------- shell/agents/AIShell.Ollama.Agent/README.md | 6 +- shell/agents/AIShell.Ollama.Agent/Settings.cs | 4 + shell/nuget.config | 3 +- 7 files changed, 75 insertions(+), 161 deletions(-) delete mode 100644 shell/agents/AIShell.Ollama.Agent/OllamaChatService.cs delete mode 100644 shell/agents/AIShell.Ollama.Agent/OllamaSchema.cs diff --git a/shell/agents/AIShell.Ollama.Agent/AIShell.Ollama.Agent.csproj b/shell/agents/AIShell.Ollama.Agent/AIShell.Ollama.Agent.csproj index 56068bee..57d0bcdc 100644 --- a/shell/agents/AIShell.Ollama.Agent/AIShell.Ollama.Agent.csproj +++ b/shell/agents/AIShell.Ollama.Agent/AIShell.Ollama.Agent.csproj @@ -3,7 +3,8 @@ net8.0 enable - true + true + true false @@ -16,6 +17,10 @@ + + + + false diff --git a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs index d3a4dcba..7897e447 100644 --- a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs +++ b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs @@ -2,6 +2,8 @@ using System.Text; using System.Text.Json; using AIShell.Abstraction; +using OllamaSharp; +using OllamaSharp.Models; namespace AIShell.Ollama.Agent; @@ -11,6 +13,8 @@ public sealed class OllamaAgent : ILLMAgent private bool _isDisposed; private string _configRoot; private Settings _settings; + private OllamaApiClient _client; + private GenerateRequest _request; private FileSystemWatcher _watcher; /// @@ -48,12 +52,11 @@ public sealed class OllamaAgent : ILLMAgent /// /// These are any optional legal/additional information links you want to provide at start up /// - public Dictionary LegalLinks { private set; get; } - - /// - /// This is the chat service to call the API from - /// - private OllamaChatService _chatService; + public Dictionary LegalLinks { private set; get; } = new(StringComparer.OrdinalIgnoreCase) + { + ["Ollama Docs"] = "https://github.com/ollama/ollama", + ["Prerequisites"] = "https://github.com/PowerShell/AIShell/tree/main/shell/agents/AIShell.Ollama.Agent" + }; /// /// Dispose method to clean up the unmanaged resource of the chatService @@ -66,7 +69,6 @@ public void Dispose() } GC.SuppressFinalize(this); - _chatService?.Dispose(); _watcher.Dispose(); _isDisposed = true; } @@ -89,7 +91,11 @@ public void Initialize(AgentConfig config) _settings = ReadSettings(); } - _chatService = new OllamaChatService(_settings); + _request = new GenerateRequest() + { + Model = _settings.Model, + Stream = _settings.Stream + }; _watcher = new FileSystemWatcher(_configRoot, SettingFileName) { @@ -97,12 +103,6 @@ public void Initialize(AgentConfig config) EnableRaisingEvents = true, }; _watcher.Changed += OnSettingFileChange; - - LegalLinks = new(StringComparer.OrdinalIgnoreCase) - { - ["Ollama Docs"] = "https://github.com/ollama/ollama", - ["Prerequisites"] = "https://github.com/PowerShell/AIShell/tree/main/shell/agents/AIShell.Ollama.Agent" - }; } /// @@ -166,34 +166,62 @@ public async Task ChatAsync(string input, IShell shell) return false; } + // Self check settings Model and Endpoint if (!SelfCheck(host)) { return false; } + // Update request + _request.Prompt = input; + _request.Model = _settings.Model; + _request.Stream = _settings.Stream; + + _client = new OllamaApiClient(_settings.Endpoint); + try { - ResponseData ollamaResponse = await host.RunWithSpinnerAsync( - status: "Thinking ...", - func: async context => await _chatService.GetChatResponseAsync(context, input, token) - ).ConfigureAwait(false); - - if (ollamaResponse is not null) + if (_request.Stream) + { + using IStreamRender streamingRender = host.NewStreamRender(token); + + // Directly process the stream when no spinner is needed + await foreach (var ollamaStream in _client.GenerateAsync(_request, token)) + { + // Update the render + streamingRender.Refresh(ollamaStream.Response); + } + } + else { - // render the content - host.RenderFullResponse(ollamaResponse.response); + // Build single response with spinner + var ollamaResponse = await host.RunWithSpinnerAsync( + status: "Thinking ...", + func: async () => { return await _client.GenerateAsync(_request, token).StreamToEndAsync(); } + ).ConfigureAwait(false); + + // Render the full response + host.RenderFullResponse(ollamaResponse.Response); } } - catch (HttpRequestException) + catch (OperationCanceledException) + { + // Ignore the cancellation exception. + } + catch (HttpRequestException e) { - host.WriteErrorLine($"[{Name}]: Cannot serve the query due to the Endpoint or Model misconfiguration. Please properly update the setting file."); + host.WriteErrorLine($"[{Name}]: {e.Message}"); + host.WriteErrorLine($"[{Name}]: Selected Model : \"{_settings.Model}\""); + host.WriteErrorLine($"[{Name}]: Selected Endpoint : \"{_settings.Endpoint}\""); + host.WriteErrorLine($"[{Name}]: Configuration File: \"{SettingFile}\""); return false; } return true; } - internal void ReloadSettings() + + private void ReloadSettings() { if (_reloadSettings) { @@ -205,7 +233,6 @@ internal void ReloadSettings() } _settings = settings; - _chatService.RefreshSettings(_settings); } } @@ -239,19 +266,19 @@ private void OnSettingFileChange(object sender, FileSystemEventArgs e) } } - internal bool SelfCheck(IHost host) + private bool SelfCheck(IHost host) { var settings = new (string settingValue, string settingName)[] { - (_settings?.Model, "Model"), - (_settings?.Endpoint, "Endpoint") + (_settings.Model, "Model"), + (_settings.Endpoint, "Endpoint") }; foreach (var (settingValue, settingName) in settings) { if (string.IsNullOrWhiteSpace(settingValue)) { - host.WriteErrorLine($"[{Name}]: {settingName} is undefined. Please declare it in the setting file."); + host.WriteErrorLine($"[{Name}]: {settingName} is undefined in the settings file: \"{SettingFile}\""); return false; } } @@ -274,7 +301,9 @@ private void NewExampleSettingFile() // Declare Ollama model "Model": "phi3", // Declare Ollama endpoint - "Endpoint": "http://localhost:11434" + "Endpoint": "http://localhost:11434", + // Enable Ollama streaming + "Stream": false } """; File.WriteAllText(SettingFile, SampleContent, Encoding.UTF8); diff --git a/shell/agents/AIShell.Ollama.Agent/OllamaChatService.cs b/shell/agents/AIShell.Ollama.Agent/OllamaChatService.cs deleted file mode 100644 index 746e5e4d..00000000 --- a/shell/agents/AIShell.Ollama.Agent/OllamaChatService.cs +++ /dev/null @@ -1,91 +0,0 @@ -using System.Text; -using System.Text.Json; - -using AIShell.Abstraction; - -namespace AIShell.Ollama.Agent; - -internal class OllamaChatService : IDisposable -{ - private Settings _settings; - /// - /// Http client - /// - private readonly HttpClient _client; - - /// - /// Initialization method to initialize the http client - /// - internal OllamaChatService(Settings settings) - { - _settings = settings; - _client = new HttpClient(); - } - - /// - /// Refresh settings - /// - /// - internal void RefreshSettings(Settings settings) - { - _settings = settings; - } - - /// - /// Dispose of the http client - /// - public void Dispose() - { - _client.Dispose(); - } - - /// - /// Preparing chat with data to be sent - /// - /// The user input from the chat experience - /// The HTTP request message - private HttpRequestMessage PrepareForChat(string input) - { - // Main data to send to the endpoint - var requestData = new Query - { - model = _settings.Model, - prompt = input, - stream = false - }; - - var json = JsonSerializer.Serialize(requestData); - - var data = new StringContent(json, Encoding.UTF8, "application/json"); - var request = new HttpRequestMessage(HttpMethod.Post, $"{_settings.Endpoint}/api/generate") { Content = data }; - - return request; - } - - /// - /// Getting the chat response async - /// - /// Interface for the status context used when displaying a spinner. - /// The user input from the chat experience - /// The cancellation token to exit out of request - /// Response data from the API call - internal async Task GetChatResponseAsync(IStatusContext context, string input, CancellationToken cancellationToken) - { - try - { - HttpRequestMessage request = PrepareForChat(input); - HttpResponseMessage response = await _client.SendAsync(request, cancellationToken); - response.EnsureSuccessStatusCode(); - - context?.Status("Receiving Payload ..."); - var content = await response.Content.ReadAsStreamAsync(cancellationToken); - return JsonSerializer.Deserialize(content); - } - catch (OperationCanceledException) - { - // Operation was cancelled by user. - } - - return null; - } -} diff --git a/shell/agents/AIShell.Ollama.Agent/OllamaSchema.cs b/shell/agents/AIShell.Ollama.Agent/OllamaSchema.cs deleted file mode 100644 index 7dcd42c7..00000000 --- a/shell/agents/AIShell.Ollama.Agent/OllamaSchema.cs +++ /dev/null @@ -1,34 +0,0 @@ -namespace AIShell.Ollama.Agent; - -// Query class for the data to send to the endpoint -internal class Query -{ - public string prompt { get; set; } - public string model { get; set; } - public bool stream { get; set; } -} - -// Response data schema -internal class ResponseData -{ - public string model { get; set; } - public string created_at { get; set; } - public string response { get; set; } - public bool done { get; set; } - public string done_reason { get; set; } - public int[] context { get; set; } - public double total_duration { get; set; } - public long load_duration { get; set; } - public int prompt_eval_count { get; set; } - public int prompt_eval_duration { get; set; } - public int eval_count { get; set; } - public long eval_duration { get; set; } -} - -internal class OllamaResponse -{ - public int Status { get; set; } - public string Error { get; set; } - public string Api_version { get; set; } - public ResponseData Data { get; set; } -} diff --git a/shell/agents/AIShell.Ollama.Agent/README.md b/shell/agents/AIShell.Ollama.Agent/README.md index 87375aec..bf258309 100644 --- a/shell/agents/AIShell.Ollama.Agent/README.md +++ b/shell/agents/AIShell.Ollama.Agent/README.md @@ -26,7 +26,9 @@ To configure the agent, run `/agent config ollama` to open up the setting file i // Declare Ollama model "Model": "phi3", // Declare Ollama endpoint - "Endpoint": "http://localhost:11434" + "Endpoint": "http://localhost:11434", + // Enable Ollama streaming + "Stream": false } ``` @@ -34,5 +36,3 @@ To configure the agent, run `/agent config ollama` to open up the setting file i - There is no history shared across queries so the model will not be able to remember previous queries -- Streaming is currently not supported if you change the stream value to `true` in the data to send - to the API it will not work diff --git a/shell/agents/AIShell.Ollama.Agent/Settings.cs b/shell/agents/AIShell.Ollama.Agent/Settings.cs index ad7dccef..abe972d2 100644 --- a/shell/agents/AIShell.Ollama.Agent/Settings.cs +++ b/shell/agents/AIShell.Ollama.Agent/Settings.cs @@ -7,14 +7,17 @@ internal class Settings { private string _model; private string _endpoint; + private bool _stream; public string Model => _model; public string Endpoint => _endpoint; + public bool Stream => _stream; public Settings(ConfigData configData) { _model = configData?.Model; _endpoint = configData?.Endpoint?.TrimEnd('/'); + _stream = configData?.Stream ?? false; } } @@ -22,6 +25,7 @@ internal class ConfigData { public string Model { get; set; } public string Endpoint { get; set; } + public bool Stream { get; set; } } /// diff --git a/shell/nuget.config b/shell/nuget.config index a10ce9b3..9f3c95c6 100644 --- a/shell/nuget.config +++ b/shell/nuget.config @@ -1,8 +1,9 @@ - + + From 464d63365c60d8679a1d63e1560db105b7dbf853 Mon Sep 17 00:00:00 2001 From: Kris Borowinski Date: Thu, 21 Nov 2024 16:24:00 +0100 Subject: [PATCH 09/15] Add context support --- shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs | 14 ++++++++++++++ shell/agents/AIShell.Ollama.Agent/README.md | 4 ---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs index 7897e447..ea3f087e 100644 --- a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs +++ b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs @@ -177,6 +177,7 @@ public async Task ChatAsync(string input, IShell shell) _request.Model = _settings.Model; _request.Stream = _settings.Stream; + // Ollama client is created per chat with reloaded settings _client = new OllamaApiClient(_settings.Endpoint); try @@ -185,12 +186,22 @@ public async Task ChatAsync(string input, IShell shell) { using IStreamRender streamingRender = host.NewStreamRender(token); + // Last stream response has context value + GenerateDoneResponseStream ollamaLastStream = null; + // Directly process the stream when no spinner is needed await foreach (var ollamaStream in _client.GenerateAsync(_request, token)) { // Update the render streamingRender.Refresh(ollamaStream.Response); + if (ollamaStream.Done) + { + ollamaLastStream = (GenerateDoneResponseStream)ollamaStream; + } } + + // Update request context + _request.Context = ollamaLastStream.Context; } else { @@ -200,6 +211,9 @@ public async Task ChatAsync(string input, IShell shell) func: async () => { return await _client.GenerateAsync(_request, token).StreamToEndAsync(); } ).ConfigureAwait(false); + // Update request context + _request.Context = ollamaResponse.Context; + // Render the full response host.RenderFullResponse(ollamaResponse.Response); } diff --git a/shell/agents/AIShell.Ollama.Agent/README.md b/shell/agents/AIShell.Ollama.Agent/README.md index bf258309..1c316cfd 100644 --- a/shell/agents/AIShell.Ollama.Agent/README.md +++ b/shell/agents/AIShell.Ollama.Agent/README.md @@ -32,7 +32,3 @@ To configure the agent, run `/agent config ollama` to open up the setting file i } ``` -## Known Limitations - -- There is no history shared across queries so the model will not be able to remember previous - queries From fe68d94526a9a2981b50e85fa376a8559a3250d9 Mon Sep 17 00:00:00 2001 From: Kris Borowinski Date: Fri, 22 Nov 2024 08:00:41 +0100 Subject: [PATCH 10/15] Reset context on refresh --- shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs index ea3f087e..7ca72a01 100644 --- a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs +++ b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs @@ -138,6 +138,9 @@ public Task RefreshChatAsync(IShell shell, bool force) { // Reload the setting file if needed. ReloadSettings(); + + // Reset context + _request.Context = null; } return Task.CompletedTask; From fd95d984fc83330c69ccfa2298ecff2e16388d43 Mon Sep 17 00:00:00 2001 From: Kris Borowinski Date: Sat, 23 Nov 2024 18:44:18 +0100 Subject: [PATCH 11/15] Implement changes as suggested in the code review --- .../AIShell.Ollama.Agent.csproj | 10 +- .../AIShell.Ollama.Agent/OllamaAgent.cs | 112 ++++++++---------- shell/agents/AIShell.Ollama.Agent/README.md | 24 ++-- shell/agents/AIShell.Ollama.Agent/Settings.cs | 27 +++-- 4 files changed, 84 insertions(+), 89 deletions(-) diff --git a/shell/agents/AIShell.Ollama.Agent/AIShell.Ollama.Agent.csproj b/shell/agents/AIShell.Ollama.Agent/AIShell.Ollama.Agent.csproj index 57d0bcdc..49231b9e 100644 --- a/shell/agents/AIShell.Ollama.Agent/AIShell.Ollama.Agent.csproj +++ b/shell/agents/AIShell.Ollama.Agent/AIShell.Ollama.Agent.csproj @@ -3,8 +3,8 @@ net8.0 enable - true - true + true + true false @@ -17,10 +17,10 @@ - + - + false @@ -29,4 +29,4 @@ - + \ No newline at end of file diff --git a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs index 7ca72a01..3679a08d 100644 --- a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs +++ b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs @@ -91,12 +91,13 @@ public void Initialize(AgentConfig config) _settings = ReadSettings(); } - _request = new GenerateRequest() - { - Model = _settings.Model, - Stream = _settings.Stream - }; + // Create Ollama request + _request = new GenerateRequest(); + // Create Ollama client + _client = new OllamaApiClient(_settings.Endpoint); + + // Watch for changes to the settings file _watcher = new FileSystemWatcher(_configRoot, SettingFileName) { NotifyFilter = NotifyFilters.LastWrite, @@ -165,46 +166,53 @@ public async Task ChatAsync(string input, IShell shell) if (Process.GetProcessesByName("ollama").Length is 0) { - host.MarkupWarningLine($"[[{Name}]]: Please be sure the Ollama is installed and server is running. Check all the prerequisites in the README of this agent are met."); + host.WriteErrorLine("Please be sure the Ollama is installed and server is running. Check all the prerequisites in the README of this agent are met."); return false; } - // Self check settings Model and Endpoint - if (!SelfCheck(host)) - { - return false; - } - - // Update request + // Prepare request _request.Prompt = input; _request.Model = _settings.Model; _request.Stream = _settings.Stream; - // Ollama client is created per chat with reloaded settings - _client = new OllamaApiClient(_settings.Endpoint); - try { if (_request.Stream) { using IStreamRender streamingRender = host.NewStreamRender(token); - // Last stream response has context value - GenerateDoneResponseStream ollamaLastStream = null; + // Wait for the stream with the spinner running + var ollamaStreamEnumerator = await host.RunWithSpinnerAsync( + status: "Thinking ...", + func: async () => + { + // Start generating the stream asynchronously and return an enumerator + var enumerator = _client.GenerateAsync(_request, token).GetAsyncEnumerator(); + if (await enumerator.MoveNextAsync().ConfigureAwait(false)) + { + return enumerator; + } + return null; + } + ).ConfigureAwait(false); - // Directly process the stream when no spinner is needed - await foreach (var ollamaStream in _client.GenerateAsync(_request, token)) + if (ollamaStreamEnumerator is not null) { - // Update the render - streamingRender.Refresh(ollamaStream.Response); - if (ollamaStream.Done) + do { - ollamaLastStream = (GenerateDoneResponseStream)ollamaStream; - } + var currentStream = ollamaStreamEnumerator.Current; + + // Update the render with stream response + streamingRender.Refresh(currentStream.Response); + + if (currentStream.Done) + { + // If the stream is complete, update the request context with the last stream context + var ollamaLastStream = (GenerateDoneResponseStream)currentStream; + _request.Context = ollamaLastStream.Context; + } + } while (await ollamaStreamEnumerator.MoveNextAsync().ConfigureAwait(false)); } - - // Update request context - _request.Context = ollamaLastStream.Context; } else { @@ -227,17 +235,15 @@ public async Task ChatAsync(string input, IShell shell) } catch (HttpRequestException e) { - host.WriteErrorLine($"[{Name}]: {e.Message}"); - host.WriteErrorLine($"[{Name}]: Selected Model : \"{_settings.Model}\""); - host.WriteErrorLine($"[{Name}]: Selected Endpoint : \"{_settings.Endpoint}\""); - host.WriteErrorLine($"[{Name}]: Configuration File: \"{SettingFile}\""); - return false; + host.WriteErrorLine($"{e.Message}"); + host.WriteErrorLine($"Ollama model: \"{_settings.Model}\""); + host.WriteErrorLine($"Ollama endpoint: \"{_settings.Endpoint}\""); + host.WriteErrorLine($"Ollama settings: \"{SettingFile}\""); } return true; } - private void ReloadSettings() { if (_reloadSettings) @@ -250,6 +256,15 @@ private void ReloadSettings() } _settings = settings; + + // Check if the endpoint has changed + bool isEndpointChanged = !string.Equals(_settings.Endpoint, _client.Uri.OriginalString, StringComparison.OrdinalIgnoreCase); + + if (isEndpointChanged) + { + // Create a new client with updated endpoint + _client = new OllamaApiClient(_settings.Endpoint); + } } } @@ -283,37 +298,14 @@ private void OnSettingFileChange(object sender, FileSystemEventArgs e) } } - private bool SelfCheck(IHost host) - { - var settings = new (string settingValue, string settingName)[] - { - (_settings.Model, "Model"), - (_settings.Endpoint, "Endpoint") - }; - - foreach (var (settingValue, settingName) in settings) - { - if (string.IsNullOrWhiteSpace(settingValue)) - { - host.WriteErrorLine($"[{Name}]: {settingName} is undefined in the settings file: \"{SettingFile}\""); - return false; - } - } - - return true; - } - private void NewExampleSettingFile() { string SampleContent = $$""" { // To use Ollama API service: - // 1. Install Ollama: - // winget install Ollama.Ollama - // 2. Start Ollama API server: - // ollama serve - // 3. Install Ollama model: - // ollama pull phi3 + // 1. Install Ollama: `winget install Ollama.Ollama` + // 2. Start Ollama API server: `ollama serve` + // 3. Install Ollama model: `ollama pull phi3` // Declare Ollama model "Model": "phi3", diff --git a/shell/agents/AIShell.Ollama.Agent/README.md b/shell/agents/AIShell.Ollama.Agent/README.md index 1c316cfd..f25a27e2 100644 --- a/shell/agents/AIShell.Ollama.Agent/README.md +++ b/shell/agents/AIShell.Ollama.Agent/README.md @@ -13,22 +13,18 @@ this agent you need to have Ollama installed and running. To configure the agent, run `/agent config ollama` to open up the setting file in your default editor, and then update the file based on the following example. -```json +```jsonc { - // To use Ollama API service: - // 1. Install Ollama: - // winget install Ollama.Ollama - // 2. Start Ollama API server: - // ollama serve - // 3. Install Ollama model: - // ollama pull phi3 - - // Declare Ollama model - "Model": "phi3", - // Declare Ollama endpoint - "Endpoint": "http://localhost:11434", + // To use Ollama API service: + // 1. Install Ollama: `winget install Ollama.Ollama` + // 2. Start Ollama API server: `ollama serve` + // 3. Install Ollama model: `ollama pull phi3` + + // Declare Ollama model + "Model": "phi3", + // Declare Ollama endpoint + "Endpoint": "http://localhost:11434", // Enable Ollama streaming "Stream": false } ``` - diff --git a/shell/agents/AIShell.Ollama.Agent/Settings.cs b/shell/agents/AIShell.Ollama.Agent/Settings.cs index abe972d2..11ebd8de 100644 --- a/shell/agents/AIShell.Ollama.Agent/Settings.cs +++ b/shell/agents/AIShell.Ollama.Agent/Settings.cs @@ -5,19 +5,26 @@ namespace AIShell.Ollama.Agent; internal class Settings { - private string _model; - private string _endpoint; - private bool _stream; - - public string Model => _model; - public string Endpoint => _endpoint; - public bool Stream => _stream; + public string Model { get; } + public string Endpoint { get; } + public bool Stream { get; } public Settings(ConfigData configData) { - _model = configData?.Model; - _endpoint = configData?.Endpoint?.TrimEnd('/'); - _stream = configData?.Stream ?? false; + // Validate Model and Endpoint for null or empty values + if (string.IsNullOrWhiteSpace(configData.Model)) + { + throw new ArgumentException("\"Model\" key is missing."); + } + + if (string.IsNullOrWhiteSpace(configData.Endpoint)) + { + throw new ArgumentException("\"Endpoint\" key is missing."); + } + + Model = configData.Model; + Endpoint = configData.Endpoint; + Stream = configData.Stream; } } From ee76005ecc352df3d4aa7733b72037c401d75ad6 Mon Sep 17 00:00:00 2001 From: Kris Borowinski Date: Sat, 23 Nov 2024 21:07:25 +0100 Subject: [PATCH 12/15] Add missing new line at the end of file --- shell/agents/AIShell.Ollama.Agent/AIShell.Ollama.Agent.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/agents/AIShell.Ollama.Agent/AIShell.Ollama.Agent.csproj b/shell/agents/AIShell.Ollama.Agent/AIShell.Ollama.Agent.csproj index 49231b9e..d03afbaa 100644 --- a/shell/agents/AIShell.Ollama.Agent/AIShell.Ollama.Agent.csproj +++ b/shell/agents/AIShell.Ollama.Agent/AIShell.Ollama.Agent.csproj @@ -29,4 +29,4 @@ - \ No newline at end of file + From cdbee047e896f04e77d0078e6920470909cbdbd2 Mon Sep 17 00:00:00 2001 From: Kris Borowinski Date: Sun, 24 Nov 2024 14:11:37 +0100 Subject: [PATCH 13/15] Check if ollama is running only for local endpoint --- shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs index 3679a08d..aa834e90 100644 --- a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs +++ b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs @@ -1,13 +1,14 @@ using System.Diagnostics; using System.Text; using System.Text.Json; +using System.Text.RegularExpressions; using AIShell.Abstraction; using OllamaSharp; using OllamaSharp.Models; namespace AIShell.Ollama.Agent; -public sealed class OllamaAgent : ILLMAgent +public sealed partial class OllamaAgent : ILLMAgent { private bool _reloadSettings; private bool _isDisposed; @@ -164,7 +165,7 @@ public async Task ChatAsync(string input, IShell shell) // Reload the setting file if needed. ReloadSettings(); - if (Process.GetProcessesByName("ollama").Length is 0) + if (IsLocalHost().IsMatch(_client.Uri.Host) && Process.GetProcessesByName("ollama").Length is 0) { host.WriteErrorLine("Please be sure the Ollama is installed and server is running. Check all the prerequisites in the README of this agent are met."); return false; @@ -317,4 +318,11 @@ private void NewExampleSettingFile() """; File.WriteAllText(SettingFile, SampleContent, Encoding.UTF8); } + + /// + /// Defines a source-generated regular expression to match localhost addresses + /// (e.g., "localhost", "127.0.0.1", "::1") with case-insensitivity. + /// + [GeneratedRegex("^(localhost|127\\.0\\.0\\.1|::1)$", RegexOptions.IgnoreCase)] + internal partial Regex IsLocalHost(); } From 29b4c49ec3fa19dd7f4421974e8839bf0cf7e52f Mon Sep 17 00:00:00 2001 From: Kris Borowinski Date: Sun, 24 Nov 2024 14:52:40 +0100 Subject: [PATCH 14/15] Update description and readme that ollama is also supported remotely; Fix localhost IPv6 Uri notation --- shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs | 10 +++++----- shell/agents/AIShell.Ollama.Agent/README.md | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs index aa834e90..62d6e7ed 100644 --- a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs +++ b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs @@ -36,7 +36,7 @@ public sealed partial class OllamaAgent : ILLMAgent /// /// The description of the agent to be shown at start up /// - public string Description => "This is an AI assistant to interact with a language model running locally by utilizing the Ollama CLI tool. Be sure to follow all prerequisites in https://github.com/PowerShell/AIShell/tree/main/shell/agents/AIShell.Ollama.Agent"; + public string Description => "This is an AI assistant to interact with a language model running locally or remotely by utilizing the Ollama API. Be sure to follow all prerequisites in https://github.com/PowerShell/AIShell/tree/main/shell/agents/AIShell.Ollama.Agent"; /// /// This is the company added to /like and /dislike verbiage for who the telemetry helps. @@ -159,7 +159,7 @@ public async Task ChatAsync(string input, IShell shell) // Get the shell host IHost host = shell.Host; - // get the cancellation token + // Get the cancellation token CancellationToken token = shell.CancellationToken; // Reload the setting file if needed. @@ -320,9 +320,9 @@ private void NewExampleSettingFile() } /// - /// Defines a source-generated regular expression to match localhost addresses - /// (e.g., "localhost", "127.0.0.1", "::1") with case-insensitivity. + /// Defines a generated regular expression to match localhost addresses + /// "localhost", "127.0.0.1" and "[::1]" with case-insensitivity. /// - [GeneratedRegex("^(localhost|127\\.0\\.0\\.1|::1)$", RegexOptions.IgnoreCase)] + [GeneratedRegex("^(localhost|127\\.0\\.0\\.1|\\[::1\\])$", RegexOptions.IgnoreCase)] internal partial Regex IsLocalHost(); } diff --git a/shell/agents/AIShell.Ollama.Agent/README.md b/shell/agents/AIShell.Ollama.Agent/README.md index f25a27e2..bd8e91e1 100644 --- a/shell/agents/AIShell.Ollama.Agent/README.md +++ b/shell/agents/AIShell.Ollama.Agent/README.md @@ -1,9 +1,9 @@ # Ollama Plugin -This agent is used to interact with a language model running locally by utilizing the Ollama API. Before using -this agent you need to have Ollama installed and running. +This agent is used to interact with a language model running locally or remotely by utilizing the Ollama API. +Before using this agent locally you need to have Ollama installed and running. -## Pre-requisites to using the agent +## Pre-requisites to using the agent locally - Install [Ollama](https://github.com/ollama/ollama) - Install a [Ollama model](https://github.com/ollama/ollama?tab=readme-ov-file#model-library), we suggest using the `phi3` model as it is set as the default model in the code From b798172eb31391f319f3b648e833e9717beb59a8 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Tue, 3 Dec 2024 13:05:21 -0800 Subject: [PATCH 15/15] Minor updates --- shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs | 11 ++++++----- shell/nuget.config | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs index 62d6e7ed..480cf383 100644 --- a/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs +++ b/shell/agents/AIShell.Ollama.Agent/OllamaAgent.cs @@ -180,15 +180,13 @@ public async Task ChatAsync(string input, IShell shell) { if (_request.Stream) { - using IStreamRender streamingRender = host.NewStreamRender(token); - // Wait for the stream with the spinner running var ollamaStreamEnumerator = await host.RunWithSpinnerAsync( status: "Thinking ...", func: async () => { // Start generating the stream asynchronously and return an enumerator - var enumerator = _client.GenerateAsync(_request, token).GetAsyncEnumerator(); + var enumerator = _client.GenerateAsync(_request, token).GetAsyncEnumerator(token); if (await enumerator.MoveNextAsync().ConfigureAwait(false)) { return enumerator; @@ -199,6 +197,8 @@ public async Task ChatAsync(string input, IShell shell) if (ollamaStreamEnumerator is not null) { + using IStreamRender streamingRender = host.NewStreamRender(token); + do { var currentStream = ollamaStreamEnumerator.Current; @@ -212,7 +212,8 @@ public async Task ChatAsync(string input, IShell shell) var ollamaLastStream = (GenerateDoneResponseStream)currentStream; _request.Context = ollamaLastStream.Context; } - } while (await ollamaStreamEnumerator.MoveNextAsync().ConfigureAwait(false)); + } + while (await ollamaStreamEnumerator.MoveNextAsync().ConfigureAwait(false)); } } else @@ -301,7 +302,7 @@ private void OnSettingFileChange(object sender, FileSystemEventArgs e) private void NewExampleSettingFile() { - string SampleContent = $$""" + string SampleContent = """ { // To use Ollama API service: // 1. Install Ollama: `winget install Ollama.Ollama` diff --git a/shell/nuget.config b/shell/nuget.config index 9f3c95c6..f78f7db6 100644 --- a/shell/nuget.config +++ b/shell/nuget.config @@ -3,7 +3,6 @@ -