diff --git a/.gitignore b/.gitignore index b5457435..fd2d83a8 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ dist *.user .env *.xmldocs.xml +**/CONDUCTORSHARP_HEALTH.json \ No newline at end of file diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 17cc4b44..f505adaa 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -1,7 +1,30 @@ version: '3.4' services: + + conductorsharp.noapi: + healthcheck: + test: bash -c "[ -f /app/CONDUCTORSHARP_HEALTH.json ]" + interval: 60s + retries: 5 + start_period: 20s + timeout: 10s + + conductorsharp.definitions: + healthcheck: + test: bash -c "[ -f /app/CONDUCTORSHARP_HEALTH.json ]" + interval: 60s + retries: 5 + start_period: 20s + timeout: 10s + conductorsharp.apienabled: + healthcheck: + test: bash -c "[ -f /app/CONDUCTORSHARP_HEALTH.json ]" + interval: 60s + retries: 5 + start_period: 20s + timeout: 10s environment: - ASPNETCORE_ENVIRONMENT=Development ports: diff --git a/examples/ConductorSharp.ApiEnabled/Extensions/HostConfiguration.cs b/examples/ConductorSharp.ApiEnabled/Extensions/HostConfiguration.cs index 5ea3188d..d31a44bf 100644 --- a/examples/ConductorSharp.ApiEnabled/Extensions/HostConfiguration.cs +++ b/examples/ConductorSharp.ApiEnabled/Extensions/HostConfiguration.cs @@ -1,5 +1,6 @@ using Autofac; using ConductorSharp.Engine.Extensions; +using ConductorSharp.Engine.Health; using MediatR.Extensions.Autofac.DependencyInjection; namespace ConductorSharp.ApiEnabled.Extensions; @@ -22,6 +23,7 @@ public static IHostBuilder ConfigureApiEnabled(this IHostBuilder hostBuilder, Co longPollInterval: configuration.GetValue("Conductor:LongPollInterval"), domain: configuration.GetValue("Conductor:WorkerDomain") ) + .SetHealthCheckService() .AddPipelines(pipelines => { pipelines.AddContextLogging(); diff --git a/examples/ConductorSharp.ApiEnabled/Program.cs b/examples/ConductorSharp.ApiEnabled/Program.cs index dc2ddbe9..a1ee324e 100644 --- a/examples/ConductorSharp.ApiEnabled/Program.cs +++ b/examples/ConductorSharp.ApiEnabled/Program.cs @@ -1,5 +1,6 @@ using Autofac.Extensions.DependencyInjection; using ConductorSharp.ApiEnabled.Extensions; +using ConductorSharp.Engine.Health; using ConductorSharp.Engine.Util; using Serilog; @@ -12,6 +13,7 @@ // Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle builder.Services.AddEndpointsApiExplorer(); builder.Services.AddSwaggerGen(); +builder.Services.AddHealthChecks().AddCheck("running"); //Autofac dependency injection builder.Host.UseServiceProviderFactory(new AutofacServiceProviderFactory()).ConfigureApiEnabled(configuration); @@ -26,5 +28,7 @@ } app.UseAuthorization(); + app.MapControllers(); +app.MapHealthChecks("/health"); app.Run(); diff --git a/examples/ConductorSharp.Definitions/Program.cs b/examples/ConductorSharp.Definitions/Program.cs index 206b32bd..02c88e92 100644 --- a/examples/ConductorSharp.Definitions/Program.cs +++ b/examples/ConductorSharp.Definitions/Program.cs @@ -2,6 +2,7 @@ using Autofac.Extensions.DependencyInjection; using ConductorSharp.Definitions; using ConductorSharp.Engine.Extensions; +using ConductorSharp.Engine.Health; using MediatR.Extensions.Autofac.DependencyInjection; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -39,6 +40,7 @@ longPollInterval: configuration.GetValue("Conductor:LongPollInterval"), domain: configuration.GetValue("Conductor:WorkerDomain") ) + .SetHealthCheckService() .AddPipelines(pipelines => { pipelines.AddRequestResponseLogging(); diff --git a/examples/ConductorSharp.Definitions/appsettings.json b/examples/ConductorSharp.Definitions/appsettings.json index fa169067..c3c5ffd3 100644 --- a/examples/ConductorSharp.Definitions/appsettings.json +++ b/examples/ConductorSharp.Definitions/appsettings.json @@ -5,6 +5,6 @@ "LongPollInterval": 100, "MaxConcurrentWorkers": 10, "SleepInterval": 500, - "PreventErrorOnBadRequest": false + "PreventErrorOnBadRequest": true } } diff --git a/examples/ConductorSharp.NoApi/Program.cs b/examples/ConductorSharp.NoApi/Program.cs index 1f712530..07209c68 100644 --- a/examples/ConductorSharp.NoApi/Program.cs +++ b/examples/ConductorSharp.NoApi/Program.cs @@ -1,6 +1,7 @@ using Autofac; using Autofac.Extensions.DependencyInjection; using ConductorSharp.Engine.Extensions; +using ConductorSharp.Engine.Health; using ConductorSharp.NoApi; using MediatR.Extensions.Autofac.DependencyInjection; using Microsoft.Extensions.Configuration; @@ -39,6 +40,7 @@ longPollInterval: configuration.GetValue("Conductor:LongPollInterval"), domain: configuration.GetValue("Conductor:WorkerDomain") ) + .SetHealthCheckService() .AddPipelines(pipelines => { pipelines.AddContextLogging(); diff --git a/examples/ConductorSharp.NoApi/appsettings.json b/examples/ConductorSharp.NoApi/appsettings.json index c3c5ffd3..fa169067 100644 --- a/examples/ConductorSharp.NoApi/appsettings.json +++ b/examples/ConductorSharp.NoApi/appsettings.json @@ -5,6 +5,6 @@ "LongPollInterval": 100, "MaxConcurrentWorkers": 10, "SleepInterval": 500, - "PreventErrorOnBadRequest": true + "PreventErrorOnBadRequest": false } } diff --git a/src/ConductorSharp.Engine/ConductorSharp.Engine.csproj b/src/ConductorSharp.Engine/ConductorSharp.Engine.csproj index bd020c6f..f3116732 100644 --- a/src/ConductorSharp.Engine/ConductorSharp.Engine.csproj +++ b/src/ConductorSharp.Engine/ConductorSharp.Engine.csproj @@ -18,6 +18,7 @@ + diff --git a/src/ConductorSharp.Engine/ExecutionManager.cs b/src/ConductorSharp.Engine/ExecutionManager.cs index 7d93fdde..e8cefe68 100644 --- a/src/ConductorSharp.Engine/ExecutionManager.cs +++ b/src/ConductorSharp.Engine/ExecutionManager.cs @@ -12,11 +12,12 @@ using System.Threading.Tasks; using Autofac; using ConductorSharp.Engine.Util; +using ConductorSharp.Engine.Health; using ConductorSharp.Engine.Polling; namespace ConductorSharp.Engine { - public class ExecutionManager + internal class ExecutionManager { private readonly SemaphoreSlim _semaphore; private readonly WorkerSetConfig _configuration; diff --git a/src/ConductorSharp.Engine/Extensions/ConductorSharpBuilder.cs b/src/ConductorSharp.Engine/Extensions/ConductorSharpBuilder.cs index ec4dd4b9..2bca4b78 100644 --- a/src/ConductorSharp.Engine/Extensions/ConductorSharpBuilder.cs +++ b/src/ConductorSharp.Engine/Extensions/ConductorSharpBuilder.cs @@ -1,5 +1,6 @@ using Autofac; using ConductorSharp.Engine.Behaviors; +using ConductorSharp.Engine.Health; using ConductorSharp.Engine.Interface; using ConductorSharp.Engine.Polling; using ConductorSharp.Engine.Service; @@ -40,6 +41,8 @@ public IExecutionManagerBuilder AddExecutionManager(int maxConcurrentWorkers, in _builder.RegisterType().InstancePerLifetimeScope(); + _builder.RegisterType().As().SingleInstance(); + _builder.RegisterType().As(); _builder.RegisterType().As(); @@ -59,5 +62,11 @@ public void AddRequestResponseLogging() => public void AddValidation() => _builder.RegisterGeneric(typeof(ValidationBehavior<,>)).As(typeof(IPipelineBehavior<,>)); public void AddContextLogging() => _builder.RegisterGeneric(typeof(ContextLoggingBehavior<,>)).As(typeof(IPipelineBehavior<,>)); + + public IExecutionManagerBuilder SetHealthCheckService() where T : IConductorSharpHealthService + { + _builder.RegisterType().As().SingleInstance(); + return this; + } } } diff --git a/src/ConductorSharp.Engine/Extensions/IExecutionManagerBuilder.cs b/src/ConductorSharp.Engine/Extensions/IExecutionManagerBuilder.cs index 0ce51831..f758aff4 100644 --- a/src/ConductorSharp.Engine/Extensions/IExecutionManagerBuilder.cs +++ b/src/ConductorSharp.Engine/Extensions/IExecutionManagerBuilder.cs @@ -1,4 +1,5 @@ -using System; +using ConductorSharp.Engine.Health; +using System; using System.Collections.Generic; using System.Text; @@ -7,5 +8,6 @@ namespace ConductorSharp.Engine.Extensions public interface IExecutionManagerBuilder { IExecutionManagerBuilder AddPipelines(Action pipelines); + IExecutionManagerBuilder SetHealthCheckService() where T : IConductorSharpHealthService; } } diff --git a/src/ConductorSharp.Engine/Extensions/WorkflowEngineBuilder.cs b/src/ConductorSharp.Engine/Extensions/WorkflowEngineBuilder.cs index 3bbb1e99..82b6dd16 100644 --- a/src/ConductorSharp.Engine/Extensions/WorkflowEngineBuilder.cs +++ b/src/ConductorSharp.Engine/Extensions/WorkflowEngineBuilder.cs @@ -1,5 +1,6 @@ using Autofac; using ConductorSharp.Engine.Behaviors; +using ConductorSharp.Engine.Health; using ConductorSharp.Engine.Interface; using ConductorSharp.Engine.Polling; using ConductorSharp.Engine.Service; diff --git a/src/ConductorSharp.Engine/Health/ConductorSharpHealthCheck.cs b/src/ConductorSharp.Engine/Health/ConductorSharpHealthCheck.cs new file mode 100644 index 00000000..462c0b3b --- /dev/null +++ b/src/ConductorSharp.Engine/Health/ConductorSharpHealthCheck.cs @@ -0,0 +1,33 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace ConductorSharp.Engine.Health +{ + public class ConductorSharpHealthCheck : IHealthCheck + { + private readonly IConductorSharpHealthService _healthService; + + public ConductorSharpHealthCheck(IConductorSharpHealthService healthService) + { + _healthService = healthService; + } + + public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + var healthData = await _healthService.GetHealthData(cancellationToken); + + if (healthData.IsExecutionManagerRunning) + { + return new HealthCheckResult(HealthStatus.Healthy, "Deployment has been completed and Execution Manager is running"); + } + else + { + return new HealthCheckResult(context.Registration.FailureStatus, "Execution Manager is not running"); + } + } + } +} diff --git a/src/ConductorSharp.Engine/Health/FileHealthService.cs b/src/ConductorSharp.Engine/Health/FileHealthService.cs new file mode 100644 index 00000000..02e6962c --- /dev/null +++ b/src/ConductorSharp.Engine/Health/FileHealthService.cs @@ -0,0 +1,80 @@ +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace ConductorSharp.Engine.Health +{ + public class HealthData + { + public bool IsExecutionManagerRunning { get; set; } + } + + public class FileHealthService : IConductorSharpHealthService + { + private readonly SemaphoreSlim _semaphore = new(1); + + private const string HealthFileName = "CONDUCTORSHARP_HEALTH.json"; + + public async Task UnsetExecutionManagerRunning(CancellationToken cancellationToken = default) => + await UpdateData(data => data.IsExecutionManagerRunning = false, cancellationToken); + + public async Task SetExecutionManagerRunning(CancellationToken cancellationToken = default) => + await UpdateData(data => data.IsExecutionManagerRunning = true, cancellationToken); + + private async Task UpdateData(Action updateHealthData, CancellationToken cancellationToken = default) + { + var data = await GetHealthData(cancellationToken); + updateHealthData(data); + await WriteHealthData(data, cancellationToken); + } + + public async Task GetHealthData(CancellationToken cancellationToken = default) + { + try + { + await _semaphore.WaitAsync(cancellationToken); + if (!File.Exists(HealthFileName)) + { + return new HealthData(); + } + else + { + return JsonConvert.DeserializeObject(await File.ReadAllTextAsync(HealthFileName, cancellationToken)) + ?? new HealthData(); + } + } + finally + { + _semaphore.Release(); + } + } + + private async Task WriteHealthData(HealthData healthData, CancellationToken cancellationToken = default) + { + try + { + await _semaphore.WaitAsync(cancellationToken); + await File.WriteAllTextAsync(HealthFileName, JsonConvert.SerializeObject(healthData), cancellationToken); + } + finally + { + _semaphore.Release(); + } + } + + public async Task ResetHealthData(CancellationToken cancellationToken = default) => + await WriteHealthData(new HealthData(), cancellationToken); + + public void RemoveHealthData() + { + if (File.Exists(HealthFileName)) + File.Delete(HealthFileName); + + return; + } + } +} diff --git a/src/ConductorSharp.Engine/Health/IConductorSharpHealthService.cs b/src/ConductorSharp.Engine/Health/IConductorSharpHealthService.cs new file mode 100644 index 00000000..bfe22d5e --- /dev/null +++ b/src/ConductorSharp.Engine/Health/IConductorSharpHealthService.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace ConductorSharp.Engine.Health +{ + public interface IConductorSharpHealthService + { + Task GetHealthData(CancellationToken cancellationToken = default); + Task ResetHealthData(CancellationToken cancellationToken = default); + void RemoveHealthData(); + Task SetExecutionManagerRunning(CancellationToken cancellationToken = default); + Task UnsetExecutionManagerRunning(CancellationToken cancellationToken = default); + } +} diff --git a/src/ConductorSharp.Engine/Health/InMemoryHealthService.cs b/src/ConductorSharp.Engine/Health/InMemoryHealthService.cs new file mode 100644 index 00000000..fd485d36 --- /dev/null +++ b/src/ConductorSharp.Engine/Health/InMemoryHealthService.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace ConductorSharp.Engine.Health +{ + public class InMemoryHealthService : IConductorSharpHealthService + { + private static bool _isExecutionManagerRunning; + + public Task GetHealthData(CancellationToken cancellationToken = default) + { + return Task.FromResult(new HealthData { IsExecutionManagerRunning = _isExecutionManagerRunning }); + } + + public void RemoveHealthData() { } + + public Task ResetHealthData(CancellationToken cancellationToken = default) + { + _isExecutionManagerRunning = false; + return Task.CompletedTask; + } + + public Task SetExecutionManagerRunning(CancellationToken cancellationToken = default) + { + _isExecutionManagerRunning = true; + return Task.CompletedTask; + } + + public Task UnsetExecutionManagerRunning(CancellationToken cancellationToken = default) + { + _isExecutionManagerRunning = false; + return Task.CompletedTask; + } + } +} diff --git a/src/ConductorSharp.Engine/Service/DeploymentService.cs b/src/ConductorSharp.Engine/Service/DeploymentService.cs index 6825b37b..59d4f814 100644 --- a/src/ConductorSharp.Engine/Service/DeploymentService.cs +++ b/src/ConductorSharp.Engine/Service/DeploymentService.cs @@ -1,4 +1,5 @@ using ConductorSharp.Client.Service; +using ConductorSharp.Engine.Health; using ConductorSharp.Engine.Interface; using ConductorSharp.Engine.Model; using Microsoft.Extensions.Logging; @@ -6,7 +7,7 @@ namespace ConductorSharp.Engine.Service { - public class DeploymentService : IDeploymentService + internal class DeploymentService : IDeploymentService { private readonly IMetadataService _metadataService; diff --git a/src/ConductorSharp.Engine/Service/WorkflowEngineBackgroundService.cs b/src/ConductorSharp.Engine/Service/WorkflowEngineBackgroundService.cs index c047b3f3..a5d2c30b 100644 --- a/src/ConductorSharp.Engine/Service/WorkflowEngineBackgroundService.cs +++ b/src/ConductorSharp.Engine/Service/WorkflowEngineBackgroundService.cs @@ -1,4 +1,5 @@ -using ConductorSharp.Engine.Interface; +using ConductorSharp.Engine.Health; +using ConductorSharp.Engine.Interface; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using System; @@ -7,17 +8,19 @@ namespace ConductorSharp.Engine.Service { - public class WorkflowEngineBackgroundService : IHostedService, IDisposable + internal class WorkflowEngineBackgroundService : IHostedService, IDisposable { private readonly ILogger _logger; private readonly IHostApplicationLifetime _hostApplicationLifetime; private readonly IDeploymentService _deploymentService; private readonly ExecutionManager _executionManager; private readonly ModuleDeployment _deployment; + private readonly IConductorSharpHealthService _healthService; private Task _executingTask; private readonly CancellationTokenSource _stoppingCts = new(); public WorkflowEngineBackgroundService( + IConductorSharpHealthService healthService, ILogger logger, IHostApplicationLifetime hostApplicationLifetime, IDeploymentService deploymentService, @@ -30,6 +33,7 @@ ModuleDeployment deployment _deploymentService = deploymentService; _executionManager = executionManager; _deployment = deployment; + _healthService = healthService; } public Task StartAsync(CancellationToken cancellationToken) @@ -48,11 +52,14 @@ private async Task RunAsync(CancellationToken cancellationToken) { try { + _healthService.RemoveHealthData(); await _deploymentService.Deploy(_deployment); + await _healthService.SetExecutionManagerRunning(cancellationToken); await _executionManager.StartAsync(cancellationToken); } catch (Exception exception) { + await _healthService.UnsetExecutionManagerRunning(); _logger.LogCritical(exception, "Workflow Engine Background Service encountered an error"); throw; } @@ -64,6 +71,7 @@ private async Task RunAsync(CancellationToken cancellationToken) public async Task StopAsync(CancellationToken cancellationToken) { + _healthService.RemoveHealthData(); if (_executingTask == null) { return;