using Microsoft.AspNetCore.SignalR; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using RobotNet10.NavigationTune.Hubs; using RobotNet10.NavigationTune.Interfaces; using RobotNet10.NavigationTune.Shared.Hubs; using RobotNet10.NavigationTune.Shared.Interfaces; using RobotNet10.NavigationTune.Shared.Models; namespace RobotNet10.NavigationTune.Services; /// /// Tuning orchestrator implementation /// public class TuningOrchestrator( ITuningNavigation tuningNavigation, IMetricsCalculator metricsCalculator, ITestRepository testRepository, IParameterManager parameterManager, IServiceScopeFactory scopeFactory, IRunningTestCancellationRegistry cancellationRegistry, IHubContext? hubContext = null, ILogger? logger = null) : ITuningOrchestrator { private readonly ITuningNavigation _tuningNavigation = tuningNavigation; private readonly IMetricsCalculator _metricsCalculator = metricsCalculator; private readonly ITestRepository _testRepository = testRepository; private readonly IParameterManager _parameterManager = parameterManager; private readonly IServiceScopeFactory _scopeFactory = scopeFactory; private readonly IRunningTestCancellationRegistry _cancellationRegistry = cancellationRegistry; private readonly IHubContext? _hubContext = hubContext; private readonly ILogger? _logger = logger; private volatile bool _isTestRunning; /// /// Start test and return immediately with testRunId and status Running. /// Control loop runs on WatchThread; completion is sent via SignalR (ReceiveTestResult). /// public async Task StartTestAsync( TestScenario scenario, NavigationParameterSet parameters, string? connectionId = null, Guid? testRunId = null, CancellationToken cancellationToken = default) { if (_isTestRunning) throw new InvalidOperationException("A test is already running. Stop it before starting another."); var validation = _parameterManager.Validate(parameters); if (!validation.IsValid) throw new InvalidOperationException($"Invalid parameters: {string.Join(", ", validation.Errors)}"); var testRun = new TestRun { Id = testRunId ?? Guid.NewGuid(), ScenarioId = scenario.Id, ParameterSetId = parameters.Id, StartTime = DateTime.UtcNow, Status = TestStatus.Preparing }; if (connectionId != null && _hubContext != null) await _hubContext.Groups.AddToGroupAsync(connectionId, $"test_{testRun.Id}", cancellationToken); await _testRepository.SaveAsync(testRun); // Notify UI immediately so Stop/Pause buttons become active if (_hubContext != null) { await _hubContext.Clients.Group($"test_{testRun.Id}") .SendAsync("ReceiveTestStatus", new TestStatusUpdateDto { TestRunId = testRun.Id, Status = TestStatus.Running, ProgressPercent = 0, Message = "Running" }, cancellationToken: cancellationToken); } _isTestRunning = true; // Register CTS so Stop/EMC Stop (different HTTP request) can cancel this test. // Do NOT use "using var cts" - the CTS must stay alive until the test completes (onComplete calls Unregister which disposes it). var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); _cancellationRegistry.Register(testRun.Id, cts); // Execute on WatchThread; returns immediately; onComplete runs when test finishes var runningResult = await _tuningNavigation.ExecuteTestAsync( scenario, parameters, testRun.Id, cts.Token, onComplete: result => _ = SaveResultAndNotifyAsync(testRun.Id, testRun.StartTime, result, parameters)); return runningResult; } private async Task SaveResultAndNotifyAsync(Guid testRunId, DateTime startTime, TestExecutionResult result, NavigationParameterSet? parameters = null) { // Use new scope: completion runs on WatchThread after HTTP request may have ended using var scope = _scopeFactory.CreateScope(); var testRepository = scope.ServiceProvider.GetRequiredService(); try { await testRepository.UpdateFromResultAsync( testRunId, result.Status, result.EndTime, result.Duration, result.ErrorMessage, result.Metrics, result.SafetyViolations); // Generate tuning suggestions if test completed with enough telemetry if (result.Status == TestStatus.Completed && result.TelemetryData?.Count > 10 && result.Metrics != null && parameters != null) { try { var tuningAdvisor = scope.ServiceProvider.GetRequiredService(); result.TuningReport = tuningAdvisor.Analyze( result.TelemetryData, result.Metrics, parameters); result.TuningReport.TestRunId = testRunId; } catch (Exception ex) { _logger?.LogWarning(ex, "Tuning advisor analysis failed for test {TestRunId}", testRunId); } } if (_hubContext != null) { await _hubContext.Clients.Group($"test_{testRunId}") .SendAsync("ReceiveTestResult", result); } } catch (Exception ex) { _logger?.LogError(ex, "Error saving test result"); if (_hubContext != null) { var errorResult = new TestExecutionResult { TestRunId = testRunId, Status = TestStatus.Error, ErrorMessage = ex.Message, StartTime = startTime, EndTime = DateTime.UtcNow }; await _hubContext.Clients.Group($"test_{testRunId}") .SendAsync("ReceiveTestResult", errorResult); } } finally { _cancellationRegistry.Unregister(testRunId); _isTestRunning = false; } } public async Task RunSingleTestAsync( TestScenario scenario, NavigationParameterSet parameters, string? connectionId = null, Guid? testRunId = null, CancellationToken cancellationToken = default) { // Validate parameters var validation = _parameterManager.Validate(parameters); if (!validation.IsValid) { throw new InvalidOperationException($"Invalid parameters: {string.Join(", ", validation.Errors)}"); } // Create test run record (use provided testRunId so client can join group before execute and receive real-time telemetry) var testRun = new TestRun { Id = testRunId ?? Guid.NewGuid(), ScenarioId = scenario.Id, ParameterSetId = parameters.Id, StartTime = DateTime.UtcNow, Status = TestStatus.Preparing }; // Join SignalR group if connectionId provided if (connectionId != null && _hubContext != null) { await _hubContext.Groups.AddToGroupAsync(connectionId, $"test_{testRun.Id}", cancellationToken); } try { // Execute test (pass testRun.Id so real-time telemetry is sent to group test_{testRun.Id}) var result = await _tuningNavigation.ExecuteTestAsync(scenario, parameters, testRun.Id, cancellationToken); // Update test run testRun.Status = result.Status; testRun.EndTime = result.EndTime; testRun.Duration = result.Duration; testRun.ErrorMessage = result.ErrorMessage; testRun.SafetyViolations = result.SafetyViolations; testRun.Metrics = result.Metrics; // Save to database await _testRepository.SaveAsync(testRun); // Generate tuning suggestions if (result.Status == TestStatus.Completed && result.TelemetryData?.Count > 10 && result.Metrics != null) { try { using var scope = _scopeFactory.CreateScope(); var tuningAdvisor = scope.ServiceProvider.GetRequiredService(); result.TuningReport = tuningAdvisor.Analyze( result.TelemetryData, result.Metrics, parameters); result.TuningReport.TestRunId = testRun.Id; } catch (Exception ex) { _logger?.LogWarning(ex, "Tuning advisor analysis failed for test {TestRunId}", testRun.Id); } } // Publish completion if (_hubContext != null) { await _hubContext.Clients.Group($"test_{testRun.Id}") .SendAsync("ReceiveTestResult", result, cancellationToken: cancellationToken); } return result; } catch (Exception ex) { _logger?.LogError(ex, "Error executing test"); testRun.Status = TestStatus.Error; testRun.ErrorMessage = ex.Message; testRun.EndTime = DateTime.UtcNow; await _testRepository.SaveAsync(testRun); throw; } } public async Task RunBatchTestsAsync( List scenarios, NavigationParameterSet parameters, CancellationToken cancellationToken = default) { var results = new List(); var batchId = Guid.NewGuid(); _logger?.LogInformation("Starting batch test with {Count} scenarios", scenarios.Count); for (int i = 0; i < scenarios.Count; i++) { if (cancellationToken.IsCancellationRequested) { _logger?.LogWarning("Batch test cancelled at scenario {Index}", i); break; } var scenario = scenarios[i]; try { var result = await RunSingleTestAsync(scenario, parameters, cancellationToken: cancellationToken); results.Add(result); _logger?.LogInformation( "Completed scenario {Index}/{Total}: {Name}", i + 1, scenarios.Count, scenario.Name ); } catch (Exception ex) { _logger?.LogError( ex, "Failed scenario {Index}/{Total}: {Name}", i + 1, scenarios.Count, scenario.Name ); // Continue with remaining scenarios } } var batchResult = new BatchTestResult { BatchId = batchId, Parameters = parameters, Results = results, SuccessCount = results.Count(r => r.Status == TestStatus.Completed), FailureCount = results.Count(r => r.Status != TestStatus.Completed), AverageScore = results.Where(r => r.Metrics != null).Average(r => r.Metrics!.OverallScore) }; return batchResult; } public async Task CompareConfigurationsAsync( List parameterSets, TestScenario scenario, CancellationToken cancellationToken = default) { var results = new Dictionary(); foreach (var parameters in parameterSets) { try { var result = await RunSingleTestAsync(scenario, parameters, cancellationToken: cancellationToken); results[parameters.Name] = result; } catch (Exception ex) { _logger?.LogError(ex, "Failed to test configuration {Name}", parameters.Name); } } var comparison = new ComparisonResult { Scenario = scenario, Configurations = parameterSets, Results = results, BestConfiguration = results .Where(r => r.Value.Metrics != null) .OrderByDescending(r => r.Value.Metrics!.OverallScore) .FirstOrDefault() .Key ?? string.Empty }; return comparison; } public void PauseTest(string testRunId) { _tuningNavigation.Pause(); } public void ResumeTest(string testRunId) { _tuningNavigation.Resume(); } public void StopTest(string testRunId) { if (Guid.TryParse(testRunId, out var id) && _cancellationRegistry.TryCancel(id)) return; _tuningNavigation.Stop(); } public void EmergencyStop(string testRunId) { if (Guid.TryParse(testRunId, out var id) && _cancellationRegistry.TryCancel(id)) return; _tuningNavigation.EmergencyStop(); } }