370 lines
14 KiB
C#
370 lines
14 KiB
C#
using Microsoft.AspNetCore.SignalR;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Logging;
|
|
using RobotNet10.NavigationTune.Hubs;
|
|
using RobotNet10.NavigationTune.Interfaces;
|
|
using RobotNet10.NavigationTune.Shared.Hubs;
|
|
using RobotNet10.NavigationTune.Shared.Interfaces;
|
|
using RobotNet10.NavigationTune.Shared.Models;
|
|
|
|
namespace RobotNet10.NavigationTune.Services;
|
|
|
|
/// <summary>
|
|
/// Tuning orchestrator implementation
|
|
/// </summary>
|
|
public class TuningOrchestrator(
|
|
ITuningNavigation tuningNavigation,
|
|
IMetricsCalculator metricsCalculator,
|
|
ITestRepository testRepository,
|
|
IParameterManager parameterManager,
|
|
IServiceScopeFactory scopeFactory,
|
|
IRunningTestCancellationRegistry cancellationRegistry,
|
|
IHubContext<TuningHub>? hubContext = null,
|
|
ILogger<TuningOrchestrator>? logger = null) : ITuningOrchestrator
|
|
{
|
|
private readonly ITuningNavigation _tuningNavigation = tuningNavigation;
|
|
private readonly IMetricsCalculator _metricsCalculator = metricsCalculator;
|
|
private readonly ITestRepository _testRepository = testRepository;
|
|
private readonly IParameterManager _parameterManager = parameterManager;
|
|
private readonly IServiceScopeFactory _scopeFactory = scopeFactory;
|
|
private readonly IRunningTestCancellationRegistry _cancellationRegistry = cancellationRegistry;
|
|
private readonly IHubContext<TuningHub>? _hubContext = hubContext;
|
|
private readonly ILogger<TuningOrchestrator>? _logger = logger;
|
|
private volatile bool _isTestRunning;
|
|
|
|
/// <summary>
|
|
/// Start test and return immediately with testRunId and status Running.
|
|
/// Control loop runs on WatchThread; completion is sent via SignalR (ReceiveTestResult).
|
|
/// </summary>
|
|
public async Task<TestExecutionResult> StartTestAsync(
|
|
TestScenario scenario,
|
|
NavigationParameterSet parameters,
|
|
string? connectionId = null,
|
|
Guid? testRunId = null,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
if (_isTestRunning)
|
|
throw new InvalidOperationException("A test is already running. Stop it before starting another.");
|
|
|
|
var validation = _parameterManager.Validate(parameters);
|
|
if (!validation.IsValid)
|
|
throw new InvalidOperationException($"Invalid parameters: {string.Join(", ", validation.Errors)}");
|
|
|
|
var testRun = new TestRun
|
|
{
|
|
Id = testRunId ?? Guid.NewGuid(),
|
|
ScenarioId = scenario.Id,
|
|
ParameterSetId = parameters.Id,
|
|
StartTime = DateTime.UtcNow,
|
|
Status = TestStatus.Preparing
|
|
};
|
|
|
|
if (connectionId != null && _hubContext != null)
|
|
await _hubContext.Groups.AddToGroupAsync(connectionId, $"test_{testRun.Id}", cancellationToken);
|
|
|
|
await _testRepository.SaveAsync(testRun);
|
|
|
|
// Notify UI immediately so Stop/Pause buttons become active
|
|
if (_hubContext != null)
|
|
{
|
|
await _hubContext.Clients.Group($"test_{testRun.Id}")
|
|
.SendAsync("ReceiveTestStatus", new TestStatusUpdateDto
|
|
{
|
|
TestRunId = testRun.Id,
|
|
Status = TestStatus.Running,
|
|
ProgressPercent = 0,
|
|
Message = "Running"
|
|
}, cancellationToken: cancellationToken);
|
|
}
|
|
|
|
_isTestRunning = true;
|
|
|
|
// Register CTS so Stop/EMC Stop (different HTTP request) can cancel this test.
|
|
// Do NOT use "using var cts" - the CTS must stay alive until the test completes (onComplete calls Unregister which disposes it).
|
|
var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
|
_cancellationRegistry.Register(testRun.Id, cts);
|
|
|
|
// Execute on WatchThread; returns immediately; onComplete runs when test finishes
|
|
var runningResult = await _tuningNavigation.ExecuteTestAsync(
|
|
scenario,
|
|
parameters,
|
|
testRun.Id,
|
|
cts.Token,
|
|
onComplete: result => _ = SaveResultAndNotifyAsync(testRun.Id, testRun.StartTime, result, parameters));
|
|
|
|
return runningResult;
|
|
}
|
|
|
|
private async Task SaveResultAndNotifyAsync(Guid testRunId, DateTime startTime, TestExecutionResult result, NavigationParameterSet? parameters = null)
|
|
{
|
|
// Use new scope: completion runs on WatchThread after HTTP request may have ended
|
|
using var scope = _scopeFactory.CreateScope();
|
|
var testRepository = scope.ServiceProvider.GetRequiredService<ITestRepository>();
|
|
try
|
|
{
|
|
await testRepository.UpdateFromResultAsync(
|
|
testRunId,
|
|
result.Status,
|
|
result.EndTime,
|
|
result.Duration,
|
|
result.ErrorMessage,
|
|
result.Metrics,
|
|
result.SafetyViolations);
|
|
|
|
// Generate tuning suggestions if test completed with enough telemetry
|
|
if (result.Status == TestStatus.Completed &&
|
|
result.TelemetryData?.Count > 10 &&
|
|
result.Metrics != null &&
|
|
parameters != null)
|
|
{
|
|
try
|
|
{
|
|
var tuningAdvisor = scope.ServiceProvider.GetRequiredService<ITuningAdvisor>();
|
|
result.TuningReport = tuningAdvisor.Analyze(
|
|
result.TelemetryData,
|
|
result.Metrics,
|
|
parameters);
|
|
result.TuningReport.TestRunId = testRunId;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger?.LogWarning(ex, "Tuning advisor analysis failed for test {TestRunId}", testRunId);
|
|
}
|
|
}
|
|
|
|
if (_hubContext != null)
|
|
{
|
|
await _hubContext.Clients.Group($"test_{testRunId}")
|
|
.SendAsync("ReceiveTestResult", result);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger?.LogError(ex, "Error saving test result");
|
|
if (_hubContext != null)
|
|
{
|
|
var errorResult = new TestExecutionResult
|
|
{
|
|
TestRunId = testRunId,
|
|
Status = TestStatus.Error,
|
|
ErrorMessage = ex.Message,
|
|
StartTime = startTime,
|
|
EndTime = DateTime.UtcNow
|
|
};
|
|
await _hubContext.Clients.Group($"test_{testRunId}")
|
|
.SendAsync("ReceiveTestResult", errorResult);
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
_cancellationRegistry.Unregister(testRunId);
|
|
_isTestRunning = false;
|
|
}
|
|
}
|
|
|
|
public async Task<TestExecutionResult> RunSingleTestAsync(
|
|
TestScenario scenario,
|
|
NavigationParameterSet parameters,
|
|
string? connectionId = null,
|
|
Guid? testRunId = null,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
// Validate parameters
|
|
var validation = _parameterManager.Validate(parameters);
|
|
if (!validation.IsValid)
|
|
{
|
|
throw new InvalidOperationException($"Invalid parameters: {string.Join(", ", validation.Errors)}");
|
|
}
|
|
|
|
// Create test run record (use provided testRunId so client can join group before execute and receive real-time telemetry)
|
|
var testRun = new TestRun
|
|
{
|
|
Id = testRunId ?? Guid.NewGuid(),
|
|
ScenarioId = scenario.Id,
|
|
ParameterSetId = parameters.Id,
|
|
StartTime = DateTime.UtcNow,
|
|
Status = TestStatus.Preparing
|
|
};
|
|
|
|
// Join SignalR group if connectionId provided
|
|
if (connectionId != null && _hubContext != null)
|
|
{
|
|
await _hubContext.Groups.AddToGroupAsync(connectionId, $"test_{testRun.Id}", cancellationToken);
|
|
}
|
|
|
|
try
|
|
{
|
|
// Execute test (pass testRun.Id so real-time telemetry is sent to group test_{testRun.Id})
|
|
var result = await _tuningNavigation.ExecuteTestAsync(scenario, parameters, testRun.Id, cancellationToken);
|
|
|
|
// Update test run
|
|
testRun.Status = result.Status;
|
|
testRun.EndTime = result.EndTime;
|
|
testRun.Duration = result.Duration;
|
|
testRun.ErrorMessage = result.ErrorMessage;
|
|
testRun.SafetyViolations = result.SafetyViolations;
|
|
testRun.Metrics = result.Metrics;
|
|
|
|
// Save to database
|
|
await _testRepository.SaveAsync(testRun);
|
|
|
|
// Generate tuning suggestions
|
|
if (result.Status == TestStatus.Completed &&
|
|
result.TelemetryData?.Count > 10 &&
|
|
result.Metrics != null)
|
|
{
|
|
try
|
|
{
|
|
using var scope = _scopeFactory.CreateScope();
|
|
var tuningAdvisor = scope.ServiceProvider.GetRequiredService<ITuningAdvisor>();
|
|
result.TuningReport = tuningAdvisor.Analyze(
|
|
result.TelemetryData,
|
|
result.Metrics,
|
|
parameters);
|
|
result.TuningReport.TestRunId = testRun.Id;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger?.LogWarning(ex, "Tuning advisor analysis failed for test {TestRunId}", testRun.Id);
|
|
}
|
|
}
|
|
|
|
// Publish completion
|
|
if (_hubContext != null)
|
|
{
|
|
await _hubContext.Clients.Group($"test_{testRun.Id}")
|
|
.SendAsync("ReceiveTestResult", result, cancellationToken: cancellationToken);
|
|
}
|
|
|
|
return result;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger?.LogError(ex, "Error executing test");
|
|
testRun.Status = TestStatus.Error;
|
|
testRun.ErrorMessage = ex.Message;
|
|
testRun.EndTime = DateTime.UtcNow;
|
|
await _testRepository.SaveAsync(testRun);
|
|
throw;
|
|
}
|
|
}
|
|
|
|
public async Task<BatchTestResult> RunBatchTestsAsync(
|
|
List<TestScenario> scenarios,
|
|
NavigationParameterSet parameters,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var results = new List<TestExecutionResult>();
|
|
var batchId = Guid.NewGuid();
|
|
|
|
_logger?.LogInformation("Starting batch test with {Count} scenarios", scenarios.Count);
|
|
|
|
for (int i = 0; i < scenarios.Count; i++)
|
|
{
|
|
if (cancellationToken.IsCancellationRequested)
|
|
{
|
|
_logger?.LogWarning("Batch test cancelled at scenario {Index}", i);
|
|
break;
|
|
}
|
|
|
|
var scenario = scenarios[i];
|
|
|
|
try
|
|
{
|
|
var result = await RunSingleTestAsync(scenario, parameters, cancellationToken: cancellationToken);
|
|
results.Add(result);
|
|
|
|
_logger?.LogInformation(
|
|
"Completed scenario {Index}/{Total}: {Name}",
|
|
i + 1,
|
|
scenarios.Count,
|
|
scenario.Name
|
|
);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger?.LogError(
|
|
ex,
|
|
"Failed scenario {Index}/{Total}: {Name}",
|
|
i + 1,
|
|
scenarios.Count,
|
|
scenario.Name
|
|
);
|
|
|
|
// Continue with remaining scenarios
|
|
}
|
|
}
|
|
|
|
var batchResult = new BatchTestResult
|
|
{
|
|
BatchId = batchId,
|
|
Parameters = parameters,
|
|
Results = results,
|
|
SuccessCount = results.Count(r => r.Status == TestStatus.Completed),
|
|
FailureCount = results.Count(r => r.Status != TestStatus.Completed),
|
|
AverageScore = results.Where(r => r.Metrics != null).Average(r => r.Metrics!.OverallScore)
|
|
};
|
|
|
|
return batchResult;
|
|
}
|
|
|
|
public async Task<ComparisonResult> CompareConfigurationsAsync(
|
|
List<NavigationParameterSet> parameterSets,
|
|
TestScenario scenario,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var results = new Dictionary<string, TestExecutionResult>();
|
|
|
|
foreach (var parameters in parameterSets)
|
|
{
|
|
try
|
|
{
|
|
var result = await RunSingleTestAsync(scenario, parameters, cancellationToken: cancellationToken);
|
|
results[parameters.Name] = result;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger?.LogError(ex, "Failed to test configuration {Name}", parameters.Name);
|
|
}
|
|
}
|
|
|
|
var comparison = new ComparisonResult
|
|
{
|
|
Scenario = scenario,
|
|
Configurations = parameterSets,
|
|
Results = results,
|
|
BestConfiguration = results
|
|
.Where(r => r.Value.Metrics != null)
|
|
.OrderByDescending(r => r.Value.Metrics!.OverallScore)
|
|
.FirstOrDefault()
|
|
.Key ?? string.Empty
|
|
};
|
|
|
|
return comparison;
|
|
}
|
|
|
|
public void PauseTest(string testRunId)
|
|
{
|
|
_tuningNavigation.Pause();
|
|
}
|
|
|
|
public void ResumeTest(string testRunId)
|
|
{
|
|
_tuningNavigation.Resume();
|
|
}
|
|
|
|
public void StopTest(string testRunId)
|
|
{
|
|
if (Guid.TryParse(testRunId, out var id) && _cancellationRegistry.TryCancel(id))
|
|
return;
|
|
_tuningNavigation.Stop();
|
|
}
|
|
|
|
public void EmergencyStop(string testRunId)
|
|
{
|
|
if (Guid.TryParse(testRunId, out var id) && _cancellationRegistry.TryCancel(id))
|
|
return;
|
|
_tuningNavigation.EmergencyStop();
|
|
}
|
|
}
|