using Appccelerate.StateMachine; using Appccelerate.StateMachine.Machine; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using RobotNet10.Script; using RobotNet10.ScriptEngine.Helpers; using RobotNet10.ScriptEngine.HubContexts; using RobotNet10.ScriptEngine.Data; using RobotNet10.ScriptEngine.Models; using RobotNet10.ScriptEngine.Shared; using RobotNet10.Shared; using System.Collections.Concurrent; using System.Diagnostics; using System.Text.Json; namespace RobotNet10.ScriptEngine; /// /// State enum for MissionManager. /// public enum MissionManagerState { Idle = 0, Running, Stopping, } /// /// Triggers for MissionManager state machine. /// public enum MissionManagerTrigger { Start, Stop, StoppingCompleted, } /// /// Manages script missions with state machine support. /// public class MissionManager : IDisposable { private readonly PassiveStateMachine _stateMachine; private readonly ConcurrentDictionary _missionModels = new(); private readonly ConcurrentQueue _idleMissions = new(); private readonly ConcurrentQueue _runningMissions = new(); private readonly ConcurrentDictionary _allMissions = new(); private readonly Lock _lockObject = new(); private readonly Lock _stateLockObject = new(); private readonly Lock _queueLockObject = new(); private readonly IServiceScopeFactory _scopeFactory; private readonly VariableManager _variableManager; private readonly IScriptEngineResource _scriptResource; private readonly ILogger _logger; private readonly ConsoleHubContext _consoleHubContext; private readonly IConfiguration _configuration; private Task? _runningHandlerTask; private CancellationTokenSource? _runningHandlerCts; private readonly ManualResetEventSlim _stoppedWaitHandle = new(false); private bool _disposed; private MissionManagerState _currentState = MissionManagerState.Idle; /// /// Gets the current state of the MissionManager. /// public MissionManagerState State => _currentState; /// /// Gets all mission models. /// public IReadOnlyDictionary MissionModels => _missionModels; /// /// Initializes a new instance of MissionManager. /// public MissionManager( IServiceScopeFactory scopeFactory, VariableManager variableManager, IScriptEngineResource scriptResource, ILogger logger, ConsoleHubContext consoleHubContext, IConfiguration configuration) { _scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory)); _variableManager = variableManager ?? throw new ArgumentNullException(nameof(variableManager)); _scriptResource = scriptResource ?? throw new ArgumentNullException(nameof(scriptResource)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _consoleHubContext = consoleHubContext ?? throw new ArgumentNullException(nameof(consoleHubContext)); _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); var builder = new StateMachineDefinitionBuilder(); // Idle state - can add/remove mission models builder.In(MissionManagerState.Idle) .ExecuteOnEntry(() => { lock (_stateLockObject) { _currentState = MissionManagerState.Idle; } }) .On(MissionManagerTrigger.Start) .Goto(MissionManagerState.Running) .Execute(() => { lock (_stateLockObject) { _currentState = MissionManagerState.Running; } OnEnterRunning(); }); // Running state - can create missions builder.In(MissionManagerState.Running) .ExecuteOnEntry(() => { lock (_stateLockObject) { _currentState = MissionManagerState.Running; } }) .On(MissionManagerTrigger.Stop) .Goto(MissionManagerState.Stopping) .Execute(() => { lock (_stateLockObject) { _currentState = MissionManagerState.Stopping; } OnEnterStopping(); }); // Stopping state - waiting for all missions to complete builder.In(MissionManagerState.Stopping) .ExecuteOnEntry(() => { lock (_stateLockObject) { _currentState = MissionManagerState.Stopping; } }) .On(MissionManagerTrigger.StoppingCompleted) .Goto(MissionManagerState.Idle) .Execute(() => { lock (_stateLockObject) { _currentState = MissionManagerState.Idle; } }); _stateMachine = builder .WithInitialState(MissionManagerState.Idle) .Build() .CreatePassiveStateMachine(); _stateMachine.Start(); } private void OnEnterRunning() { // Start running handler thread _runningHandlerCts = new CancellationTokenSource(); _stoppedWaitHandle.Reset(); // Use standard thread pool _runningHandlerTask = Task.Run(() => RunningHandlerAsync(_runningHandlerCts.Token)); // Create missions for models with AutoStart == true lock (_lockObject) { foreach (var model in _missionModels.Values) { if (model.AutoStart) { try { // Validate auto-start mission parameters: must be empty or have exactly one CancellationToken parameter var paramList = model.Parameters.ToList(); if (paramList.Count > 1) { var message = $"Mission '{model.Name}' has AutoStart=true but has {paramList.Count} parameters. Auto-start missions must have 0 or 1 parameter (CancellationToken). Skipping auto-start."; _logger.LogWarning(message); _consoleHubContext.LogWarning(message); continue; } if (paramList.Count == 1) { var param = paramList[0]; if (param.Type != typeof(CancellationToken)) { var message = $"Mission '{model.Name}' has AutoStart=true but parameter '{param.Name}' is not CancellationToken. Auto-start missions must have 0 or 1 CancellationToken parameter. Skipping auto-start."; _logger.LogWarning(message); _consoleHubContext.LogWarning(message); continue; } } // Generate mission ID first var missionId = Guid.NewGuid(); // Create mission instance with default parameters var parameters = model.Parameters.Select(p => { object? value = p.Type == typeof(CancellationToken) ? CancellationToken.None : p.DefaultValue; return new ScriptMissionParameterModel(p.Name, p.Type, value); }); var result = CreateMissionInternal(model.Name, parameters, missionId); if (!result.IsSuccess) { // Log error but continue creating other missions _logger.LogError($"Failed to auto-start mission '{model.Name}': {result.Message}"); _consoleHubContext.LogError($"Failed to auto-start mission '{model.Name}': {result.Message}"); } } catch (Exception ex) { // Log error but continue creating other missions _logger.LogError(ex, $"Failed to auto-start mission '{model.Name}'"); _consoleHubContext.LogError($"Failed to auto-start mission '{model.Name}': {ex.Message}"); } } } } } private void OnEnterStopping() { // Fire and forget async operation with proper error handling _ = Task.Run(async () => { try { // Stop running handler _runningHandlerCts?.Cancel(); // Wait for running handler to complete (with timeout) if (_runningHandlerTask != null) { try { await _runningHandlerTask.WaitAsync(TimeSpan.FromSeconds(10)); } catch (TimeoutException) { _logger.LogWarning("MissionManager running handler task did not complete within timeout. Proceeding anyway."); _consoleHubContext.LogWarning("MissionManager running handler task did not complete within timeout. Proceeding anyway."); } catch (Exception ex) { _logger.LogError(ex, "Error waiting for running handler task to complete"); } } // Wait for stopped signal (with timeout) // If timeout, check if missions are actually executing if (!_stoppedWaitHandle.Wait(TimeSpan.FromSeconds(30))) { // Check if there are actually running missions var runningMissions = GetAllMissions().Where(m => m.IsExecuting).ToList(); if (runningMissions.Count > 0) { _logger.LogWarning($"Timeout waiting for {runningMissions.Count} mission(s) to stop. ScriptRunner may be blocking. Proceeding anyway."); _consoleHubContext.LogWarning($"Timeout waiting for {runningMissions.Count} mission(s) to stop. ScriptRunner may be blocking. Proceeding anyway."); } else { _logger.LogInformation("All missions stopped. Proceeding with state transition."); _consoleHubContext.LogInfo("All missions stopped. Proceeding with state transition."); } } // Fire StoppingCompleted trigger (always proceed) _stateMachine.Fire(MissionManagerTrigger.StoppingCompleted); } catch (Exception ex) { _logger.LogError(ex, "Unexpected error during MissionManager stopping"); _consoleHubContext.LogError($"MissionManager stopping error: {ex.Message}"); // Ensure WaitHandle is set even on error to prevent deadlock try { _stoppedWaitHandle.Set(); } catch { /* Ignore */ } // Fire trigger to proceed _stateMachine.Fire(MissionManagerTrigger.StoppingCompleted); } }); } private async Task RunningHandlerAsync(CancellationToken cancellationToken) { var stopwatch = Stopwatch.StartNew(); int elapsed; int remaining; int interval = 1000; int processTime = (int)(interval * 0.8); int count; _stoppedWaitHandle.Reset(); while (!cancellationToken.IsCancellationRequested) { stopwatch.Restart(); using (var scope = _scopeFactory.CreateScope()) { var dbContext = scope.ServiceProvider.GetRequiredService(); // Process idle queue: transition to running or complete immediately count = _idleMissions.Count; for (int i = 0; i < count; i++) { if (!_idleMissions.TryDequeue(out var mission)) break; // Find corresponding database record var dbMission = await dbContext.InstanceMissions.FindAsync([mission.Id], cancellationToken); if (dbMission == null) { mission.Dispose(); _allMissions.TryRemove(mission.Id, out _); continue; // Skip if mission not found in database } if (mission.State == ScriptMissionState.Idle) { // Start Mission and move to running queue mission.Start(); _runningMissions.Enqueue(mission); dbMission.State = mission.State; } else if (mission.State == ScriptMissionState.Completed || mission.State == ScriptMissionState.Canceled || mission.State == ScriptMissionState.Error) { // Mission completed/canceled/errored before running dbMission.State = mission.State; dbMission.Log += $"{Environment.NewLine}{mission.GetLog()}"; dbMission.Score = mission.CurrentScore; dbMission.StoppedAt = DateTime.UtcNow; mission.Dispose(); _allMissions.TryRemove(mission.Id, out _); } else { // Invalid state dbMission.Log += $"{Environment.NewLine}{mission.GetLog()}{Environment.NewLine}{DateTime.UtcNow:O}: Mission is not in idle state. [{mission.State}]"; dbMission.State = ScriptMissionState.Error; mission.Dispose(); _allMissions.TryRemove(mission.Id, out _); } } // Process running queue: check completion or keep count = _runningMissions.Count; for (int i = 0; i < count; i++) { if (!_runningMissions.TryDequeue(out var mission)) break; // Find corresponding database record var dbMission = await dbContext.InstanceMissions.FindAsync([mission.Id], cancellationToken); if (dbMission == null) { mission.Dispose(); _allMissions.TryRemove(mission.Id, out _); continue; // Skip if mission not found in database } if (mission.State == ScriptMissionState.Completed || mission.State == ScriptMissionState.Canceled || mission.State == ScriptMissionState.Error) { // Mission completed - save results to database dbMission.State = mission.State; dbMission.Log += $"{Environment.NewLine}{mission.GetLog()}"; dbMission.Score = mission.CurrentScore; dbMission.StoppedAt = DateTime.UtcNow; mission.Dispose(); _allMissions.TryRemove(mission.Id, out _); } else { // Mission still running - update log and state, keep in queue dbMission.State = mission.State; var newLog = mission.GetLog(); if (!string.IsNullOrEmpty(newLog)) { dbMission.Log += $"{Environment.NewLine}{newLog}"; } dbMission.Score = mission.CurrentScore; _runningMissions.Enqueue(mission); } } // Save all changes to database await dbContext.SaveChangesAsync(cancellationToken); } // Collect memory after each cycle GC.Collect(); stopwatch.Stop(); elapsed = (int)stopwatch.ElapsedMilliseconds; remaining = interval - elapsed; // If execution time exceeds 80% of interval, add another cycle if (elapsed > processTime) { remaining += interval; } if (remaining > 0) { try { await Task.Delay(remaining, cancellationToken); } catch (OperationCanceledException) { break; } } } // Cleanup on stop: dispose all unstarted Missions while (_idleMissions.TryDequeue(out var mission)) { mission.Dispose(); _allMissions.TryRemove(mission.Id, out _); } // Cancel and save state of all running Missions using (var scope = _scopeFactory.CreateScope()) { var dbContext = scope.ServiceProvider.GetRequiredService(); while (_runningMissions.TryDequeue(out var mission)) { try { mission.Cancel("engin is stopping"); // Wait for stop, but only timeout if actually executing mission.WaitForStop(); } catch (Exception ex) { // Log error but continue disposing _logger.LogError(ex, $"Error stopping mission '{mission.Name}'"); } finally { // Update final state to database try { var dbMission = await dbContext.InstanceMissions.FindAsync([mission.Id], CancellationToken.None); if (dbMission != null) { dbMission.State = mission.State; dbMission.Log += $"{Environment.NewLine}{mission.GetLog()}"; dbMission.Score = mission.CurrentScore; dbMission.StoppedAt = DateTime.UtcNow; } } catch (Exception ex) { // Log error but continue _logger.LogError(ex, $"Error updating mission '{mission.Name}' state in database"); } mission.Dispose(); _allMissions.TryRemove(mission.Id, out _); } } await dbContext.SaveChangesAsync(CancellationToken.None); } // Signal that stopping is complete _stoppedWaitHandle.Set(); } /// /// Resets all mission models. Clears all mission models. /// Only allowed when state is Idle. /// public MessageResult Reset() { try { lock (_stateLockObject) { if (_currentState != MissionManagerState.Idle) return new MessageResult(false, $"Cannot reset mission models when MissionManager is in state: {_currentState}"); } lock (_lockObject) { var count = _missionModels.Count; _missionModels.Clear(); return new MessageResult(true, $"Reset {count} mission model(s) successfully"); } } catch (Exception ex) { return new MessageResult(false, $"Failed to reset mission models: {ex.Message}"); } } /// /// Loads all mission models from a collection of ScriptMissionModel. This clears existing mission models first. /// Only allowed when state is Idle. /// /// The collection of mission models to load. public MessageResult Load(IEnumerable missionModels) { try { if (missionModels == null) throw new ArgumentNullException(nameof(missionModels)); lock (_stateLockObject) { if (_currentState != MissionManagerState.Idle) return new MessageResult(false, $"Cannot load mission models when MissionManager is in state: {_currentState}"); } // Reset existing mission models first var resetResult = Reset(); if (!resetResult.IsSuccess) return resetResult; // Load all mission models lock (_lockObject) { var loadedCount = 0; var errorCount = 0; var errors = new List(); foreach (var model in missionModels) { if (string.IsNullOrWhiteSpace(model.Name)) continue; try { _missionModels.TryAdd(model.Name, model); loadedCount++; } catch (Exception ex) { errorCount++; errors.Add($"Failed to load mission model '{model.Name}': {ex.Message}"); } } if (errorCount > 0) { return new MessageResult(false, $"Loaded {loadedCount} mission model(s) successfully, {errorCount} failed. Errors: {string.Join("; ", errors)}"); } return new MessageResult(true, $"Loaded {loadedCount} mission model(s) successfully"); } } catch (Exception ex) { return new MessageResult(false, $"Failed to load mission models: {ex.Message}"); } } /// /// Creates a mission instance with parameters provided as a dictionary (matched by parameter name). /// Only allowed when state is Running. /// /// The name of the mission model. /// Dictionary of parameter values keyed by parameter name. /// MessageResult containing the mission ID if successful. public MessageResult CreateMission(string missionName, Dictionary parameters) { try { if (string.IsNullOrWhiteSpace(missionName)) return new MessageResult(false, default, "Mission name cannot be null or empty"); ArgumentNullException.ThrowIfNull(parameters); lock (_stateLockObject) { if (_currentState != MissionManagerState.Running) return new MessageResult(false, default, $"Cannot create mission when MissionManager is in state: {_currentState}"); } if (!_missionModels.TryGetValue(missionName, out var model)) return new MessageResult(false, default, $"Mission model '{missionName}' not found"); // Convert dictionary to ScriptMissionParameterModel list var parameterModels = new List(); foreach (var paramModel in model.Parameters) { // Skip CancellationToken parameters if (paramModel.Type == typeof(CancellationToken)) continue; // Get value from dictionary or use default object? value = parameters.TryGetValue(paramModel.Name, out var paramValue) ? paramValue : paramModel.DefaultValue; parameterModels.Add(new ScriptMissionParameterModel(paramModel.Name, paramModel.Type, value)); } var missionId = Guid.NewGuid(); return CreateMissionInternal(missionName, parameterModels, missionId); } catch (Exception ex) { return new MessageResult(false, default, $"Failed to create mission: {ex.Message}"); } } /// /// Creates a mission instance with parameters provided as an object array (matched by order, skipping CancellationToken). /// Only allowed when state is Running. /// /// The name of the mission model. /// Array of parameter values in order (CancellationToken parameters in model are skipped). /// MessageResult containing the mission ID if successful. public MessageResult CreateMission(string missionName, object[] parameters) { try { if (string.IsNullOrWhiteSpace(missionName)) return new MessageResult(false, default, "Mission name cannot be null or empty"); ArgumentNullException.ThrowIfNull(parameters); lock (_stateLockObject) { if (_currentState != MissionManagerState.Running) return new MessageResult(false, default, $"Cannot create mission when MissionManager is in state: {_currentState}"); } if (!_missionModels.TryGetValue(missionName, out var model)) return new MessageResult(false, default, $"Mission model '{missionName}' not found"); // Convert object array to ScriptMissionParameterModel list // Skip CancellationToken parameters when mapping var parameterModels = new List(); var paramIndex = 0; foreach (var paramModel in model.Parameters) { // Skip CancellationToken parameters if (paramModel.Type == typeof(CancellationToken)) continue; // Get value from array or use default object? value = paramIndex < parameters.Length ? parameters[paramIndex] : paramModel.DefaultValue; parameterModels.Add(new ScriptMissionParameterModel(paramModel.Name, paramModel.Type, value)); paramIndex++; } // Validate that all provided parameters were used if (paramIndex < parameters.Length) { return new MessageResult(false, default, $"Too many parameters provided. Expected {paramIndex} parameters (excluding CancellationToken), but got {parameters.Length}."); } var missionId = Guid.NewGuid(); return CreateMissionInternal(missionName, parameterModels, missionId); } catch (Exception ex) { return new MessageResult(false, default, $"Failed to create mission: {ex.Message}"); } } private MessageResult CreateMissionInternal(string missionName, IEnumerable parameters, Guid missionId) { try { if (!_missionModels.TryGetValue(missionName, out var model)) return new MessageResult(false, default, $"Mission model '{missionName}' not found"); // Populate MissionParameters dictionary with provided parameters var missionParameters = new Dictionary(); foreach (var param in parameters) { // Use DefaultValue from parameter (which may have been set to actual value by caller) missionParameters[param.Name] = param.DefaultValue; } // Add CancellationToken parameters from model if they exist // (These are skipped in CreateMission overloads but need to be in MissionParameters for script execution) foreach (var paramModel in model.Parameters) { if (paramModel.Type == typeof(CancellationToken)) { // CancellationToken will be provided by mission's internal token source missionParameters[paramModel.Name] = CancellationToken.None; } } // Serialize parameters to JSON (include all parameters from model, including CancellationToken) var allParametersForJson = model.Parameters.Select(p => new ScriptMissionParameterDto(p.Name, p.Type.FullName ?? "", p.DefaultValue?.ToString() ?? "null")); var parametersJson = JsonSerializer.Serialize(allParametersForJson); // Create InstanceMission in database FIRST with initial values using (var scope = _scopeFactory.CreateScope()) { var dbContext = scope.ServiceProvider.GetRequiredService(); var dbMission = new InstanceMission { Id = missionId, MissionName = missionName, CreatedAt = DateTime.UtcNow, Parameters = parametersJson, TotalScore = model.TotalScore, State = ScriptMissionState.Idle, Score = 0, StoppedAt = DateTime.UtcNow, Log = string.Empty }; dbContext.InstanceMissions.Add(dbMission); dbContext.SaveChanges(); } // Create LoggerMission with mission ID var loggerMission = new LoggerMission(missionId, _consoleHubContext); // Create ScriptEngineGlobals with LoggerMission var scriptEngineGlobals = new ScriptEngineGlobals(loggerMission, _scopeFactory); var robotNetDict = ScriptHelper.ConvertGlobalsToDictionary(scriptEngineGlobals, typeof(IScriptGlobals)); // Get AppApis with mission ID var appApisDict = _scriptResource.GetMissionGlobals(missionId, CancellationToken.None); // Get GlobalVariables from VariableManager var globalVariablesDict = _variableManager.Globals; // Create ScriptGlobals with LoggerMission, mission ID, GlobalVariables, and MissionParameters var populatedGlobals = new ScriptGlobals( robotNetDict, appApisDict, globalVariablesDict, missionParameters ); // Create ScriptMission instance var mission = new ScriptMission(missionId, model, populatedGlobals); // Add to idle queue _idleMissions.Enqueue(mission); _allMissions.TryAdd(missionId, mission); return new MessageResult(true, missionId, $"Mission '{missionName}' created successfully with ID: {missionId}"); } catch (Exception ex) { return new MessageResult(false, default, $"Failed to create mission '{missionName}': {ex.Message}"); } } /// /// Gets a mission instance by ID. /// public ScriptMission? GetMission(Guid missionId) { _allMissions.TryGetValue(missionId, out var mission); return mission; } /// /// Gets a mission model by name. /// public ScriptMissionModel? GetMissionModel(string name) { _missionModels.TryGetValue(name, out var model); return model; } /// /// Gets all mission instances. /// public IEnumerable GetAllMissions() { return _allMissions.Values; } /// /// Gets all mission models as ScriptMissionDto array. /// public ScriptMissionDto[] GetScriptMissions() { lock (_lockObject) { return _missionModels.Values.Select(m => new ScriptMissionDto( m.Name, m.Parameters.Select(p => new ScriptMissionParameterDto( p.Name, p.Type.FullName ?? p.Type.Name, p.DefaultValue?.ToString())).ToArray())).ToArray(); } } /// /// Finds specific mission models by names as ScriptMissionDto array. /// public ScriptMissionDto[] FindScriptMissions(string[] names) { if (names == null) throw new ArgumentNullException(nameof(names)); lock (_lockObject) { return _missionModels.Values .Where(m => names.Contains(m.Name)) .Select(m => new ScriptMissionDto( m.Name, m.Parameters.Select(p => new ScriptMissionParameterDto( p.Name, p.Type.FullName ?? p.Type.Name, p.DefaultValue?.ToString())).ToArray())).ToArray(); } } /// /// Starts the MissionManager. This will start missions with AutoStart == true. /// public MessageResult Start() { try { _stateMachine.Fire(MissionManagerTrigger.Start); return new MessageResult(true, "MissionManager started successfully"); } catch (Exception ex) { return new MessageResult(false, $"Failed to start MissionManager: {ex.Message}"); } } /// /// Stops the MissionManager. This will ensure all missions are stopped. /// public MessageResult Stop() { try { _stateMachine.Fire(MissionManagerTrigger.Stop); return new MessageResult(true, "MissionManager stop initiated"); } catch (Exception ex) { return new MessageResult(false, $"Failed to stop MissionManager: {ex.Message}"); } } /// /// Checks if all missions are not running. /// public bool AreAllMissionsNotRunning() { lock (_queueLockObject) { return _idleMissions.IsEmpty && _runningMissions.IsEmpty; } } /// /// Gets the count of mission models. /// public int MissionModelCount => _missionModels.Count; /// /// Gets the count of active missions. /// public int ActiveMissionCount => _allMissions.Count; /// /// Disposes the MissionManager and all missions. /// public void Dispose() { if (_disposed) return; // Stop state machine first try { _stateMachine.Stop(); } catch { // Ignore errors when stopping state machine } // Stop running handler _runningHandlerCts?.Cancel(); if (_runningHandlerTask != null) { try { _runningHandlerTask.Wait(TimeSpan.FromSeconds(5)); } catch { // Ignore errors } } // Dispose all missions lock (_queueLockObject) { while (_idleMissions.TryDequeue(out var mission)) { try { mission.Dispose(); } catch { // Ignore disposal errors } } while (_runningMissions.TryDequeue(out var mission)) { try { mission.Dispose(); } catch { // Ignore disposal errors } } } _allMissions.Clear(); _missionModels.Clear(); _runningHandlerCts?.Dispose(); _stoppedWaitHandle.Dispose(); _disposed = true; GC.SuppressFinalize(this); } }