Current state: Strategy builds and loads correctly, passes 240+ tests, backtest (Strategy Analyzer) works but zero trades execute on live/SIM. Root cause identified: NT8OrderAdapter.ExecuteInNT8() is a stub - it logs to an internal list but never calls EnterLong/EnterShort/SetStopLoss/ SetProfitTarget. Fix is ready in TASK_01_WIRE_NT8_EXECUTION.md. Task files added (ready for Kilocode): - TASK_01_WIRE_NT8_EXECUTION.md (CRITICAL - INT8ExecutionBridge + wiring) - TASK_02_EMERGENCY_KILL_SWITCH.md (CRITICAL - kill switch + verbose logging) - TASK_03_WIRE_CIRCUIT_BREAKER.md (HIGH - wire ExecutionCircuitBreaker) Build Status: All 240+ tests passing, zero errors Next: Run Kilocode against TASK_01, TASK_02, TASK_03 in order
347 lines
12 KiB
C#
347 lines
12 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using System.Runtime.CompilerServices;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
[assembly: InternalsVisibleTo("NT8.Core.Tests")]
|
|
[assembly: InternalsVisibleTo("NT8.Integration.Tests")]
|
|
|
|
namespace NT8.Core.Execution
|
|
{
|
|
/// <summary>
|
|
/// Circuit breaker implementation for execution systems to prevent cascading failures
|
|
/// </summary>
|
|
public class ExecutionCircuitBreaker
|
|
{
|
|
private readonly ILogger _logger;
|
|
private readonly NT8.Core.Logging.ILogger _sdkLogger;
|
|
private readonly object _lock = new object();
|
|
|
|
private CircuitBreakerStatus _status;
|
|
private DateTime _lastFailureTime;
|
|
private int _failureCount;
|
|
private DateTime _nextRetryTime;
|
|
private readonly TimeSpan _timeout;
|
|
private readonly int _failureThreshold;
|
|
private readonly TimeSpan _retryTimeout;
|
|
|
|
private readonly Queue<TimeSpan> _executionTimes;
|
|
private readonly int _latencyWindowSize;
|
|
|
|
private readonly Queue<DateTime> _rejectionTimes;
|
|
private readonly int _rejectionWindowSize;
|
|
|
|
// Log helpers — route through whichever logger is available
|
|
private void LogDebug(string message) { if (_logger != null) _logger.LogDebug(message); else if (_sdkLogger != null) _sdkLogger.LogDebug(message); }
|
|
private void LogInfo(string message) { if (_logger != null) _logger.LogInformation(message); else if (_sdkLogger != null) _sdkLogger.LogInformation(message); }
|
|
private void LogWarn(string message) { if (_logger != null) _logger.LogWarning(message); else if (_sdkLogger != null) _sdkLogger.LogWarning(message); }
|
|
private void LogErr(string message) { if (_logger != null) _logger.LogError(message); else if (_sdkLogger != null) _sdkLogger.LogError(message); }
|
|
|
|
/// <summary>
|
|
/// Constructor accepting NT8.Core.Logging.ILogger.
|
|
/// Use this overload from NinjaScript (.cs) files — no Microsoft.Extensions.Logging reference required.
|
|
/// </summary>
|
|
public ExecutionCircuitBreaker(
|
|
NT8.Core.Logging.ILogger sdkLogger,
|
|
int failureThreshold = 3,
|
|
TimeSpan? timeout = null,
|
|
TimeSpan? retryTimeout = null,
|
|
int latencyWindowSize = 100,
|
|
int rejectionWindowSize = 10)
|
|
{
|
|
_sdkLogger = sdkLogger;
|
|
_logger = null;
|
|
_status = CircuitBreakerStatus.Closed;
|
|
_failureCount = 0;
|
|
_lastFailureTime = DateTime.MinValue;
|
|
_timeout = timeout ?? TimeSpan.FromSeconds(30);
|
|
_retryTimeout = retryTimeout ?? TimeSpan.FromSeconds(5);
|
|
_failureThreshold = failureThreshold;
|
|
_latencyWindowSize = latencyWindowSize;
|
|
_rejectionWindowSize = rejectionWindowSize;
|
|
_executionTimes = new Queue<TimeSpan>();
|
|
_rejectionTimes = new Queue<DateTime>();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Constructor accepting Microsoft.Extensions.Logging.ILogger.
|
|
/// Use this overload from DLL projects and unit tests.
|
|
/// </summary>
|
|
internal ExecutionCircuitBreaker(
|
|
ILogger<ExecutionCircuitBreaker> logger,
|
|
int failureThreshold = 3,
|
|
TimeSpan? timeout = null,
|
|
TimeSpan? retryTimeout = null,
|
|
int latencyWindowSize = 100,
|
|
int rejectionWindowSize = 10)
|
|
{
|
|
if (logger == null)
|
|
throw new ArgumentNullException("logger");
|
|
|
|
_logger = logger;
|
|
_sdkLogger = null;
|
|
_status = CircuitBreakerStatus.Closed;
|
|
_failureCount = 0;
|
|
_lastFailureTime = DateTime.MinValue;
|
|
_timeout = timeout ?? TimeSpan.FromSeconds(30);
|
|
_retryTimeout = retryTimeout ?? TimeSpan.FromSeconds(5);
|
|
_failureThreshold = failureThreshold;
|
|
_latencyWindowSize = latencyWindowSize;
|
|
_rejectionWindowSize = rejectionWindowSize;
|
|
_executionTimes = new Queue<TimeSpan>();
|
|
_rejectionTimes = new Queue<DateTime>();
|
|
}
|
|
|
|
/// <summary>Records execution time for latency monitoring.</summary>
|
|
public void RecordExecutionTime(TimeSpan latency)
|
|
{
|
|
try
|
|
{
|
|
lock (_lock)
|
|
{
|
|
_executionTimes.Enqueue(latency);
|
|
while (_executionTimes.Count > _latencyWindowSize)
|
|
_executionTimes.Dequeue();
|
|
|
|
if (_status == CircuitBreakerStatus.Closed && HasExcessiveLatency())
|
|
TripCircuitBreaker("Excessive execution latency detected");
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
LogErr(string.Format("Failed to record execution time: {0}", ex.Message));
|
|
throw;
|
|
}
|
|
}
|
|
|
|
/// <summary>Records an order rejection.</summary>
|
|
public void RecordOrderRejection(string reason)
|
|
{
|
|
if (string.IsNullOrEmpty(reason))
|
|
reason = "Unknown";
|
|
|
|
try
|
|
{
|
|
lock (_lock)
|
|
{
|
|
_rejectionTimes.Enqueue(DateTime.UtcNow);
|
|
while (_rejectionTimes.Count > _rejectionWindowSize)
|
|
_rejectionTimes.Dequeue();
|
|
|
|
if (_status == CircuitBreakerStatus.Closed && HasExcessiveRejections())
|
|
TripCircuitBreaker(string.Format("Excessive order rejections: {0}", reason));
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
LogErr(string.Format("Failed to record order rejection: {0}", ex.Message));
|
|
throw;
|
|
}
|
|
}
|
|
|
|
/// <summary>Returns true if an order should be allowed through.</summary>
|
|
public bool ShouldAllowOrder()
|
|
{
|
|
try
|
|
{
|
|
lock (_lock)
|
|
{
|
|
switch (_status)
|
|
{
|
|
case CircuitBreakerStatus.Closed:
|
|
return true;
|
|
|
|
case CircuitBreakerStatus.Open:
|
|
if (DateTime.UtcNow >= _nextRetryTime)
|
|
{
|
|
_status = CircuitBreakerStatus.HalfOpen;
|
|
LogWarn("Circuit breaker transitioning to Half-Open state");
|
|
return true;
|
|
}
|
|
LogDebug("Circuit breaker is Open - blocking order");
|
|
return false;
|
|
|
|
case CircuitBreakerStatus.HalfOpen:
|
|
LogDebug("Circuit breaker is Half-Open - allowing test order");
|
|
return true;
|
|
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
LogErr(string.Format("Failed to check ShouldAllowOrder: {0}", ex.Message));
|
|
throw;
|
|
}
|
|
}
|
|
|
|
/// <summary>Returns the current circuit breaker state.</summary>
|
|
public CircuitBreakerState GetState()
|
|
{
|
|
try
|
|
{
|
|
lock (_lock)
|
|
{
|
|
return new CircuitBreakerState(
|
|
_status != CircuitBreakerStatus.Closed,
|
|
_status,
|
|
GetStatusReason(),
|
|
_failureCount);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
LogErr(string.Format("Failed to get state: {0}", ex.Message));
|
|
throw;
|
|
}
|
|
}
|
|
|
|
/// <summary>Resets the circuit breaker to Closed state.</summary>
|
|
public void Reset()
|
|
{
|
|
try
|
|
{
|
|
lock (_lock)
|
|
{
|
|
_status = CircuitBreakerStatus.Closed;
|
|
_failureCount = 0;
|
|
_lastFailureTime = DateTime.MinValue;
|
|
LogInfo("Circuit breaker reset to Closed state");
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
LogErr(string.Format("Failed to reset circuit breaker: {0}", ex.Message));
|
|
throw;
|
|
}
|
|
}
|
|
|
|
/// <summary>Call after a successful order submission.</summary>
|
|
public void OnSuccess()
|
|
{
|
|
try
|
|
{
|
|
lock (_lock)
|
|
{
|
|
if (_status == CircuitBreakerStatus.HalfOpen)
|
|
{
|
|
Reset();
|
|
LogInfo("Circuit breaker reset after successful test operation");
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
LogErr(string.Format("Failed to handle OnSuccess: {0}", ex.Message));
|
|
throw;
|
|
}
|
|
}
|
|
|
|
/// <summary>Call after a failed order submission.</summary>
|
|
public void OnFailure()
|
|
{
|
|
try
|
|
{
|
|
lock (_lock)
|
|
{
|
|
_failureCount++;
|
|
_lastFailureTime = DateTime.UtcNow;
|
|
|
|
if (_status == CircuitBreakerStatus.HalfOpen ||
|
|
(_status == CircuitBreakerStatus.Closed && _failureCount >= _failureThreshold))
|
|
{
|
|
TripCircuitBreaker("Failure threshold exceeded");
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
LogErr(string.Format("Failed to handle OnFailure: {0}", ex.Message));
|
|
throw;
|
|
}
|
|
}
|
|
|
|
private void TripCircuitBreaker(string reason)
|
|
{
|
|
_status = CircuitBreakerStatus.Open;
|
|
_nextRetryTime = DateTime.UtcNow.Add(_timeout);
|
|
LogWarn(string.Format("Circuit breaker TRIPPED: {0}. Will retry at {1}", reason, _nextRetryTime));
|
|
}
|
|
|
|
private bool HasExcessiveLatency()
|
|
{
|
|
if (_executionTimes.Count < 3)
|
|
return false;
|
|
var avgLatency = TimeSpan.FromMilliseconds(_executionTimes.Average(ts => ts.TotalMilliseconds));
|
|
return avgLatency.TotalSeconds > 5.0;
|
|
}
|
|
|
|
private bool HasExcessiveRejections()
|
|
{
|
|
if (_rejectionTimes.Count < _rejectionWindowSize)
|
|
return false;
|
|
var recentWindow = TimeSpan.FromMinutes(1);
|
|
var recentRejections = _rejectionTimes.Count(dt => DateTime.UtcNow - dt <= recentWindow);
|
|
return recentRejections >= _rejectionWindowSize;
|
|
}
|
|
|
|
private string GetStatusReason()
|
|
{
|
|
switch (_status)
|
|
{
|
|
case CircuitBreakerStatus.Closed:
|
|
return "Normal operation";
|
|
case CircuitBreakerStatus.Open:
|
|
return string.Format("Tripped due to failures. Count: {0}, Last: {1}", _failureCount, _lastFailureTime);
|
|
case CircuitBreakerStatus.HalfOpen:
|
|
return "Testing recovery after timeout";
|
|
default:
|
|
return "Unknown";
|
|
}
|
|
}
|
|
|
|
/// <summary>Returns average execution latency.</summary>
|
|
public TimeSpan GetAverageExecutionTime()
|
|
{
|
|
try
|
|
{
|
|
lock (_lock)
|
|
{
|
|
if (_executionTimes.Count == 0)
|
|
return TimeSpan.Zero;
|
|
return TimeSpan.FromMilliseconds(_executionTimes.Average(ts => ts.TotalMilliseconds));
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
LogErr(string.Format("Failed to get average execution time: {0}", ex.Message));
|
|
throw;
|
|
}
|
|
}
|
|
|
|
/// <summary>Returns rejection rate as a percentage.</summary>
|
|
public double GetRejectionRate()
|
|
{
|
|
try
|
|
{
|
|
lock (_lock)
|
|
{
|
|
if (_rejectionTimes.Count == 0)
|
|
return 0.0;
|
|
var oneMinuteAgo = DateTime.UtcNow.AddMinutes(-1);
|
|
var recentRejections = _rejectionTimes.Count(dt => dt >= oneMinuteAgo);
|
|
return (double)recentRejections / _rejectionWindowSize * 100.0;
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
LogErr(string.Format("Failed to get rejection rate: {0}", ex.Message));
|
|
throw;
|
|
}
|
|
}
|
|
}
|
|
}
|