From 52c1dbe6feb06404bff7d68ea9b7eb44a56b8955 Mon Sep 17 00:00:00 2001 From: hawthorne-abendsen <49230725+hawthorne-abendsen@users.noreply.github.com> Date: Wed, 17 Nov 2021 18:44:34 +0200 Subject: [PATCH 01/13] Sync refactoring --- Centaurus.Domain/Contexts/ExecutionContext.cs | 7 + Centaurus.Domain/DataProvider/DataProvider.cs | 28 --- ...r.cs => ConstellationQuantumExtesnions.cs} | 2 +- .../ExecutionContextExtensions.cs | 0 .../RawPubKeyExtensions.cs | 0 .../Helpers/SyncCursorUpdateExtensions.cs | 39 ++++ .../AlphaQuantaBatchHandler.cs | 9 +- .../AuditorStatisticsMessageHandler.cs | 2 +- .../HandshakeRequestHandler.cs | 16 +- .../AuditorHandshakeResponseHandler.cs | 6 +- ...teHandler.cs => SyncCursorResetHandler.cs} | 14 +- .../Quanta/Handlers/QuantumHandler.cs | 5 +- .../Sync}/ApexItemsBatch.cs | 37 ++- .../Quanta/Sync/ApexItemsBatchPortion.cs | 73 ++++++ .../Quanta/Sync/SyncCursorUpdate.cs | 39 ++++ Centaurus.Domain/Quanta/Sync/SyncPortion.cs | 19 ++ .../Quanta/Sync/SyncQuantaDataWorker.cs | 215 ++++++++++++++++++ .../Quanta/{ => Sync}/SyncStorage.cs | 189 ++++++++------- .../ResultManagers/ResultManager.cs | 2 +- .../PerformanceStatisticsManager.cs | 2 +- .../WebSockets/Centaurus/ConnectionBase.cs | 32 ++- .../Incoming/IncomingAuditorConnection.cs | 55 +++-- .../Centaurus/Incoming/QuantumSyncWorker.cs | 182 --------------- .../InternalMessages/SyncCursor.cs | 30 +++ .../InternalMessages/SyncCursorReset.cs | 14 ++ .../InternalMessages/SyncCursorUpdate.cs | 18 -- .../Messages/HandshakeResponse.cs | 8 +- Centaurus.Models/Messages/Message.cs | 1 + Centaurus.Models/Messages/MessageTypes.cs | 2 +- Centaurus.NetSDK/Connection.cs | 2 +- Centaurus.Test.Domain/QuantumStorageTests.cs | 20 +- 31 files changed, 695 insertions(+), 373 deletions(-) rename Centaurus.Domain/Helpers/{ConstellationQuantumHelper.cs => ConstellationQuantumExtesnions.cs} (97%) rename Centaurus.Domain/{Extensions => Helpers}/ExecutionContextExtensions.cs (100%) rename Centaurus.Domain/{Extensions => Helpers}/RawPubKeyExtensions.cs (100%) create mode 100644 Centaurus.Domain/Helpers/SyncCursorUpdateExtensions.cs rename Centaurus.Domain/MessageHandlers/{SyncCursorUpdateHandler.cs => SyncCursorResetHandler.cs} (56%) rename Centaurus.Domain/{Helpers => Quanta/Sync}/ApexItemsBatch.cs (71%) create mode 100644 Centaurus.Domain/Quanta/Sync/ApexItemsBatchPortion.cs create mode 100644 Centaurus.Domain/Quanta/Sync/SyncCursorUpdate.cs create mode 100644 Centaurus.Domain/Quanta/Sync/SyncPortion.cs create mode 100644 Centaurus.Domain/Quanta/Sync/SyncQuantaDataWorker.cs rename Centaurus.Domain/Quanta/{ => Sync}/SyncStorage.cs (50%) delete mode 100644 Centaurus.Domain/WebSockets/Centaurus/Incoming/QuantumSyncWorker.cs create mode 100644 Centaurus.Models/InternalMessages/SyncCursor.cs create mode 100644 Centaurus.Models/InternalMessages/SyncCursorReset.cs delete mode 100644 Centaurus.Models/InternalMessages/SyncCursorUpdate.cs diff --git a/Centaurus.Domain/Contexts/ExecutionContext.cs b/Centaurus.Domain/Contexts/ExecutionContext.cs index 39a7a2f3..6d7f3e29 100644 --- a/Centaurus.Domain/Contexts/ExecutionContext.cs +++ b/Centaurus.Domain/Contexts/ExecutionContext.cs @@ -55,6 +55,8 @@ public ExecutionContext(Settings settings, IPersistentStorage storage, PaymentPr InfoConnectionManager = new InfoConnectionManager(this); + SyncQuantaDataWorker = new SyncQuantaDataWorker(this); + Catchup = new Catchup(this); StateManager = new StateManager(this); @@ -124,6 +126,9 @@ private void HandlePendingQuanta(List pendingQuanta) //add signatures ResultManager.Add(new QuantumSignatures { Apex = quantum.Quantum.Apex, Signatures = quantum.Signatures }); + + if (quantum.Quantum.Apex % 1000 == 0) + logger.Info($"Pending quanta handling. Current apex: {quantum.Quantum.Apex}, last pending quanta: {pendingQuanta.Last().Quantum.Apex}"); } catch (AggregateException exc) { @@ -343,6 +348,8 @@ private void PendingUpdatesManager_OnBatchSaved(BatchSavedInfo batchInfo) public InfoConnectionManager InfoConnectionManager { get; } + public SyncQuantaDataWorker SyncQuantaDataWorker { get; } + public Catchup Catchup { get; } public InfoCommandsHandlers InfoCommandsHandlers { get; } diff --git a/Centaurus.Domain/DataProvider/DataProvider.cs b/Centaurus.Domain/DataProvider/DataProvider.cs index 4c34ae1f..6724dcdb 100644 --- a/Centaurus.Domain/DataProvider/DataProvider.cs +++ b/Centaurus.Domain/DataProvider/DataProvider.cs @@ -84,34 +84,6 @@ public QuantumInfoResponse LoadQuantaInfo(string cursor, bool isDesc, int limit, }; } - /// - /// Fetches all quanta where apex is greater than the specified one. - /// - /// - /// Count of quanta to load. Loads all if equal or less than 0 - /// - public List GetQuantaSyncBatchItemsAboveApex(ulong apex, int count = 0) - { - var query = (IEnumerable)Context.PersistentStorage.LoadQuantaAboveApex(apex, count) - .OrderBy(q => q.Apex); - if (count > 0) - query = query.Take(count); - return query.Select(q => q.ToBatchItemQuantum()).ToList(); - } - - /// - /// Fetches all quanta's signatures where apex is greater than the specified one. - /// - /// - /// Count of quanta to load. Loads all if equal or less than 0 - /// - public List GetSignaturesSyncBatchItemsAboveApex(ulong apex, int count = 0) - { - var query = (IEnumerable)Context.PersistentStorage.LoadQuantaAboveApex(apex, count) - .OrderBy(q => q.Apex); - return query.Select(q => q.ToQuantumSignatures()).ToList(); - } - /// /// Get last persisted apex /// diff --git a/Centaurus.Domain/Helpers/ConstellationQuantumHelper.cs b/Centaurus.Domain/Helpers/ConstellationQuantumExtesnions.cs similarity index 97% rename from Centaurus.Domain/Helpers/ConstellationQuantumHelper.cs rename to Centaurus.Domain/Helpers/ConstellationQuantumExtesnions.cs index 0dcf0688..7c6e3ea5 100644 --- a/Centaurus.Domain/Helpers/ConstellationQuantumHelper.cs +++ b/Centaurus.Domain/Helpers/ConstellationQuantumExtesnions.cs @@ -6,7 +6,7 @@ namespace Centaurus.Domain { - public static class ConstellationQuantumHelper + public static class ConstellationQuantumExtesnions { public static void Validate(this ConstellationQuantum constellationQuantum, ExecutionContext context) { diff --git a/Centaurus.Domain/Extensions/ExecutionContextExtensions.cs b/Centaurus.Domain/Helpers/ExecutionContextExtensions.cs similarity index 100% rename from Centaurus.Domain/Extensions/ExecutionContextExtensions.cs rename to Centaurus.Domain/Helpers/ExecutionContextExtensions.cs diff --git a/Centaurus.Domain/Extensions/RawPubKeyExtensions.cs b/Centaurus.Domain/Helpers/RawPubKeyExtensions.cs similarity index 100% rename from Centaurus.Domain/Extensions/RawPubKeyExtensions.cs rename to Centaurus.Domain/Helpers/RawPubKeyExtensions.cs diff --git a/Centaurus.Domain/Helpers/SyncCursorUpdateExtensions.cs b/Centaurus.Domain/Helpers/SyncCursorUpdateExtensions.cs new file mode 100644 index 00000000..d5e497d1 --- /dev/null +++ b/Centaurus.Domain/Helpers/SyncCursorUpdateExtensions.cs @@ -0,0 +1,39 @@ +using Centaurus.Domain.Quanta.Sync; +using Centaurus.Models; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Centaurus.Domain +{ + public static class SyncCursorUpdateExtensions + { + public static List ToDomainModel(this List syncCursors) + { + if (syncCursors == null) + throw new ArgumentNullException(nameof(syncCursors)); + + var cursors = new List(); + + foreach (var cursorReset in syncCursors) + { + var cursorType = default(SyncCursorType); + switch (cursorReset.Type) + { + case XdrSyncCursorType.Quanta: + cursorType = SyncCursorType.Quanta; + break; + case XdrSyncCursorType.Signatures: + cursorType = SyncCursorType.Signatures; + break; + default: + throw new ArgumentNullException($"{cursorReset.Type} cursor type is not supported."); + } + var cursor = cursorReset.ClearCursor ? null : (ulong?)cursorReset.Cursor; + cursors.Add(new SyncCursorUpdate(default(DateTime), cursor, cursorType)); + } + + return cursors; + } + } +} diff --git a/Centaurus.Domain/MessageHandlers/AlphaQuantaBatchHandler.cs b/Centaurus.Domain/MessageHandlers/AlphaQuantaBatchHandler.cs index 6265918d..9df00182 100644 --- a/Centaurus.Domain/MessageHandlers/AlphaQuantaBatchHandler.cs +++ b/Centaurus.Domain/MessageHandlers/AlphaQuantaBatchHandler.cs @@ -51,9 +51,14 @@ private async Task AddQuantaToQueue(OutgoingConnection connection, IncomingMessa if (quantum.Apex != lastKnownApex + 1) { - await connection.SendMessage(new SyncCursorUpdate + await connection.SendMessage(new SyncCursorReset { - QuantaCursor = quantumHandler.LastAddedQuantumApex + SyncCursors = new List { + new SyncCursor { + Type = XdrSyncCursorType.Quanta, + Cursor = quantumHandler.LastAddedQuantumApex + } + } }.CreateEnvelope()); logger.Warn($"Batch has invalid quantum apexes (current: {quantumHandler.LastAddedQuantumApex}, received: {quantum.Apex}). New apex cursor request sent."); return; diff --git a/Centaurus.Domain/MessageHandlers/AuditorStatisticsMessageHandler.cs b/Centaurus.Domain/MessageHandlers/AuditorStatisticsMessageHandler.cs index 85896575..2465d6a5 100644 --- a/Centaurus.Domain/MessageHandlers/AuditorStatisticsMessageHandler.cs +++ b/Centaurus.Domain/MessageHandlers/AuditorStatisticsMessageHandler.cs @@ -25,7 +25,7 @@ public override Task HandleMessage(ConnectionBase connection, IncomingMessage me var auditor = connection.PubKeyAddress; var statistics = (AuditorPerfStatistics)message.Envelope.Message; - Context.PerformanceStatisticsManager.AddAuditorStatistics(auditor, statistics); + Context.PerformanceStatisticsManager?.AddAuditorStatistics(auditor, statistics); return Task.CompletedTask; } } diff --git a/Centaurus.Domain/MessageHandlers/HandshakeRequestHandler.cs b/Centaurus.Domain/MessageHandlers/HandshakeRequestHandler.cs index 8599308d..cfa32848 100644 --- a/Centaurus.Domain/MessageHandlers/HandshakeRequestHandler.cs +++ b/Centaurus.Domain/MessageHandlers/HandshakeRequestHandler.cs @@ -22,17 +22,19 @@ public override async Task HandleMessage(OutgoingConnection connection, Incoming { var handshakeRequest = (HandshakeRequest)message.Envelope.Message; - var quantaCursor = Context.QuantumHandler.LastAddedQuantumApex; + var quantaCursor = new SyncCursor { Type = XdrSyncCursorType.Quanta, Cursor = Context.QuantumHandler.LastAddedQuantumApex }; + //if Prime than the node will receive results from auditors - var resultCursor = Context.RoleManager.ParticipationLevel == CentaurusNodeParticipationLevel.Prime - ? ulong.MaxValue - : Context.PendingUpdatesManager.LastSavedApex; + var signaturesCursor = new SyncCursor + { + Type = XdrSyncCursorType.Signatures, + Cursor = Context.PendingUpdatesManager.LastSavedApex, + ClearCursor = Context.RoleManager.ParticipationLevel == CentaurusNodeParticipationLevel.Prime + }; await connection.SendMessage(new AuditorHandshakeResponse { HandshakeData = handshakeRequest.HandshakeData, - QuantaCursor = quantaCursor, - ResultCursor = resultCursor, - State = Context.StateManager.State + Cursors = new List { quantaCursor, signaturesCursor } }); //after sending auditor handshake the connection becomes ready diff --git a/Centaurus.Domain/MessageHandlers/HandshakeResponseHandlers/AuditorHandshakeResponseHandler.cs b/Centaurus.Domain/MessageHandlers/HandshakeResponseHandlers/AuditorHandshakeResponseHandler.cs index 4d55ea2d..b111d0fb 100644 --- a/Centaurus.Domain/MessageHandlers/HandshakeResponseHandlers/AuditorHandshakeResponseHandler.cs +++ b/Centaurus.Domain/MessageHandlers/HandshakeResponseHandlers/AuditorHandshakeResponseHandler.cs @@ -1,4 +1,5 @@ -using Centaurus.Models; +using Centaurus.Domain.Quanta.Sync; +using Centaurus.Models; using NLog; using System; using System.Collections.Generic; @@ -37,8 +38,9 @@ await incomingAuditorConnection.SendMessage(new StateUpdateMessage State = Context.StateManager.State }.CreateEnvelope()); + incomingAuditorConnection.SetSyncCursor(true, auditorHandshake.Cursors.ToDomainModel().ToArray()); + Context.StateManager.SetAuditorState(connection.PubKey, auditorHandshake.State); - incomingAuditorConnection.SetSyncCursor(auditorHandshake.QuantaCursor, auditorHandshake.ResultCursor); //request quanta on rising if (Context.StateManager.State == State.Rising) diff --git a/Centaurus.Domain/MessageHandlers/SyncCursorUpdateHandler.cs b/Centaurus.Domain/MessageHandlers/SyncCursorResetHandler.cs similarity index 56% rename from Centaurus.Domain/MessageHandlers/SyncCursorUpdateHandler.cs rename to Centaurus.Domain/MessageHandlers/SyncCursorResetHandler.cs index e326689a..91414acf 100644 --- a/Centaurus.Domain/MessageHandlers/SyncCursorUpdateHandler.cs +++ b/Centaurus.Domain/MessageHandlers/SyncCursorResetHandler.cs @@ -4,18 +4,19 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Centaurus.Domain.Quanta.Sync; using Centaurus.Models; namespace Centaurus.Domain.Handlers.AlphaHandlers { - public class SyncCursorUpdateHandler : MessageHandlerBase + public class SyncCursorResetHandler : MessageHandlerBase { - public SyncCursorUpdateHandler(ExecutionContext context) + public SyncCursorResetHandler(ExecutionContext context) : base(context) { } - public override string SupportedMessageType { get; } = typeof(SyncCursorUpdate).Name; + public override string SupportedMessageType { get; } = typeof(SyncCursorReset).Name; public override bool IsAuditorOnly { get; } = true; @@ -26,10 +27,9 @@ public SyncCursorUpdateHandler(ExecutionContext context) public override Task HandleMessage(IncomingAuditorConnection connection, IncomingMessage message) { - var batchRequest = (SyncCursorUpdate)message.Envelope.Message; - var quantaCursor = batchRequest.QuantaCursor == ulong.MaxValue ? null : (ulong?)batchRequest.QuantaCursor; - var signaturesCursor = batchRequest.SignaturesCursor == ulong.MaxValue ? null : (ulong?)batchRequest.SignaturesCursor; - connection.SetSyncCursor(quantaCursor, signaturesCursor); + var cursorResetRequest = (SyncCursorReset)message.Envelope.Message; + + connection.SetSyncCursor(true, cursorResetRequest.SyncCursors.ToDomainModel().ToArray()); return Task.CompletedTask; } } diff --git a/Centaurus.Domain/Quanta/Handlers/QuantumHandler.cs b/Centaurus.Domain/Quanta/Handlers/QuantumHandler.cs index 9c53dc80..62a4241c 100644 --- a/Centaurus.Domain/Quanta/Handlers/QuantumHandler.cs +++ b/Centaurus.Domain/Quanta/Handlers/QuantumHandler.cs @@ -37,7 +37,7 @@ public QuantumProcessingItem HandleAsync(Quantum quantum, Task signatureVa && QuantaThrottlingManager.Current.MaxItemsPerSecond <= awaitedQuanta.Count) throw new TooManyRequestsException("Server is too busy. Try again later."); var processingItem = new QuantumProcessingItem(quantum, signatureValidation); - if (processingItem.Quantum.Apex > 0) + if (!Context.IsAlpha) LastAddedQuantumApex = processingItem.Quantum.Apex; lock (awaitedQuantaSyncRoot) awaitedQuanta.Enqueue(processingItem); @@ -187,6 +187,9 @@ async Task ProcessQuantum(QuantumProcessingItem processingItem) ProcessResult(processingItem); + if (Context.IsAlpha) + LastAddedQuantumApex = processingItem.Quantum.Apex; + processingItem.Processed(); logger.Trace($"Message of type {quantum.GetType().Name} with apex {quantum.Apex} is handled."); diff --git a/Centaurus.Domain/Helpers/ApexItemsBatch.cs b/Centaurus.Domain/Quanta/Sync/ApexItemsBatch.cs similarity index 71% rename from Centaurus.Domain/Helpers/ApexItemsBatch.cs rename to Centaurus.Domain/Quanta/Sync/ApexItemsBatch.cs index 23980962..e8d26aa1 100644 --- a/Centaurus.Domain/Helpers/ApexItemsBatch.cs +++ b/Centaurus.Domain/Quanta/Sync/ApexItemsBatch.cs @@ -6,15 +6,17 @@ using System.Threading; using System.Threading.Tasks; -namespace Centaurus +namespace Centaurus.Domain.Quanta.Sync { - public class ApexItemsBatch + public partial class ApexItemsBatch: ContextualBase where T : IApex { - public ApexItemsBatch(ulong start, int size, List initData) + public ApexItemsBatch(ExecutionContext context, ulong start, int size, int portionSize, List initData) + :base(context) { Start = start; Size = size; + PortionSize = portionSize; if (initData == null) throw new ArgumentNullException(nameof(initData)); if (initData.Count > size) @@ -33,18 +35,20 @@ public ApexItemsBatch(ulong start, int size, List initData) /// /// Returns range of items /// - /// Exclusive from + /// /// + /// /// - public List GetItems(ulong from, int limit) + public List GetItems(ulong from, int limit, bool inclusiveFrom = false) { - from = from + 1; + if (!inclusiveFrom || from == 0) //we don't have 0 apex, so force exclusive for 0 + from = from + 1; var skip = from - Start; - limit = Math.Min((int)(LastApex - Start + skip), limit); var result = data .Skip((int)skip) .Take(limit).ToList(); + //last item can be null, if inserting is in progress if (result.Count > 0 && result[result.Count - 1] == null) result.RemoveAt(result.Count - 1); @@ -57,6 +61,10 @@ public List GetItems(ulong from, int limit) public int Size { get; } + public int PortionSize { get; } + + public bool IsFulfilled => data.Count == Size; + public void Add(ulong apex, T item) { if (apex == LastApex + 1 || apex == Start) @@ -106,5 +114,18 @@ private void AddToData(ulong apex, T item) data.Add(item); LastApex = apex; } + + private Dictionary portions = new Dictionary(); + + public SyncPortion GetData(ulong apex, bool force) + { + var portionStart = apex - (apex % (ulong)PortionSize); + if (!portions.TryGetValue(portionStart, out var portion)) + { + portion = new ApexItemsBatchPortion(portionStart, PortionSize, this); + portions.Add(portionStart, portion); + } + return portion.GetBatch(force); + } } -} +} \ No newline at end of file diff --git a/Centaurus.Domain/Quanta/Sync/ApexItemsBatchPortion.cs b/Centaurus.Domain/Quanta/Sync/ApexItemsBatchPortion.cs new file mode 100644 index 00000000..364cd82e --- /dev/null +++ b/Centaurus.Domain/Quanta/Sync/ApexItemsBatchPortion.cs @@ -0,0 +1,73 @@ +using Centaurus.Models; +using System; +using System.Linq; + +namespace Centaurus.Domain.Quanta.Sync +{ + public partial class ApexItemsBatch + { + class ApexItemsBatchPortion + { + public ApexItemsBatchPortion(ulong start, int size, ApexItemsBatch source) + { + Start = lastSerializedBatchApex = start; + Size = size; + LastApex = start + (ulong)size; + this.source = source ?? throw new ArgumentNullException(nameof(source)); + } + + protected ApexItemsBatch source; + + private SyncPortion batch; + private ulong lastSerializedBatchApex = 0; + + public SyncPortion GetBatch(bool force) + { + if (batch != null && lastSerializedBatchApex == LastApex) + return batch; + + if (force && source.LastApex >= lastSerializedBatchApex || source.LastApex >= LastApex) + if (lastSerializedBatchApex < source.LastApex) + { + batch = GetBatchData(); + lastSerializedBatchApex = batch.LastDataApex; + } + return batch; + } + + protected SyncPortion GetBatchData() + { + switch (source) + { + case ApexItemsBatch signaturesBatch: + { + var items = signaturesBatch.GetItems(Start, Size, true); + var batch = new QuantumMajoritySignaturesBatch + { + Signatures = items + }; + return new SyncPortion(batch.CreateEnvelope().ToByteArray(), items.Last().Apex); + } + case ApexItemsBatch quantaBatch: + { + var items = quantaBatch.GetItems(Start, Size, true); + var batch = new SyncQuantaBatch + { + Quanta = items, + LastKnownApex = source.Context.QuantumHandler.CurrentApex + }; + return new SyncPortion(batch.CreateEnvelope().ToByteArray(), items.Last().Apex); + } + default: + throw new NotImplementedException($"{nameof(source)} is not supported."); + } + } + + public ulong Start { get; } + + public ulong LastApex { get; } + + public int Size { get; } + } + } +} diff --git a/Centaurus.Domain/Quanta/Sync/SyncCursorUpdate.cs b/Centaurus.Domain/Quanta/Sync/SyncCursorUpdate.cs new file mode 100644 index 00000000..90517197 --- /dev/null +++ b/Centaurus.Domain/Quanta/Sync/SyncCursorUpdate.cs @@ -0,0 +1,39 @@ +using System; + +namespace Centaurus.Domain.Quanta.Sync +{ + public enum SyncCursorType + { + Quanta = 0, + Signatures = 1 + } + + public class SyncCursorUpdate + { + public SyncCursorUpdate(DateTime timeToken, ulong? newCursor, SyncCursorType cursorType) + { + NewCursor = newCursor; + CursorType = cursorType; + TimeToken = timeToken; + } + + public ulong? NewCursor { get; } + + public SyncCursorType CursorType { get; } + + public DateTime TimeToken { get; } + } + + public class AuditorSyncCursorUpdate + { + public AuditorSyncCursorUpdate(IncomingAuditorConnection connection, SyncCursorUpdate syncCursorUpdate) + { + Connection = connection ?? throw new ArgumentNullException(nameof(connection)); + CursorUpdate = syncCursorUpdate ?? throw new ArgumentNullException(nameof(syncCursorUpdate)); + } + + public IncomingAuditorConnection Connection { get; } + + public SyncCursorUpdate CursorUpdate { get; set; } + } +} diff --git a/Centaurus.Domain/Quanta/Sync/SyncPortion.cs b/Centaurus.Domain/Quanta/Sync/SyncPortion.cs new file mode 100644 index 00000000..ab419472 --- /dev/null +++ b/Centaurus.Domain/Quanta/Sync/SyncPortion.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Centaurus.Domain.Quanta.Sync +{ + public class SyncPortion + { + public SyncPortion(byte[] data, ulong lastDataApex) + { + Data = data ?? throw new ArgumentNullException(nameof(data)); + LastDataApex = lastDataApex; + } + + public byte[] Data { get; } + + public ulong LastDataApex { get; } + } +} diff --git a/Centaurus.Domain/Quanta/Sync/SyncQuantaDataWorker.cs b/Centaurus.Domain/Quanta/Sync/SyncQuantaDataWorker.cs new file mode 100644 index 00000000..a4b22ca0 --- /dev/null +++ b/Centaurus.Domain/Quanta/Sync/SyncQuantaDataWorker.cs @@ -0,0 +1,215 @@ +using Centaurus.Domain; +using Centaurus.Domain.Quanta.Sync; +using Centaurus.Models; +using NLog; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace Centaurus.Domain +{ + public class SyncQuantaDataWorker : ContextualBase, IDisposable + { + static Logger logger = LogManager.GetCurrentClassLogger(); + + public SyncQuantaDataWorker(Domain.ExecutionContext context) + : base(context) + { + batchSize = Context.Settings.SyncBatchSize; + Task.Factory.StartNew(SyncQuantaData); + } + + private readonly int batchSize; + private CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + + + private bool IsAuditorReadyToHandleQuanta(IncomingAuditorConnection auditor) + { + return auditor.ConnectionState == ConnectionState.Ready //connection is validated + && (auditor.AuditorState.IsRunning || auditor.AuditorState.IsWaitingForInit); //auditor is ready to handle quanta + } + + private bool IsCurrentNodeReady => Context.StateManager.State != State.Rising + && Context.StateManager.State != State.Undefined + && Context.StateManager.State != State.Failed; + + private List GetCursors() + { + var auditorConnections = Context.IncomingConnectionManager.GetAuditorConnections(); + + var quantaCursors = new Dictionary(); + var signaturesCursors = new Dictionary(); + foreach (var connection in auditorConnections) + { + var (quantaCursor, signaturesCursor, timeToken) = connection.GetCursors(); + SetCursors(quantaCursors, connection, SyncCursorType.Quanta, quantaCursor, timeToken); + SetCursors(signaturesCursors, connection, SyncCursorType.Signatures, signaturesCursor, timeToken); + } + + return quantaCursors.Values.Union(signaturesCursors.Values).ToList(); + } + + private void SetCursors(Dictionary cursors, IncomingAuditorConnection connection, SyncCursorType cursorType, ulong? cursor, DateTime timeToken) + { + if (cursor.HasValue && cursor.Value < Context.QuantumHandler.CurrentApex) + { + var cursorToLookup = cursor.Value + 1; + var quantaCursor = cursorToLookup - cursorToLookup % SyncStorage.PortionSize; + if (!cursors.TryGetValue(cursorToLookup, out var currentCursorGroup)) + { + currentCursorGroup = new AuditorCursorGroup(cursorType, quantaCursor); + cursors.Add(cursorToLookup, currentCursorGroup); + } + currentCursorGroup.Auditors.Add(new AuditorCursorGroup.AuditorCursorData(connection, timeToken, cursor.Value)); + if (currentCursorGroup.LastUpdate == default || currentCursorGroup.LastUpdate > timeToken) + currentCursorGroup.LastUpdate = timeToken; + } + } + + const int ForceTimeOut = 100; + + private List> SendQuantaData(List cursorGroups) + { + var sendingQuantaTasks = new List>(); + foreach (var cursorGroup in cursorGroups) + { + var currentCursor = cursorGroup.Cursor; + if (currentCursor == Context.QuantumHandler.CurrentApex) + continue; + if (currentCursor > Context.QuantumHandler.CurrentApex && IsCurrentNodeReady) + { + var message = $"Auditors {string.Join(',', cursorGroup.Auditors.Select(a => a.Connection.PubKeyAddress))} is above current constellation state."; + if (cursorGroup.Auditors.Count >= Context.GetMajorityCount() - 1) //-1 is current server + logger.Error(message); + else + logger.Info(message); + continue; + } + + var force = (DateTime.UtcNow - cursorGroup.LastUpdate).TotalMilliseconds > ForceTimeOut; + + var batch = default(SyncPortion); + var cursorType = cursorGroup.CursorType; + switch (cursorType) + { + case SyncCursorType.Quanta: + batch = Context.SyncStorage.GetQuanta(currentCursor, force); + break; + case SyncCursorType.Signatures: + batch = Context.SyncStorage.GetSignatures(currentCursor, force); + break; + default: + throw new NotImplementedException($"{cursorType} cursor type is not supported."); + } + + if (batch == null) + continue; + + var lastBatchApex = batch.LastDataApex; + + foreach (var auditorConnection in cursorGroup.Auditors) + { + var currentAuditor = auditorConnection; + if (IsAuditorReadyToHandleQuanta(currentAuditor.Connection) && auditorConnection.Cursor < lastBatchApex) + sendingQuantaTasks.Add( + currentAuditor.Connection.SendMessage(batch.Data.AsMemory()) + .ContinueWith(t => + { + if (t.IsFaulted) + { + logger.Error(t.Exception, $"Unable to send quanta data to {currentAuditor.Connection.PubKeyAddress}. Cursor: {currentCursor}; CurrentApex: {Context.QuantumHandler.CurrentApex}"); + return null; + } + return new AuditorSyncCursorUpdate(currentAuditor.Connection, + new SyncCursorUpdate(currentAuditor.TimeToken, (ulong?)lastBatchApex, cursorType) + ); + }) + ); + } + } + return sendingQuantaTasks; + } + + private async Task SyncQuantaData() + { + var hasPendingQuanta = true; + while (!cancellationTokenSource.Token.IsCancellationRequested) + { + if (!hasPendingQuanta) + Thread.Sleep(50); + + if (!Context.IsAlpha || !IsCurrentNodeReady) + { + hasPendingQuanta = false; + continue; + } + + try + { + var cursors = GetCursors(); + + var sendingQuantaTasks = SendQuantaData(cursors); + + if (sendingQuantaTasks.Count > 0) + { + hasPendingQuanta = true; + + var cursorUpdates = await Task.WhenAll(sendingQuantaTasks); + var auditorUpdates = cursorUpdates.Where(u => u != null).GroupBy(u => u.Connection); + foreach (var connection in auditorUpdates) + connection.Key.SetSyncCursor(false, connection.Select(u => u.CursorUpdate).ToArray()); + } + } + catch (Exception exc) + { + if (exc is ObjectDisposedException + || exc.GetBaseException() is ObjectDisposedException) + throw; + logger.Error(exc, "Error on quanta data sync."); + } + } + } + + public void Dispose() + { + cancellationTokenSource.Cancel(); + cancellationTokenSource.Dispose(); + } + + class AuditorCursorGroup + { + public AuditorCursorGroup(SyncCursorType cursorType, ulong cursor) + { + CursorType = cursorType; + Cursor = cursor; + } + + public SyncCursorType CursorType { get; } + + public ulong Cursor { get; } + + public DateTime LastUpdate { get; set; } + + public List Auditors { get; } = new List(); + + public class AuditorCursorData + { + public AuditorCursorData(IncomingAuditorConnection connection, DateTime timeToken, ulong cursor) + { + Connection = connection; + TimeToken = timeToken; + Cursor = cursor; + } + + public IncomingAuditorConnection Connection { get; } + + public DateTime TimeToken { get; } + + public ulong Cursor { get; } + } + } + } +} \ No newline at end of file diff --git a/Centaurus.Domain/Quanta/SyncStorage.cs b/Centaurus.Domain/Quanta/Sync/SyncStorage.cs similarity index 50% rename from Centaurus.Domain/Quanta/SyncStorage.cs rename to Centaurus.Domain/Quanta/Sync/SyncStorage.cs index 16d8402e..1e545dd8 100644 --- a/Centaurus.Domain/Quanta/SyncStorage.cs +++ b/Centaurus.Domain/Quanta/Sync/SyncStorage.cs @@ -1,13 +1,11 @@ -using Centaurus.Models; +using Centaurus.Domain.Quanta.Sync; +using Centaurus.Models; using Centaurus.PersistentStorage; -using Centaurus.PersistentStorage.Abstraction; using Microsoft.Extensions.Caching.Memory; using NLog; using System; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; @@ -15,25 +13,52 @@ namespace Centaurus.Domain { public class SyncStorage : ContextualBase { - private MemoryCache quantaCache = new MemoryCache(new MemoryCacheOptions { ExpirationScanFrequency = TimeSpan.FromSeconds(5) }); - private MemoryCache signaturesCache = new MemoryCache(new MemoryCacheOptions { ExpirationScanFrequency = TimeSpan.FromSeconds(5) }); + private MemoryCache quantaDataCache = new MemoryCache(new MemoryCacheOptions { ExpirationScanFrequency = TimeSpan.FromSeconds(5) }); static Logger logger = LogManager.GetCurrentClassLogger(); public SyncStorage(ExecutionContext context, ulong lastApex) : base(context) { context.PendingUpdatesManager.OnBatchSaved += PendingUpdatesManager_OnBatchSaved; - //LastAddedQuantumApex = LastAddedSignaturesApex = lastApex; + var batchId = GetBatchApexStart(lastApex); //load current quanta batch - GetQuantaBatch(batchId); - //load current signatures batch - GetSignaturesBatch(batchId); + GetBatch(batchId); //run cache cleanup worker CacheCleanupWorker(); } - const int batchSize = 1_000_000; + public const int BatchSize = 1_000_000; + + public const int PortionSize = 500; + + /// + /// + /// + /// Exclusive + /// if force, than batch would be return even if it's not fulfilled yet + /// + public SyncPortion GetQuanta(ulong from, bool force) + { + var quantaBatchStart = GetBatchApexStart(from + 1); + var batch = GetBatch(quantaBatchStart); + + return batch.Quanta.GetData(from, force); + } + + /// + /// + /// + /// Exclusive + /// if force, than batch would be return even if it's not fulfilled yet + /// + public SyncPortion GetSignatures(ulong from, bool force) + { + var quantaBatchStart = GetBatchApexStart(from + 1); + var batch = GetBatch(quantaBatchStart); + + return batch.Signatures.GetData(from, force); + } /// /// @@ -44,9 +69,9 @@ public SyncStorage(ExecutionContext context, ulong lastApex) public List GetQuanta(ulong from, int limit) { var quantaBatchStart = GetBatchApexStart(from + 1); - var batch = GetQuantaBatch(quantaBatchStart); + var batch = GetBatch(quantaBatchStart); - return batch.GetItems(from, limit); + return batch.Quanta.GetItems(from, limit); } /// @@ -58,23 +83,23 @@ public List GetQuanta(ulong from, int limit) public List GetSignatures(ulong from, int limit) { var quantaBatchStart = GetBatchApexStart(from + 1); - var batch = GetSignaturesBatch(quantaBatchStart); + var batch = GetBatch(quantaBatchStart); - return batch.GetItems(from, limit); + return batch.Signatures.GetItems(from, limit); } public void AddQuantum(ulong apex, SyncQuantaBatchItem quantum) { var batchStart = GetBatchApexStart(apex); - var batch = GetQuantaBatch(batchStart); - batch.Add(apex, quantum); + var batch = GetBatch(batchStart); + batch.Quanta.Add(apex, quantum); } public void AddSignatures(ulong apex, QuantumSignatures signatures) { var batchStart = GetBatchApexStart(apex); - var batch = GetSignaturesBatch(batchStart); - batch.Add(apex, signatures); + var batch = GetBatch(batchStart); + batch.Signatures.Add(apex, signatures); } private object persistedBatchIdsSyncRoot = new { }; @@ -105,7 +130,7 @@ private void CacheCleanupWorker() if (persistedBatchIds.Count > 0) { var batchId = persistedBatchIds[0]; - if (MarkBatchAsFulfilled(signaturesCache, batchId) && MarkBatchAsFulfilled(quantaCache, batchId)) + if (MarkBatchAsFulfilled(batchId)) { persistedBatchIds.RemoveAt(0); hasItemToRemove = true; @@ -117,10 +142,9 @@ private void CacheCleanupWorker() }); } - private bool MarkBatchAsFulfilled(MemoryCache cache, ulong batchId) - where T : IApex + private bool MarkBatchAsFulfilled(ulong batchId) { - if (!cache.TryGetValue>(batchId, out var batch) || batch.LastApex != (batchId + batchSize - 1)) + if (!quantaDataCache.TryGetValue(batchId, out var batch) || !batch.IsFulfilled) { logger.Info("Batch is not found or not full yet"); return false; @@ -128,87 +152,54 @@ private bool MarkBatchAsFulfilled(MemoryCache cache, ulong batchId) var options = new MemoryCacheEntryOptions(); options.SetSlidingExpiration(TimeSpan.FromSeconds(15)); - options.RegisterPostEvictionCallback(OnPostEviction); + options.RegisterPostEvictionCallback(OnPostEviction); //replace old entry - cache.Set(batchId, batch, options); + quantaDataCache.Set(batchId, batch, options); return true; } private object quantaSyncRoot = new { }; - private ApexItemsBatch GetQuantaBatch(ulong batchId) + private SyncStorageItem GetBatch(ulong batchId) { - var batch = default(ApexItemsBatch); - if (!quantaCache.TryGetValue(batchId, out batch)) + var batch = default(SyncStorageItem); + if (!quantaDataCache.TryGetValue(batchId, out batch)) lock (quantaSyncRoot) { - if (!quantaCache.TryGetValue(batchId, out batch)) + if (!quantaDataCache.TryGetValue(batchId, out batch)) { var options = new MemoryCacheEntryOptions(); - var items = LoadQuantaFromDB(batchId); - if (batchId == 0) //insert null at 0 position, otherwise index will not be relevant to apex - items.Insert(0, null); - batch = new ApexItemsBatch(batchId, batchSize, items); - if (items.Count != batchSize) + batch = LoadBatch(batchId, BatchSize); + if (!batch.IsFulfilled) options.Priority = CacheItemPriority.NeverRemove; else options.SetSlidingExpiration(TimeSpan.FromSeconds(15)); - options.RegisterPostEvictionCallback(OnPostEviction); + options.RegisterPostEvictionCallback(OnPostEviction); - quantaCache.Set(batchId, batch, options); + quantaDataCache.Set(batchId, batch, options); } } return batch; } - object signaturesSyncRoot = new { }; - private ApexItemsBatch GetSignaturesBatch(ulong batchId) + private ulong GetBatchApexStart(ulong apex) { - var batch = default(ApexItemsBatch); - if (!signaturesCache.TryGetValue(batchId, out batch)) - lock (signaturesSyncRoot) - { - if (!signaturesCache.TryGetValue(batchId, out batch)) - { - var options = new MemoryCacheEntryOptions(); - var items = LoadSignaturesFromDB(batchId); - if (batchId == 0) //insert null at 0 position, otherwise index will not be relevant to apex - items.Insert(0, null); - batch = new ApexItemsBatch(batchId, batchSize, items); - if (items.Count != batchSize) - options.Priority = CacheItemPriority.NeverRemove; - else - options.SetSlidingExpiration(TimeSpan.FromSeconds(15)); - options.RegisterPostEvictionCallback(OnPostEviction); - - signaturesCache.Set(batchId, batch, options); - } - } - return batch; + return apex - (apex % BatchSize); } - private ulong GetBatchApexStart(ulong apex) + private void OnPostEviction(object key, object value, EvictionReason reason, object state) { - return apex - (apex % batchSize); + var batch = (SyncStorageItem)value; + logger.Info($"Batch {key} of quantum data with range from {batch.BatchStart} to {batch.BatchEnd} is removed because of {reason}. State: {state}."); } - private List LoadQuantaFromDB(ulong batchStartApex) + private void PendingUpdatesManager_OnBatchSaved(BatchSavedInfo batchSavedInfo) { - logger.Info($"About to load quanta batch from db. Start apex: {batchStartApex}, size: {batchSize}."); - var limit = batchSize; - var aboveApex = batchStartApex; - if (batchStartApex == 0) - //first quantum has 1 apex, and null will be set at 0 index. So we need to load batch size - 1 - limit = batchSize - 1; - else - aboveApex = batchStartApex - 1; - var quanta = Context.DataProvider.GetQuantaSyncBatchItemsAboveApex(aboveApex, limit); - logger.Info($"Quanta batch with apex start {batchStartApex} is loaded."); - return quanta; + QuantaRangePersisted(batchSavedInfo.FromApex, batchSavedInfo.ToApex); } - private List LoadSignaturesFromDB(ulong batchStartApex) + private SyncStorageItem LoadBatch(ulong batchStartApex, int batchSize) { logger.Info($"About to load quanta batch from db. Start apex: {batchStartApex}, size: {batchSize}."); var limit = batchSize; @@ -218,21 +209,49 @@ private List LoadSignaturesFromDB(ulong batchStartApex) limit = batchSize - 1; else aboveApex = batchStartApex - 1; - var quanta = Context.DataProvider.GetSignaturesSyncBatchItemsAboveApex(aboveApex, limit); - logger.Info($"Signatures batch with apex start {batchStartApex} is loaded."); - return quanta; - } - private void OnPostEviction(object key, object value, EvictionReason reason, object state) - where T : IApex - { - var batch = (ApexItemsBatch)value; - logger.Info($"Batch {key} of {typeof(T).Name} with range from {batch.Start} to {batch.LastApex} is removed because of {reason}. State: {state}."); + var rawQuanta = (IEnumerable)Context.PersistentStorage.LoadQuantaAboveApex(aboveApex, batchSize) + .OrderBy(q => q.Apex); + + logger.Info($"Quanta batch with apex start {batchStartApex} is loaded."); + + var quanta = new List(); + var signatures = new List(); + foreach (var rawQuantum in rawQuanta) + { + quanta.Add(rawQuantum.ToBatchItemQuantum()); + signatures.Add(rawQuantum.ToQuantumSignatures()); + } + + if (batchStartApex == 0) //insert null at 0 position, otherwise index will not be relevant to apex + { + quanta.Insert(0, null); + signatures.Insert(0, null); + } + + return new SyncStorageItem(Context, batchStartApex, quanta, signatures); } - private void PendingUpdatesManager_OnBatchSaved(BatchSavedInfo batchSavedInfo) + class SyncStorageItem: ContextualBase { - QuantaRangePersisted(batchSavedInfo.FromApex, batchSavedInfo.ToApex); + public SyncStorageItem(ExecutionContext context, ulong batchId, List quanta, List signatures) + :base(context) + { + BatchStart = batchId; + BatchEnd = batchId + BatchSize - 1; //batch start is inclusive + Quanta = new ApexItemsBatch(Context, batchId, BatchSize, PortionSize, quanta); + Signatures = new ApexItemsBatch(Context, batchId, BatchSize, PortionSize, signatures); + } + + public ApexItemsBatch Signatures { get; } + + public ApexItemsBatch Quanta { get; } + + public bool IsFulfilled => Quanta.IsFulfilled && Signatures.IsFulfilled; + + public ulong BatchStart { get; } + + public ulong BatchEnd { get; } } } } diff --git a/Centaurus.Domain/ResultManagers/ResultManager.cs b/Centaurus.Domain/ResultManagers/ResultManager.cs index 2751a6c7..7622d4b7 100644 --- a/Centaurus.Domain/ResultManagers/ResultManager.cs +++ b/Centaurus.Domain/ResultManagers/ResultManager.cs @@ -179,7 +179,7 @@ private void UpdateCache() { var currentBatchId = currentBatchIds.Last(); var nextBatchId = currentBatchId + batchSize; - if (nextBatchId - Context.QuantumHandler.CurrentApex < advanceThreshold) + if (nextBatchId - Context.QuantumHandler.LastAddedQuantumApex < advanceThreshold) CreateBatch(nextBatchId); //copy batch to be able to modify original var _currentBatchIds = currentBatchIds.ToList(); diff --git a/Centaurus.Domain/Statistics/PerformanceStatisticsManager.cs b/Centaurus.Domain/Statistics/PerformanceStatisticsManager.cs index 4bdd7d78..859e7ee7 100644 --- a/Centaurus.Domain/Statistics/PerformanceStatisticsManager.cs +++ b/Centaurus.Domain/Statistics/PerformanceStatisticsManager.cs @@ -87,7 +87,7 @@ List GetAuditorsStatistics() var accountId = auditorConnection?.PubKeyAddress; if (!auditorsStatistics.TryGetValue(accountId, out var statistics)) continue; - var auditorApex = auditorConnection.CurrentCursor; + var auditorApex = auditorConnection.CurrentQuantaCursor; if (auditorApex >= 0) statistics.Delay = (int)(Context.QuantumHandler.CurrentApex - auditorApex); } diff --git a/Centaurus.Domain/WebSockets/Centaurus/ConnectionBase.cs b/Centaurus.Domain/WebSockets/Centaurus/ConnectionBase.cs index 9ca6a793..64d33e9d 100644 --- a/Centaurus.Domain/WebSockets/Centaurus/ConnectionBase.cs +++ b/Centaurus.Domain/WebSockets/Centaurus/ConnectionBase.cs @@ -153,12 +153,10 @@ public virtual async Task SendMessage(MessageEnvelopeBase envelope) using (var writer = new XdrBufferWriter(outgoingBuffer.Buffer)) { XdrConverter.Serialize(envelope, writer); - if (webSocket == null) - throw new ObjectDisposedException(nameof(webSocket)); - if (webSocket.State == WebSocketState.Open) - await webSocket.SendAsync(outgoingBuffer.Buffer.AsMemory(0, writer.Length), WebSocketMessageType.Binary, true, cancellationToken); Context.ExtensionsManager.AfterSendMessage(this, envelope); + await SendRawMessageInternal(outgoingBuffer.Buffer.AsMemory(0, writer.Length)); + if (!(envelope.Message is AuditorPerfStatistics)) logger.Trace($"Connection {PubKeyAddress}, message {envelope.Message.GetMessageType()} sent. Size: {writer.Length}"); } @@ -184,6 +182,32 @@ public virtual async Task SendMessage(MessageEnvelopeBase envelope) } } + public async Task SendMessage(Memory message) + { + await sendMessageSemaphore.WaitAsync(); + try + { + await SendRawMessageInternal(message); + } + //TODO: log failed messages + catch (OperationCanceledException) { } + catch (WebSocketException exc) + { + if (!(exc.WebSocketErrorCode == WebSocketError.InvalidState && cancellationToken.IsCancellationRequested)) + throw; + } + finally + { + sendMessageSemaphore.Release(); + } + } + + private async Task SendRawMessageInternal(Memory message) + { + if (webSocket.State == WebSocketState.Open) + await webSocket.SendAsync(message, WebSocketMessageType.Binary, true, cancellationToken); + } + public async Task Listen() { try diff --git a/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingAuditorConnection.cs b/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingAuditorConnection.cs index c6d29335..e8982fcd 100644 --- a/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingAuditorConnection.cs +++ b/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingAuditorConnection.cs @@ -1,6 +1,7 @@ using Centaurus.Domain; -using Centaurus.Models; +using Centaurus.Domain.Quanta.Sync; using NLog; +using System; using System.Net.WebSockets; using static Centaurus.Domain.StateManager; @@ -14,7 +15,6 @@ public IncomingAuditorConnection(ExecutionContext context, KeyPair keyPair, WebS : base(context, keyPair, webSocket, ip) { AuditorState = Context.StateManager.GetAuditorState(keyPair); - quantumWorker = new QuantumSyncWorker(Context, this); SendHandshake(); } @@ -24,25 +24,52 @@ public IncomingAuditorConnection(ExecutionContext context, KeyPair keyPair, WebS protected override int outBufferSize => AuditorBufferSize; - private QuantumSyncWorker quantumWorker; + public AuditorState AuditorState { get; } - public ulong CurrentCursor => quantumWorker.CurrentQuantaCursor ?? 0; + private object syncRoot = new { }; - public AuditorState AuditorState { get; } + public ulong? CurrentQuantaCursor { get; private set; } + public ulong? CurrentSignaturesCursor { get; private set; } + public DateTime LastCursorUpdateDate { get; private set; } - public void SetSyncCursor(ulong? quantumCursor, ulong? signaturesCursor) + /// + /// + /// + /// Force means that auditor requested cursor reset + /// + public void SetSyncCursor(bool force, params SyncCursorUpdate[] cursorUpdates) { - logger.Trace($"Connection {PubKeyAddress}, apex cursor reset requested. New quantum cursor {(quantumCursor.HasValue ? quantumCursor.ToString() : "null")}, new result cursor {(signaturesCursor.HasValue ? signaturesCursor.ToString() : "null")}"); - - //set new quantum and result cursors - quantumWorker.SetCursors(quantumCursor, signaturesCursor); - logger.Trace($"Connection {PubKeyAddress}, cursors reseted."); + //set new quantum and signature cursors + lock (syncRoot) + { + if (cursorUpdates.Length == 0) + return; + foreach (var cursorUpdate in cursorUpdates) + { + if (!(force || cursorUpdate.TimeToken == LastCursorUpdateDate)) + continue; + switch (cursorUpdate.CursorType) + { + case SyncCursorType.Quanta: + CurrentQuantaCursor = cursorUpdate.NewCursor; + break; + case SyncCursorType.Signatures: + CurrentSignaturesCursor = cursorUpdate.NewCursor; + break; + default: + throw new NotImplementedException($"{cursorUpdate.CursorType} cursor type is not supported."); + } + } + LastCursorUpdateDate = DateTime.UtcNow; + if (force) + logger.Info($"Connection {PubKeyAddress}, cursors reset. Quanta cursor: {CurrentQuantaCursor}, signatures cursor: {CurrentSignaturesCursor}"); + } } - public override void Dispose() + public (ulong? quantaCursor, ulong? signaturesCursor, DateTime timeToken) GetCursors() { - base.Dispose(); - quantumWorker.Dispose(); + lock (syncRoot) + return (CurrentQuantaCursor, CurrentSignaturesCursor, LastCursorUpdateDate); } } } \ No newline at end of file diff --git a/Centaurus.Domain/WebSockets/Centaurus/Incoming/QuantumSyncWorker.cs b/Centaurus.Domain/WebSockets/Centaurus/Incoming/QuantumSyncWorker.cs deleted file mode 100644 index 88ac70a5..00000000 --- a/Centaurus.Domain/WebSockets/Centaurus/Incoming/QuantumSyncWorker.cs +++ /dev/null @@ -1,182 +0,0 @@ -using Centaurus.Domain; -using Centaurus.Models; -using NLog; -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; - -namespace Centaurus -{ - public class QuantumSyncWorker : ContextualBase, IDisposable - { - static Logger logger = LogManager.GetCurrentClassLogger(); - - public QuantumSyncWorker(Domain.ExecutionContext context, IncomingAuditorConnection auditor) - : base(context) - { - this.auditor = auditor ?? throw new ArgumentNullException(nameof(auditor)); - batchSize = Context.Settings.SyncBatchSize; - Task.Factory.StartNew(SyncQuantaData); - } - - private readonly IncomingAuditorConnection auditor; - private readonly int batchSize; - private CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); - - private object syncRoot = new { }; - - public ulong? CurrentQuantaCursor { get; private set; } - public ulong? CurrentSignaturesCursor { get; private set; } - - public void SetCursors(ulong? quantaCursor, ulong? signaturesCursor) - { - lock (syncRoot) - { - if (quantaCursor != null) - CurrentQuantaCursor = quantaCursor; - if (signaturesCursor != null) - CurrentSignaturesCursor = signaturesCursor; - } - } - - - private (ulong? quantaCursor, ulong? signaturesCursor) GetCursors() - { - lock (syncRoot) - return (CurrentQuantaCursor, CurrentSignaturesCursor); - } - - private bool IsAuditorReadyToHandleQuanta - { - get - { - return auditor.ConnectionState == ConnectionState.Ready //connection is validated - && (auditor.AuditorState.IsRunning || auditor.AuditorState.IsWaitingForInit); //auditor is ready to handle quanta - } - } - - private bool IsCurrentNodeReady => Context.StateManager.State != State.Rising - && Context.StateManager.State != State.Undefined - && Context.StateManager.State != State.Failed; - - private async Task SyncQuantaData() - { - var hasPendingQuanta = true; - while (!cancellationTokenSource.Token.IsCancellationRequested) - { - if (!hasPendingQuanta) - Thread.Sleep(50); - - var cursors = GetCursors(); - try - { - if (cursors.quantaCursor > Context.QuantumHandler.CurrentApex && IsCurrentNodeReady) - { - logger.Error($"Auditor {auditor.PubKey.GetAccountId()} is above current constellation state."); - await auditor.CloseConnection(System.Net.WebSockets.WebSocketCloseStatus.ProtocolError, "Auditor is above all constellation."); - return; - } - - var hasQuantaToSend = ((Context.QuantumHandler.CurrentApex - cursors.quantaCursor) ?? 0) > 0; - var hasSignaturesToSend = ((Context.PendingUpdatesManager.LastSavedApex - cursors.signaturesCursor) ?? 0) > 0; - - if (!Context.IsAlpha //only Alpha should broadcast quanta - || !IsAuditorReadyToHandleQuanta //auditor is not ready to handle quanta - || (!hasQuantaToSend && !hasSignaturesToSend) //nothing to sync - || !IsCurrentNodeReady) - { - hasPendingQuanta = false; - continue; - } - - var quantaSendResult = SendQuanta(hasQuantaToSend, cursors.quantaCursor.Value); - - var signatureSendResult = SendSignatures(hasSignaturesToSend, cursors.signaturesCursor.Value); - - await Task.WhenAll(quantaSendResult.sendTask, signatureSendResult.sendTask); - - lock (syncRoot) - { - //if quanta cursor is different, than auditor requested new cursor - if (quantaSendResult.lastQuantumApex > 0 && CurrentQuantaCursor == cursors.quantaCursor) - CurrentQuantaCursor = quantaSendResult.lastQuantumApex; - - if (signatureSendResult.lastSignaturesApex > 0 && CurrentSignaturesCursor.HasValue && CurrentSignaturesCursor == cursors.signaturesCursor) - CurrentSignaturesCursor = signatureSendResult.lastSignaturesApex; - } - } - catch (Exception exc) - { - if (exc is ObjectDisposedException - || exc.GetBaseException() is ObjectDisposedException) - throw; - logger.Error(exc, $"Unable to get quanta. Cursor: {cursors.quantaCursor}; CurrentApex: {Context.QuantumHandler.CurrentApex}"); - } - } - } - - private (Task sendTask, ulong lastQuantumApex) SendQuanta(bool hasQuantaToSend, ulong quantaCursor) - { - var lastQuantumApex = 0ul; - var quantaBatchSendTask = Task.CompletedTask; - if (hasQuantaToSend && GetPendingQuanta(quantaCursor, batchSize, out var quantaBatch)) - { - var firstQuantumApex = ((Quantum)quantaBatch.Quanta.First().Quantum).Apex; - lastQuantumApex = ((Quantum)quantaBatch.Quanta.Last().Quantum).Apex; - - logger.Trace($"About to sent {quantaBatch.Quanta.Count} quanta. Range from {firstQuantumApex} to {lastQuantumApex}."); - - quantaBatchSendTask = auditor.SendMessage(quantaBatch.CreateEnvelope()); - } - return (quantaBatchSendTask, lastQuantumApex); - } - - private (Task sendTask, ulong lastSignaturesApex) SendSignatures(bool hasSignaturesToSend, ulong signaturesCursor) - { - var lastSignaturesApex = 0ul; - var resultBatchSendTask = Task.CompletedTask; - if (hasSignaturesToSend && GetMajoritySignatures(signaturesCursor, 500, out var signaturesBatch)) - { - var firstSignaturesApex = signaturesBatch.Signatures.First().Apex; - lastSignaturesApex = signaturesBatch.Signatures.Last().Apex; - logger.Trace($"About to sent {signaturesBatch.Signatures.Count} signatures. Range from {firstSignaturesApex} to {lastSignaturesApex}."); - - resultBatchSendTask = auditor.SendMessage(signaturesBatch.CreateEnvelope()); - } - return (resultBatchSendTask, lastSignaturesApex); - } - - private bool GetPendingQuanta(ulong from, int count, out SyncQuantaBatch batch) - { - var quanta = Context.SyncStorage - .GetQuanta(from, count); - - batch = new SyncQuantaBatch - { - Quanta = quanta, - LastKnownApex = Context.QuantumHandler.CurrentApex - }; - return batch.Quanta.Count > 0; - } - - private bool GetMajoritySignatures(ulong from, int count, out Models.QuantumMajoritySignaturesBatch batch) - { - var signatures = Context.SyncStorage - .GetSignatures(from, count); - batch = new QuantumMajoritySignaturesBatch - { - Signatures = signatures - }; - return batch.Signatures.Count > 0; - } - - public void Dispose() - { - cancellationTokenSource.Cancel(); - cancellationTokenSource.Dispose(); - } - } -} \ No newline at end of file diff --git a/Centaurus.Models/InternalMessages/SyncCursor.cs b/Centaurus.Models/InternalMessages/SyncCursor.cs new file mode 100644 index 00000000..1afb0a3c --- /dev/null +++ b/Centaurus.Models/InternalMessages/SyncCursor.cs @@ -0,0 +1,30 @@ +using Centaurus.Xdr; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Centaurus.Models +{ + //TODO: convert ulong to ulong? after migration to MessagePack + [XdrContract] + public class SyncCursor + { + [XdrField(0)] + public ulong Cursor { get; set; } + + [XdrField(1)] + public XdrSyncCursorType Type { get; set; } + + /// + /// Set to false, to cancel sync. + /// + [XdrField(2)] + public bool ClearCursor { get; set; } + } + + public enum XdrSyncCursorType + { + Quanta = 0, + Signatures = 1 + } +} diff --git a/Centaurus.Models/InternalMessages/SyncCursorReset.cs b/Centaurus.Models/InternalMessages/SyncCursorReset.cs new file mode 100644 index 00000000..f9db18b1 --- /dev/null +++ b/Centaurus.Models/InternalMessages/SyncCursorReset.cs @@ -0,0 +1,14 @@ +using Centaurus.Xdr; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Centaurus.Models +{ + [XdrContract] + public class SyncCursorReset : Message + { + [XdrField(0)] + public List SyncCursors { get; set; } = new List(); + } +} \ No newline at end of file diff --git a/Centaurus.Models/InternalMessages/SyncCursorUpdate.cs b/Centaurus.Models/InternalMessages/SyncCursorUpdate.cs deleted file mode 100644 index 0482f42c..00000000 --- a/Centaurus.Models/InternalMessages/SyncCursorUpdate.cs +++ /dev/null @@ -1,18 +0,0 @@ -using Centaurus.Xdr; -using System; -using System.Collections.Generic; -using System.Text; - -namespace Centaurus.Models -{ - //TODO: convert ulong to ulong? after migration to MessagePack - [XdrContract] - public class SyncCursorUpdate : Message - { - [XdrField(0)] - public ulong QuantaCursor { get; set; } = ulong.MaxValue; - - [XdrField(1)] - public ulong SignaturesCursor { get; set; } = ulong.MaxValue; - } -} \ No newline at end of file diff --git a/Centaurus.Models/Messages/HandshakeResponse.cs b/Centaurus.Models/Messages/HandshakeResponse.cs index 1503f46f..13febbf2 100644 --- a/Centaurus.Models/Messages/HandshakeResponse.cs +++ b/Centaurus.Models/Messages/HandshakeResponse.cs @@ -1,4 +1,5 @@ using Centaurus.Xdr; +using System.Collections.Generic; namespace Centaurus.Models { @@ -18,12 +19,9 @@ public class AuditorHandshakeResponse : HandshakeResponseBase { [XdrField(0)] - public ulong QuantaCursor { get; set; } + public State State { get; set; } [XdrField(1)] - public ulong ResultCursor { get; set; } - - [XdrField(2)] - public State State { get; set; } + public List Cursors { get; set; } } } diff --git a/Centaurus.Models/Messages/Message.cs b/Centaurus.Models/Messages/Message.cs index 7ae6349e..bde8e76c 100644 --- a/Centaurus.Models/Messages/Message.cs +++ b/Centaurus.Models/Messages/Message.cs @@ -25,6 +25,7 @@ namespace Centaurus.Models [XdrUnion((int)MessageTypes.AuditorSignaturesBatch, typeof(AuditorSignaturesBatch))] [XdrUnion((int)MessageTypes.ConstellationUpdate, typeof(ConstellationUpdate))] [XdrUnion((int)MessageTypes.ConstellationQuantum, typeof(ConstellationQuantum))] + [XdrUnion((int)MessageTypes.SyncCursorReset, typeof(SyncCursorReset))] [XdrUnion((int)MessageTypes.AlphaQuantaBatch, typeof(SyncQuantaBatch))] [XdrUnion((int)MessageTypes.QuantumMajoritySignaturesBatch, typeof(QuantumMajoritySignaturesBatch))] [XdrUnion((int)MessageTypes.CatchupQuantaBatchRequest, typeof(CatchupQuantaBatchRequest))] diff --git a/Centaurus.Models/Messages/MessageTypes.cs b/Centaurus.Models/Messages/MessageTypes.cs index c32bfccc..fbbed0a7 100644 --- a/Centaurus.Models/Messages/MessageTypes.cs +++ b/Centaurus.Models/Messages/MessageTypes.cs @@ -114,7 +114,7 @@ public enum MessageTypes /// /// Contains cursors for quanta and signatures /// - SyncCursorUpdate = 213, + SyncCursorReset = 213, /// /// Contains batch of quanta, that were processed by Alpha /// diff --git a/Centaurus.NetSDK/Connection.cs b/Centaurus.NetSDK/Connection.cs index 956bf3e4..b26dd92e 100644 --- a/Centaurus.NetSDK/Connection.cs +++ b/Centaurus.NetSDK/Connection.cs @@ -39,7 +39,7 @@ public Connection(CentaurusClient client) static Logger logger = LogManager.GetCurrentClassLogger(); - private const int RequestTimeout = 5000; + private const int RequestTimeout = 15000; public event Action OnException; diff --git a/Centaurus.Test.Domain/QuantumStorageTests.cs b/Centaurus.Test.Domain/QuantumStorageTests.cs index 8e1032f1..1440cae3 100644 --- a/Centaurus.Test.Domain/QuantumStorageTests.cs +++ b/Centaurus.Test.Domain/QuantumStorageTests.cs @@ -45,17 +45,29 @@ public void QuantumStorageTest() Timestamp = DateTime.UtcNow.Ticks }; - context.SyncStorage.AddQuantum(quantum.Apex, new SyncQuantaBatchItem { Quantum = quantum }); + context.SyncStorage.AddQuantum(quantum.Apex, new SyncQuantaBatchItem + { + Quantum = quantum, + AlphaSignature = new AuditorSignatureInternal + { + AuditorId = 0, + PayloadSignature = new TinySignature + { + Data = new byte[64] + } + } + }); items.Add(new QuantumPersistentModel { Apex = quantum.Apex, RawQuantum = XdrConverter.Serialize(quantum), Signatures = new List(), TimeStamp = quantum.Timestamp, Effects = new List() }); } context.PersistentStorage.SaveBatch(items); - var random = new Random(); for (var i = 0ul; i < context.QuantumHandler.CurrentApex;) { - var quanta = context.SyncStorage.GetQuanta(i, random.Next(200, 500)); - foreach (var q in quanta) + var quantaData = context.SyncStorage.GetQuanta(i, SyncStorage.PortionSize > context.QuantumHandler.CurrentApex); + var envelope = XdrConverter.Deserialize(quantaData.Data); + var quanta = (SyncQuantaBatch)envelope.Message; + foreach (var q in quanta.Quanta) { Assert.AreEqual(((Quantum)q.Quantum).Apex, ++i); } From 67d0e12f3e6517d7c72d159836257fd1453dff6c Mon Sep 17 00:00:00 2001 From: hawthorne-abendsen <49230725+hawthorne-abendsen@users.noreply.github.com> Date: Thu, 18 Nov 2021 14:15:55 +0200 Subject: [PATCH 02/13] Tests update --- Centaurus.Domain/Contexts/ExecutionContext.cs | 1 - Centaurus.Domain/Quanta/Handlers/QuantumHandler.cs | 7 +------ Centaurus.Test.Domain/AlphaMessageHandlersTests.cs | 6 +++--- Centaurus.Test.Domain/BaseQuantumHandlerTests.cs | 4 ++-- Centaurus.Test.Domain/LoadEffectsTests.cs | 4 ++-- Centaurus.Test.Domain/PendingQuantaPersistentTest.cs | 4 ++-- Centaurus.Test.Domain/QuantumHandlerPerformanceTest.cs | 4 ++-- Centaurus.Test.Domain/QuantumStorageTests.cs | 4 ++-- 8 files changed, 14 insertions(+), 20 deletions(-) diff --git a/Centaurus.Domain/Contexts/ExecutionContext.cs b/Centaurus.Domain/Contexts/ExecutionContext.cs index 6d7f3e29..422c1848 100644 --- a/Centaurus.Domain/Contexts/ExecutionContext.cs +++ b/Centaurus.Domain/Contexts/ExecutionContext.cs @@ -294,7 +294,6 @@ public void Dispose() ExtensionsManager?.Dispose(); PaymentProvidersManager?.Dispose(); - QuantumHandler.Dispose(); ResultManager.Dispose(); DisposeAnalyticsManager(); PerformanceStatisticsManager?.Dispose(); diff --git a/Centaurus.Domain/Quanta/Handlers/QuantumHandler.cs b/Centaurus.Domain/Quanta/Handlers/QuantumHandler.cs index 62a4241c..117076c0 100644 --- a/Centaurus.Domain/Quanta/Handlers/QuantumHandler.cs +++ b/Centaurus.Domain/Quanta/Handlers/QuantumHandler.cs @@ -11,7 +11,7 @@ namespace Centaurus.Domain { - public class QuantumHandler : ContextualBase, IDisposable + public class QuantumHandler : ContextualBase { public QuantumHandler(ExecutionContext context, ulong lastApex, byte[] lastQuantumHash) : base(context) @@ -231,11 +231,6 @@ string GetMessageType(Quantum quantum) return quantum.GetType().Name; } - public void Dispose() - { - //awaitedQuanta.Dispose(); - } - class QuantumValidationException : Exception { public QuantumValidationException(Exception innerException) diff --git a/Centaurus.Test.Domain/AlphaMessageHandlersTests.cs b/Centaurus.Test.Domain/AlphaMessageHandlersTests.cs index 3b4493d9..21da0dda 100644 --- a/Centaurus.Test.Domain/AlphaMessageHandlersTests.cs +++ b/Centaurus.Test.Domain/AlphaMessageHandlersTests.cs @@ -13,10 +13,10 @@ namespace Centaurus.Test public class AlphaMessageHandlersTests : BaseMessageHandlerTests { [OneTimeSetUp] - public void Setup() + public async Task Setup() { EnvironmentHelper.SetTestEnvironmentVariable(); - context = GlobalInitHelper.DefaultAlphaSetup().Result; + context = await GlobalInitHelper.DefaultAlphaSetup(); } [OneTimeTearDown] @@ -84,7 +84,7 @@ public async Task AuditorHandshakeTest(KeyPair clientKeyPair, State alphaState, .GetField("handshakeData", BindingFlags.Instance | BindingFlags.NonPublic) .GetValue(connection); - var message = new AuditorHandshakeResponse { HandshakeData = handshakeData }; + var message = new AuditorHandshakeResponse { HandshakeData = handshakeData, Cursors = new List(), State = State.Running }; var envelope = message.CreateEnvelope(); envelope.Sign(clientKeyPair); diff --git a/Centaurus.Test.Domain/BaseQuantumHandlerTests.cs b/Centaurus.Test.Domain/BaseQuantumHandlerTests.cs index f9e2054f..cfb7a73e 100644 --- a/Centaurus.Test.Domain/BaseQuantumHandlerTests.cs +++ b/Centaurus.Test.Domain/BaseQuantumHandlerTests.cs @@ -15,10 +15,10 @@ namespace Centaurus.Test public abstract class BaseQuantumHandlerTests { [OneTimeSetUp] - public void Setup() + public async Task Setup() { EnvironmentHelper.SetTestEnvironmentVariable(); - context = GlobalInitHelper.DefaultAlphaSetup().Result; + context = await GlobalInitHelper.DefaultAlphaSetup(); } [OneTimeTearDown] diff --git a/Centaurus.Test.Domain/LoadEffectsTests.cs b/Centaurus.Test.Domain/LoadEffectsTests.cs index 6cc7b5b2..b3a5cad0 100644 --- a/Centaurus.Test.Domain/LoadEffectsTests.cs +++ b/Centaurus.Test.Domain/LoadEffectsTests.cs @@ -9,10 +9,10 @@ namespace Centaurus.Test public class LoadEffectsTests { [SetUp] - public void Setup() + public async Task Setup() { EnvironmentHelper.SetTestEnvironmentVariable(); - context = GlobalInitHelper.DefaultAlphaSetup().Result; + context = await GlobalInitHelper.DefaultAlphaSetup(); } static object[] EffectsLoadTestCases = new object[] diff --git a/Centaurus.Test.Domain/PendingQuantaPersistentTest.cs b/Centaurus.Test.Domain/PendingQuantaPersistentTest.cs index 31f05750..03920af5 100644 --- a/Centaurus.Test.Domain/PendingQuantaPersistentTest.cs +++ b/Centaurus.Test.Domain/PendingQuantaPersistentTest.cs @@ -16,11 +16,11 @@ public class PendingQuantaPersistentTest private ExecutionContext context; [SetUp] - public void Setup() + public async Task Setup() { settings = GlobalInitHelper.GetAlphaSettings(); storage = new MockStorage(); - context = GlobalInitHelper.DefaultAlphaSetup(storage, settings).Result; + context = await GlobalInitHelper.DefaultAlphaSetup(storage, settings); } [TearDown] diff --git a/Centaurus.Test.Domain/QuantumHandlerPerformanceTest.cs b/Centaurus.Test.Domain/QuantumHandlerPerformanceTest.cs index 357a9342..13f8f3e8 100644 --- a/Centaurus.Test.Domain/QuantumHandlerPerformanceTest.cs +++ b/Centaurus.Test.Domain/QuantumHandlerPerformanceTest.cs @@ -15,10 +15,10 @@ public class QuantumHandlerPerformanceTest : BaseMessageHandlerTests private ExecutionContext context; [SetUp] - public void Setup() + public async Task Setup() { EnvironmentHelper.SetTestEnvironmentVariable(); - context = GlobalInitHelper.DefaultAlphaSetup().Result; + context = await GlobalInitHelper.DefaultAlphaSetup(); } diff --git a/Centaurus.Test.Domain/QuantumStorageTests.cs b/Centaurus.Test.Domain/QuantumStorageTests.cs index 1440cae3..6d0e9b25 100644 --- a/Centaurus.Test.Domain/QuantumStorageTests.cs +++ b/Centaurus.Test.Domain/QuantumStorageTests.cs @@ -17,10 +17,10 @@ public class QuantumStorageTests private ExecutionContext context; [SetUp] - public void Setup() + public async Task Setup() { EnvironmentHelper.SetTestEnvironmentVariable(); - context = GlobalInitHelper.DefaultAlphaSetup().Result; + context = await GlobalInitHelper.DefaultAlphaSetup(); } [Test] From 25b97419f25023e52f5086fb082b11e65d32b2ae Mon Sep 17 00:00:00 2001 From: hawthorne-abendsen <49230725+hawthorne-abendsen@users.noreply.github.com> Date: Fri, 19 Nov 2021 17:45:50 +0200 Subject: [PATCH 03/13] Client message proxying to Alpha --- Centaurus.Domain/Contexts/ExecutionContext.cs | 4 +- .../Helpers/RequestQuantumHelper.cs | 31 +++++++ .../Quanta/AccountDataRequestHandler.cs | 5 - .../Quanta/QuantumHandlerBase.cs | 9 +- .../Quanta/WithdrawalMessageHandler.cs | 5 - .../RequestQuantaBatchHandler.cs | 37 ++++++++ Centaurus.Domain/Quanta/Sync/ProxyWorker.cs | 91 +++++++++++++++++++ .../Quanta/Sync/SyncQuantaDataWorker.cs | 12 +-- Centaurus.Domain/Quanta/Sync/SyncStorage.cs | 12 ++- .../StateManagers/StateManager.cs | 2 +- .../Outgoing/OutgoingConnectionManager.cs | 7 ++ .../InternalMessages/RequestQuantaBatch.cs | 13 +++ Centaurus.Models/Messages/Message.cs | 3 +- Centaurus.Models/Messages/MessageTypes.cs | 10 +- .../PersistentStoragePerfTests.cs | 1 - Centaurus.Test.Domain/QuantumStorageTests.cs | 2 +- Centaurus.Test.Utils/GlobalInitHelper.cs | 1 + 17 files changed, 207 insertions(+), 38 deletions(-) create mode 100644 Centaurus.Domain/Helpers/RequestQuantumHelper.cs create mode 100644 Centaurus.Domain/MessageHandlers/RequestQuantaBatchHandler.cs create mode 100644 Centaurus.Domain/Quanta/Sync/ProxyWorker.cs create mode 100644 Centaurus.Models/InternalMessages/RequestQuantaBatch.cs diff --git a/Centaurus.Domain/Contexts/ExecutionContext.cs b/Centaurus.Domain/Contexts/ExecutionContext.cs index 422c1848..a959c80b 100644 --- a/Centaurus.Domain/Contexts/ExecutionContext.cs +++ b/Centaurus.Domain/Contexts/ExecutionContext.cs @@ -57,6 +57,8 @@ public ExecutionContext(Settings settings, IPersistentStorage storage, PaymentPr SyncQuantaDataWorker = new SyncQuantaDataWorker(this); + ProxyWorker = new ProxyWorker(this); + Catchup = new Catchup(this); StateManager = new StateManager(this); @@ -348,7 +350,7 @@ private void PendingUpdatesManager_OnBatchSaved(BatchSavedInfo batchInfo) public InfoConnectionManager InfoConnectionManager { get; } public SyncQuantaDataWorker SyncQuantaDataWorker { get; } - + public ProxyWorker ProxyWorker { get; } public Catchup Catchup { get; } public InfoCommandsHandlers InfoCommandsHandlers { get; } diff --git a/Centaurus.Domain/Helpers/RequestQuantumHelper.cs b/Centaurus.Domain/Helpers/RequestQuantumHelper.cs new file mode 100644 index 00000000..ccdb59f8 --- /dev/null +++ b/Centaurus.Domain/Helpers/RequestQuantumHelper.cs @@ -0,0 +1,31 @@ +using Centaurus.Models; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Centaurus.Domain +{ + public static class RequestQuantumHelper + { + /// + /// Wraps client requests to corresponding quantum type + /// + /// + /// + public static RequestQuantumBase GetQuantum(MessageEnvelopeBase envelope) + { + switch(envelope.Message) + { + case AccountDataRequest _: + return new AccountDataRequestQuantum { RequestEnvelope = envelope }; + case WithdrawalRequest _: + return new WithdrawalRequestQuantum { RequestEnvelope = envelope }; + default: + if (envelope.Message is SequentialRequestMessage) + return new RequestQuantum { RequestEnvelope = envelope }; + else + throw new Exception($"{envelope.Message.GetMessageType()} is not request quantum message."); + } + } + } +} diff --git a/Centaurus.Domain/MessageHandlers/Quanta/AccountDataRequestHandler.cs b/Centaurus.Domain/MessageHandlers/Quanta/AccountDataRequestHandler.cs index 6d3b6929..792122b9 100644 --- a/Centaurus.Domain/MessageHandlers/Quanta/AccountDataRequestHandler.cs +++ b/Centaurus.Domain/MessageHandlers/Quanta/AccountDataRequestHandler.cs @@ -14,10 +14,5 @@ public AccountDataRequestHandler(ExecutionContext context) } public override string SupportedMessageType => typeof(AccountDataRequest).Name; - - protected override Quantum GetQuantum(ConnectionBase connection, IncomingMessage message) - { - return new AccountDataRequestQuantum { RequestEnvelope = message.Envelope }; - } } } \ No newline at end of file diff --git a/Centaurus.Domain/MessageHandlers/Quanta/QuantumHandlerBase.cs b/Centaurus.Domain/MessageHandlers/Quanta/QuantumHandlerBase.cs index 9d47fde0..4794160c 100644 --- a/Centaurus.Domain/MessageHandlers/Quanta/QuantumHandlerBase.cs +++ b/Centaurus.Domain/MessageHandlers/Quanta/QuantumHandlerBase.cs @@ -23,18 +23,13 @@ public override Task HandleMessage(ConnectionBase connection, IncomingMessage me { if (Context.IsAlpha) { - Context.QuantumHandler.HandleAsync(GetQuantum(connection, message), Task.FromResult(true)); + Context.QuantumHandler.HandleAsync(RequestQuantumHelper.GetQuantum(message.Envelope), Task.FromResult(true)); } else { - //TODO: send it to Alpha + Context.ProxyWorker.AddRequestsToQueue(message.Envelope); } return Task.CompletedTask; } - - protected virtual Quantum GetQuantum(ConnectionBase connection, IncomingMessage message) - { - return new RequestQuantum { RequestEnvelope = message.Envelope }; - } } } diff --git a/Centaurus.Domain/MessageHandlers/Quanta/WithdrawalMessageHandler.cs b/Centaurus.Domain/MessageHandlers/Quanta/WithdrawalMessageHandler.cs index 72eead37..b0a4ca0d 100644 --- a/Centaurus.Domain/MessageHandlers/Quanta/WithdrawalMessageHandler.cs +++ b/Centaurus.Domain/MessageHandlers/Quanta/WithdrawalMessageHandler.cs @@ -14,10 +14,5 @@ public WithdrawalMessageHandler(ExecutionContext context) } public override string SupportedMessageType { get; } = typeof(WithdrawalRequest).Name; - - protected override Quantum GetQuantum(ConnectionBase connection, IncomingMessage message) - { - return new WithdrawalRequestQuantum { RequestEnvelope = message.Envelope }; - } } } \ No newline at end of file diff --git a/Centaurus.Domain/MessageHandlers/RequestQuantaBatchHandler.cs b/Centaurus.Domain/MessageHandlers/RequestQuantaBatchHandler.cs new file mode 100644 index 00000000..732ac59e --- /dev/null +++ b/Centaurus.Domain/MessageHandlers/RequestQuantaBatchHandler.cs @@ -0,0 +1,37 @@ +using Centaurus.Models; +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; + +namespace Centaurus.Domain +{ + public class RequestQuantaBatchHandler : MessageHandlerBase + { + public RequestQuantaBatchHandler(ExecutionContext context) + : base(context) + { + + } + + public override string SupportedMessageType { get; } = typeof(RequestQuantaBatch).Name; + + public override Task HandleMessage(IncomingAuditorConnection connection, IncomingMessage message) + { + var requests = (RequestQuantaBatch)message.Envelope.Message; + if (Context.IsAlpha) + { + foreach (var request in requests.Requests) + { + var requestQuantum = RequestQuantumHelper.GetQuantum(request); + Context.QuantumHandler.HandleAsync(requestQuantum, QuantumSignatureValidator.Validate(requestQuantum)); + } + } + else + { + Context.ProxyWorker.AddRequestsToQueue(requests.Requests.ToArray()); + } + return Task.CompletedTask; + } + } +} \ No newline at end of file diff --git a/Centaurus.Domain/Quanta/Sync/ProxyWorker.cs b/Centaurus.Domain/Quanta/Sync/ProxyWorker.cs new file mode 100644 index 00000000..a11f158c --- /dev/null +++ b/Centaurus.Domain/Quanta/Sync/ProxyWorker.cs @@ -0,0 +1,91 @@ +using Centaurus.Models; +using NLog; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Centaurus.Domain +{ + public class ProxyWorker : ContextualBase, IDisposable + { + static Logger logger = LogManager.GetCurrentClassLogger(); + public ProxyWorker(ExecutionContext context) + : base(context) + { + cancellationTokenSource = new CancellationTokenSource(); + Run(); + } + + private void Run() + { + Task.Factory.StartNew(async () => + { + var hasQuantaToSend = false; + while (!cancellationTokenSource.IsCancellationRequested) + { + var requestsToSend = default(List); + try + { + if (!hasQuantaToSend) + { + Thread.Sleep(20); + } + lock (quantaCollectionSyncRoot) + { + if (requests.Count > 0) + { + requestsToSend = requests + .Take(500) + .ToList(); + requests.RemoveRange(0, requestsToSend.Count); + } + } + hasQuantaToSend = requestsToSend != null; + if (hasQuantaToSend) + { + await Context.OutgoingConnectionManager.SendTo(Context.Constellation.Alpha, new RequestQuantaBatch { Requests = requestsToSend }.CreateEnvelope()); + } + } + catch (Exception exc) + { + logger.Error(exc, "Error on sending quanta requests to Alpha."); + if (requestsToSend != null) + foreach (var request in requestsToSend) + { + var requestMessage = (RequestMessage)request.Message; + Notifier.Notify(Context, requestMessage.Account, request.CreateResult(ResultStatusCode.InvalidState).CreateEnvelope()); + } + } + } + }); + } + + public void SetAuditorConnection(OutgoingConnection alpha) + { + AlphaConnection = alpha ?? throw new ArgumentNullException(nameof(alpha)); + } + + private object quantaCollectionSyncRoot = new { }; + private List requests = new List(); + private CancellationTokenSource cancellationTokenSource; + + public OutgoingConnection AlphaConnection { get; private set; } + + public void AddRequestsToQueue(params MessageEnvelopeBase[] clientRequests) + { + if (!Context.IsAlpha) + lock (quantaCollectionSyncRoot) + requests.AddRange(clientRequests); + //TODO: add to QuantumHandler + } + + public void Dispose() + { + cancellationTokenSource.Cancel(); + cancellationTokenSource.Dispose(); + } + } +} diff --git a/Centaurus.Domain/Quanta/Sync/SyncQuantaDataWorker.cs b/Centaurus.Domain/Quanta/Sync/SyncQuantaDataWorker.cs index a4b22ca0..b0abd3f9 100644 --- a/Centaurus.Domain/Quanta/Sync/SyncQuantaDataWorker.cs +++ b/Centaurus.Domain/Quanta/Sync/SyncQuantaDataWorker.cs @@ -1,10 +1,8 @@ -using Centaurus.Domain; -using Centaurus.Domain.Quanta.Sync; +using Centaurus.Domain.Quanta.Sync; using Centaurus.Models; using NLog; using System; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -15,14 +13,12 @@ public class SyncQuantaDataWorker : ContextualBase, IDisposable { static Logger logger = LogManager.GetCurrentClassLogger(); - public SyncQuantaDataWorker(Domain.ExecutionContext context) + public SyncQuantaDataWorker(ExecutionContext context) : base(context) { - batchSize = Context.Settings.SyncBatchSize; Task.Factory.StartNew(SyncQuantaData); } - private readonly int batchSize; private CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); @@ -57,7 +53,7 @@ private void SetCursors(Dictionary cursors, IncomingA if (cursor.HasValue && cursor.Value < Context.QuantumHandler.CurrentApex) { var cursorToLookup = cursor.Value + 1; - var quantaCursor = cursorToLookup - cursorToLookup % SyncStorage.PortionSize; + var quantaCursor = cursorToLookup - cursorToLookup % (ulong)Context.SyncStorage.PortionSize; if (!cursors.TryGetValue(cursorToLookup, out var currentCursorGroup)) { currentCursorGroup = new AuditorCursorGroup(cursorType, quantaCursor); @@ -125,7 +121,7 @@ private List> SendQuantaData(List /// @@ -229,18 +231,18 @@ private SyncStorageItem LoadBatch(ulong batchStartApex, int batchSize) signatures.Insert(0, null); } - return new SyncStorageItem(Context, batchStartApex, quanta, signatures); + return new SyncStorageItem(Context, batchStartApex, PortionSize, quanta, signatures); } class SyncStorageItem: ContextualBase { - public SyncStorageItem(ExecutionContext context, ulong batchId, List quanta, List signatures) + public SyncStorageItem(ExecutionContext context, ulong batchId, int portionSize, List quanta, List signatures) :base(context) { BatchStart = batchId; BatchEnd = batchId + BatchSize - 1; //batch start is inclusive - Quanta = new ApexItemsBatch(Context, batchId, BatchSize, PortionSize, quanta); - Signatures = new ApexItemsBatch(Context, batchId, BatchSize, PortionSize, signatures); + Quanta = new ApexItemsBatch(Context, batchId, BatchSize, portionSize, quanta); + Signatures = new ApexItemsBatch(Context, batchId, BatchSize, portionSize, signatures); } public ApexItemsBatch Signatures { get; } diff --git a/Centaurus.Domain/StateManagers/StateManager.cs b/Centaurus.Domain/StateManagers/StateManager.cs index b1157ea1..321fe3f9 100644 --- a/Centaurus.Domain/StateManagers/StateManager.cs +++ b/Centaurus.Domain/StateManagers/StateManager.cs @@ -226,7 +226,7 @@ private bool IsConstellationReady() { if (!(auditorState.Value.CurrentState == State.Ready || auditorState.Value.CurrentState == State.Running)) continue; - if (Context.Constellation.Alpha == auditorState.Key) + if (Context.Constellation.Alpha.Equals(auditorState.Key)) isAlphaReady = true; connectedCount++; } diff --git a/Centaurus.Domain/WebSockets/Centaurus/Outgoing/OutgoingConnectionManager.cs b/Centaurus.Domain/WebSockets/Centaurus/Outgoing/OutgoingConnectionManager.cs index 91c60faf..5fc3d7c5 100644 --- a/Centaurus.Domain/WebSockets/Centaurus/Outgoing/OutgoingConnectionManager.cs +++ b/Centaurus.Domain/WebSockets/Centaurus/Outgoing/OutgoingConnectionManager.cs @@ -63,6 +63,13 @@ public async Task SendToAll(MessageEnvelopeBase message) await Task.WhenAll(sendTasks); } + public async Task SendTo(RawPubKey rawPubKey, MessageEnvelopeBase message) + { + if (!connections.TryGetValue(rawPubKey, out var connection)) + throw new Exception($"Connection {rawPubKey} is not found."); + await connection.SendMessage(message); + } + public void CloseAllConnections() { foreach (var connection in connections.Values) diff --git a/Centaurus.Models/InternalMessages/RequestQuantaBatch.cs b/Centaurus.Models/InternalMessages/RequestQuantaBatch.cs new file mode 100644 index 00000000..121a733b --- /dev/null +++ b/Centaurus.Models/InternalMessages/RequestQuantaBatch.cs @@ -0,0 +1,13 @@ +using Centaurus.Xdr; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Centaurus.Models +{ + public class RequestQuantaBatch : Message + { + [XdrField(0)] + public List Requests { get; set;} + } +} diff --git a/Centaurus.Models/Messages/Message.cs b/Centaurus.Models/Messages/Message.cs index bde8e76c..b4ef6609 100644 --- a/Centaurus.Models/Messages/Message.cs +++ b/Centaurus.Models/Messages/Message.cs @@ -26,7 +26,8 @@ namespace Centaurus.Models [XdrUnion((int)MessageTypes.ConstellationUpdate, typeof(ConstellationUpdate))] [XdrUnion((int)MessageTypes.ConstellationQuantum, typeof(ConstellationQuantum))] [XdrUnion((int)MessageTypes.SyncCursorReset, typeof(SyncCursorReset))] - [XdrUnion((int)MessageTypes.AlphaQuantaBatch, typeof(SyncQuantaBatch))] + [XdrUnion((int)MessageTypes.SyncQuantaBatch, typeof(SyncQuantaBatch))] + [XdrUnion((int)MessageTypes.RequestQuantaBatch, typeof(RequestQuantaBatch))] [XdrUnion((int)MessageTypes.QuantumMajoritySignaturesBatch, typeof(QuantumMajoritySignaturesBatch))] [XdrUnion((int)MessageTypes.CatchupQuantaBatchRequest, typeof(CatchupQuantaBatchRequest))] [XdrUnion((int)MessageTypes.CatchupQuantaBatch, typeof(CatchupQuantaBatch))] diff --git a/Centaurus.Models/Messages/MessageTypes.cs b/Centaurus.Models/Messages/MessageTypes.cs index fbbed0a7..d5c86b53 100644 --- a/Centaurus.Models/Messages/MessageTypes.cs +++ b/Centaurus.Models/Messages/MessageTypes.cs @@ -118,7 +118,7 @@ public enum MessageTypes /// /// Contains batch of quanta, that were processed by Alpha /// - AlphaQuantaBatch = 214, + SyncQuantaBatch = 214, /// /// Contains majority of signatures for quanta (except Alpha signatures, it will be send with quantum) /// @@ -126,6 +126,10 @@ public enum MessageTypes /// /// Contains an array of the current auditor's quantum signatures /// - AuditorSignaturesBatch = 216 + AuditorSignaturesBatch = 216, + /// + /// Contains an array of the client requests that must be resent to Alpha + /// + RequestQuantaBatch = 220 } -} +} \ No newline at end of file diff --git a/Centaurus.PersistentStorage.Test/PersistentStoragePerfTests.cs b/Centaurus.PersistentStorage.Test/PersistentStoragePerfTests.cs index 19144d67..77e3cc34 100644 --- a/Centaurus.PersistentStorage.Test/PersistentStoragePerfTests.cs +++ b/Centaurus.PersistentStorage.Test/PersistentStoragePerfTests.cs @@ -66,7 +66,6 @@ public void QuantaLoadTest() Assert.AreEqual(2, aboveApexQuanta.Count); aboveApexQuanta = storage.Find(QueryOrder.Desc).From(UlongConverter.Encode(150)).ToList(); Assert.AreEqual(3, aboveApexQuanta.Count); - Assert.AreEqual(quantaBatch.Count - 1, aboveApexQuanta.Count()); } } diff --git a/Centaurus.Test.Domain/QuantumStorageTests.cs b/Centaurus.Test.Domain/QuantumStorageTests.cs index 6d0e9b25..34025429 100644 --- a/Centaurus.Test.Domain/QuantumStorageTests.cs +++ b/Centaurus.Test.Domain/QuantumStorageTests.cs @@ -64,7 +64,7 @@ public void QuantumStorageTest() for (var i = 0ul; i < context.QuantumHandler.CurrentApex;) { - var quantaData = context.SyncStorage.GetQuanta(i, SyncStorage.PortionSize > context.QuantumHandler.CurrentApex); + var quantaData = context.SyncStorage.GetQuanta(i, (i + (ulong)context.SyncStorage.PortionSize) > context.QuantumHandler.CurrentApex); var envelope = XdrConverter.Deserialize(quantaData.Data); var quanta = (SyncQuantaBatch)envelope.Message; foreach (var q in quanta.Quanta) diff --git a/Centaurus.Test.Utils/GlobalInitHelper.cs b/Centaurus.Test.Utils/GlobalInitHelper.cs index cf8af30b..987dad8f 100644 --- a/Centaurus.Test.Utils/GlobalInitHelper.cs +++ b/Centaurus.Test.Utils/GlobalInitHelper.cs @@ -47,6 +47,7 @@ private static void SetCommonSettings(Settings settings, string secret) settings.GenesisAuditors = new[] { TestEnvironment.AlphaKeyPair.AccountId, TestEnvironment.Auditor1KeyPair.AccountId }.Select(a => new Settings.Auditor($"{a}={a}.com")).ToList(); settings.Secret = secret; settings.ParticipationLevel = 1; + settings.SyncBatchSize = 500; } public static List GetPredefinedClients() From 34413674ec1d9139f259afdb60128b0eee0dfae7 Mon Sep 17 00:00:00 2001 From: hawthorne-abendsen <49230725+hawthorne-abendsen@users.noreply.github.com> Date: Wed, 8 Dec 2021 19:46:50 +0200 Subject: [PATCH 04/13] Nodes management refactoring --- Centaurus.Common/Helpers/UriHelper.cs | 2 +- .../PublicInfo}/ConstellationInfo.cs | 6 +- Centaurus.Domain.Models/Snapshot.cs | 2 +- Centaurus.Domain/Catchups/Catchup.cs | 48 +-- Centaurus.Domain/Centaurus.Domain.csproj | 102 ++++--- .../Contexts/ExecutionContext.Analytics.cs | 2 +- Centaurus.Domain/Contexts/ExecutionContext.cs | 177 +++++------ Centaurus.Domain/DataProvider/DataProvider.cs | 10 +- ...PendingQuantumPersistentModelExtensions.cs | 30 +- .../SignatureModelExtensions.cs | 6 +- .../Helpers/ConstellationQuantumExtesnions.cs | 1 - .../ConstellationSettingsExtensions.cs | 2 +- .../Helpers/ExecutionContextExtensions.cs | 48 ++- .../Helpers/SyncCursorUpdateExtensions.cs | 31 +- .../AuditorStatisticsMessageHandler.cs | 32 -- .../CatchupQuantaBatchHandler.cs | 4 +- .../CatchupQuantaBatchRequestHandler.cs | 10 +- .../EffectsRequestMessageHandler.cs | 2 +- .../HandshakeRequestHandler.cs | 11 +- .../AuditorHandshakeResponseHandler.cs | 55 ---- .../HandshakeResponseHandler.cs | 50 +++- .../HandshakeResponseHandlerBase.cs | 30 -- .../MessageHandlers/MessageHandlerBase.cs | 4 +- .../Quanta/AccountDataRequestHandler.cs | 2 +- .../Quanta/OrderCancellationMessageHandler.cs | 2 +- .../Quanta/OrderMessageHandler.cs | 2 +- .../Quanta/PaymentMessageHandler.cs | 2 +- .../Quanta/QuantumHandlerBase.cs | 2 +- .../Quanta/WithdrawalMessageHandler.cs | 2 +- .../QuantumMajoritySignaturesBatchHandler.cs | 2 +- .../RequestQuantaBatchHandler.cs | 4 +- .../MessageHandlers/ResultBatchHandler.cs | 4 +- ...ssageHandler.cs => StateMessageHandler.cs} | 11 +- .../MessageHandlers/SyncCursorResetHandler.cs | 14 +- ...chHandler.cs => SyncQuantaBatchHandler.cs} | 11 +- Centaurus.Domain/Nodes/Common/NodeApexes.cs | 29 ++ Centaurus.Domain/Nodes/Common/NodeBase.cs | 48 +++ .../Nodes/Common/NodeExtensions.cs | 49 ++++ .../Nodes/Common/NodeQuantaQueueLengths.cs | 25 ++ .../Nodes/Common/StateChangedEventArgs.cs | 21 ++ .../Nodes/Common/SyncCursorCollection.cs | 42 +++ .../Nodes/Common/ValueAggregation.cs | 54 ++++ .../Nodes/CurrentNode/CurrentNode.cs | 128 ++++++++ .../Nodes/Managers/NodesManager.cs | 207 +++++++++++++ .../Nodes/Managers/StateNotifierWorker.cs | 60 ++++ .../Nodes/RemoteNode/RemoteNode.cs | 142 +++++++++ .../Nodes/RemoteNode/RemoteNodeConnection.cs | 117 ++++++++ .../Nodes/RemoteNode/RemoteNodeCursor.cs | 54 ++++ .../Nodes/RemoteNode/RemoteNodesCollection.cs | 75 +++++ .../Quanta/Handlers/QuantumHandler.cs | 15 +- .../Quanta/Processors/AlphaUpdateProcessor.cs | 58 ++++ .../ConstellationUpdateProcessor.cs | 7 +- .../Quanta/Sync/ApexItemsBatch.cs | 7 +- .../Quanta/Sync/ApexItemsBatchPortion.cs | 3 +- Centaurus.Domain/Quanta/Sync/CursorGroup.cs | 43 +++ Centaurus.Domain/Quanta/Sync/ProxyWorker.cs | 9 +- .../Quanta/Sync/SyncCursorUpdate.cs | 17 +- .../Quanta/Sync/SyncQuantaDataWorker.cs | 275 +++++++++--------- Centaurus.Domain/Quanta/Sync/SyncStorage.cs | 2 +- .../ResultManagers/ResultManager.cs | 32 +- .../StateManagers/StateManager.cs | 272 ----------------- .../Statistics/BatchSavedInfoExtensions.cs | 37 --- .../Statistics/PerformanceStatistics.cs | 46 +-- .../PerformanceStatisticsExtensions.cs | 38 --- .../PerformanceStatisticsManager.cs | 150 +++------- .../UpdatesManager/BatchSavedInfo.cs | 21 ++ .../UpdatesManager/UpdatesManager.cs | 22 +- .../WebSockets/Centaurus/ConnectionBase.cs | 12 +- .../Centaurus/IAuditorConnection.cs | 15 - .../WebSockets/Centaurus/INodeConnection.cs | 12 + .../Incoming/IncomingAuditorConnection.cs | 75 ----- .../Incoming/IncomingClientConnection.cs | 4 +- .../Incoming/IncomingConnectionBase.cs | 2 +- .../Incoming/IncomingConnectionManager.cs | 101 +++---- .../Incoming/IncomingConnectionsCollection.cs | 54 ++++ .../Incoming/IncomingNodeConnection.cs | 27 ++ .../WebSockets/Centaurus/Notifier.cs | 26 +- .../Centaurus/Outgoing/OutgoingConnection.cs | 17 +- .../Outgoing/OutgoingConnectionManager.cs | 220 -------------- .../PerformanceStatisticsUpdate.cs | 54 +--- Centaurus.Domain/exchange/OrderMatcher.cs | 2 +- .../Common/AuditorSignatureBase.cs | 4 +- Centaurus.Models/Common/State.cs | 15 +- .../InternalMessages/AlphaUpdate.cs | 11 +- .../InternalMessages/AlphaUpdateBase.cs | 10 + .../InternalMessages/CatchupQuantaBatch.cs | 2 +- .../InternalMessages/ConstellationUpdate.cs | 2 +- .../InternalMessages/PendingQuantum.cs | 14 - .../InternalMessages/PerformanceStatistics.cs | 41 --- .../QuantumMajoritySignaturesBatch.cs | 2 +- .../InternalMessages/StateMessage.cs | 15 +- .../InternalMessages/SyncCursor.cs | 2 +- .../InternalMessages/SyncCursorReset.cs | 2 +- .../InternalMessages/SyncQuantaBatch.cs | 5 +- .../Messages/HandshakeResponse.cs | 16 +- Centaurus.Models/Messages/Message.cs | 7 +- Centaurus.Models/Messages/MessageTypes.cs | 20 +- Centaurus.Models/Results/AuditorResult.cs | 2 +- .../AlphaMessageHandlersTests.cs | 68 +---- .../AuditorMessageHandlersTests.cs | 2 +- .../BaseMessageHandlerTests.cs | 11 +- Centaurus.Test.Domain/OrderbookTests.cs | 4 +- .../PendingQuantaPersistentTest.cs | 19 +- .../QuantumHandlerPerformanceTest.cs | 2 +- Centaurus.Test.Domain/QuantumStorageTests.cs | 2 +- .../UpdatesTest.cs | 4 +- .../IntegrationTestEnvironmentExtensions.cs | 8 +- .../IntegrationTests.cs | 4 +- .../MockConnectionWrapper.cs | 4 +- Centaurus.Test.Integration/SDKHelper.cs | 1 + Centaurus.Test.Utils/GlobalInitHelper.cs | 9 +- .../MessageSerializationTests.cs | 4 +- Centaurus/Alpha/AlphaHostBuilder.cs | 6 +- .../Controllers/ConstellationController.cs | 37 +-- .../Alpha/Models/RequestRateLimitsModel.cs | 9 - Centaurus/Program.cs | 2 +- Centaurus/Startup.cs | 15 +- 117 files changed, 2026 insertions(+), 1804 deletions(-) rename {Centaurus/Alpha/Models => Centaurus.Domain.Models/PublicInfo}/ConstellationInfo.cs (83%) delete mode 100644 Centaurus.Domain/MessageHandlers/AuditorStatisticsMessageHandler.cs delete mode 100644 Centaurus.Domain/MessageHandlers/HandshakeResponseHandlers/AuditorHandshakeResponseHandler.cs delete mode 100644 Centaurus.Domain/MessageHandlers/HandshakeResponseHandlers/HandshakeResponseHandlerBase.cs rename Centaurus.Domain/MessageHandlers/{StateUpdateMessageHandler.cs => StateMessageHandler.cs} (52%) rename Centaurus.Domain/MessageHandlers/{AlphaQuantaBatchHandler.cs => SyncQuantaBatchHandler.cs} (85%) create mode 100644 Centaurus.Domain/Nodes/Common/NodeApexes.cs create mode 100644 Centaurus.Domain/Nodes/Common/NodeBase.cs create mode 100644 Centaurus.Domain/Nodes/Common/NodeExtensions.cs create mode 100644 Centaurus.Domain/Nodes/Common/NodeQuantaQueueLengths.cs create mode 100644 Centaurus.Domain/Nodes/Common/StateChangedEventArgs.cs create mode 100644 Centaurus.Domain/Nodes/Common/SyncCursorCollection.cs create mode 100644 Centaurus.Domain/Nodes/Common/ValueAggregation.cs create mode 100644 Centaurus.Domain/Nodes/CurrentNode/CurrentNode.cs create mode 100644 Centaurus.Domain/Nodes/Managers/NodesManager.cs create mode 100644 Centaurus.Domain/Nodes/Managers/StateNotifierWorker.cs create mode 100644 Centaurus.Domain/Nodes/RemoteNode/RemoteNode.cs create mode 100644 Centaurus.Domain/Nodes/RemoteNode/RemoteNodeConnection.cs create mode 100644 Centaurus.Domain/Nodes/RemoteNode/RemoteNodeCursor.cs create mode 100644 Centaurus.Domain/Nodes/RemoteNode/RemoteNodesCollection.cs create mode 100644 Centaurus.Domain/Quanta/Processors/AlphaUpdateProcessor.cs create mode 100644 Centaurus.Domain/Quanta/Sync/CursorGroup.cs delete mode 100644 Centaurus.Domain/StateManagers/StateManager.cs delete mode 100644 Centaurus.Domain/Statistics/BatchSavedInfoExtensions.cs delete mode 100644 Centaurus.Domain/Statistics/PerformanceStatisticsExtensions.cs create mode 100644 Centaurus.Domain/UpdatesManager/BatchSavedInfo.cs delete mode 100644 Centaurus.Domain/WebSockets/Centaurus/IAuditorConnection.cs create mode 100644 Centaurus.Domain/WebSockets/Centaurus/INodeConnection.cs delete mode 100644 Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingAuditorConnection.cs create mode 100644 Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingConnectionsCollection.cs create mode 100644 Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingNodeConnection.cs delete mode 100644 Centaurus.Domain/WebSockets/Centaurus/Outgoing/OutgoingConnectionManager.cs create mode 100644 Centaurus.Models/InternalMessages/AlphaUpdateBase.cs delete mode 100644 Centaurus.Models/InternalMessages/PendingQuantum.cs delete mode 100644 Centaurus.Models/InternalMessages/PerformanceStatistics.cs delete mode 100644 Centaurus/Alpha/Models/RequestRateLimitsModel.cs diff --git a/Centaurus.Common/Helpers/UriHelper.cs b/Centaurus.Common/Helpers/UriHelper.cs index f4eb6826..6a145c20 100644 --- a/Centaurus.Common/Helpers/UriHelper.cs +++ b/Centaurus.Common/Helpers/UriHelper.cs @@ -25,7 +25,7 @@ public static bool TryCreateUriBuilder(string address, bool useSecureConnection, public static bool TryCreateWsConnection(string address, bool useSecureConnection, out Uri uri) { uri = null; - if (!TryCreateUriBuilder(address, useSecureConnection, out var uriBuilder)) + if (!(address != null && TryCreateUriBuilder(address, useSecureConnection, out var uriBuilder))) return false; uriBuilder.Scheme = useSecureConnection ? "wss" : "ws"; uri = uriBuilder.Uri; diff --git a/Centaurus/Alpha/Models/ConstellationInfo.cs b/Centaurus.Domain.Models/PublicInfo/ConstellationInfo.cs similarity index 83% rename from Centaurus/Alpha/Models/ConstellationInfo.cs rename to Centaurus.Domain.Models/PublicInfo/ConstellationInfo.cs index 8abcfa03..a5aacf9a 100644 --- a/Centaurus/Alpha/Models/ConstellationInfo.cs +++ b/Centaurus.Domain.Models/PublicInfo/ConstellationInfo.cs @@ -1,9 +1,13 @@ using Centaurus.Models; -namespace Centaurus +namespace Centaurus.Domain.Models { public class ConstellationInfo { + public ulong Apex { get; set; } + + public string PubKey { get; set; } + public State State { get; set; } public ProviderSettings[] Providers { get; set; } diff --git a/Centaurus.Domain.Models/Snapshot.cs b/Centaurus.Domain.Models/Snapshot.cs index 6ba057cd..b82d4345 100644 --- a/Centaurus.Domain.Models/Snapshot.cs +++ b/Centaurus.Domain.Models/Snapshot.cs @@ -9,7 +9,7 @@ public class Snapshot public byte[] LastHash { get; set; } - public ConstellationSettings Settings { get; set; } + public ConstellationSettings ConstellationSettings { get; set; } public List Accounts { get; set; } diff --git a/Centaurus.Domain/Catchups/Catchup.cs b/Centaurus.Domain/Catchups/Catchup.cs index 1eb56567..ba4c0dcf 100644 --- a/Centaurus.Domain/Catchups/Catchup.cs +++ b/Centaurus.Domain/Catchups/Catchup.cs @@ -13,7 +13,7 @@ namespace Centaurus.Domain /// /// This class manages auditor snapshots and quanta when Alpha is rising /// - public class Catchup : ContextualBase, IDisposable + internal class Catchup : ContextualBase, IDisposable { public Catchup(ExecutionContext context) : base(context) @@ -34,20 +34,21 @@ public async Task AddAuditorState(RawPubKey pubKey, CatchupQuantaBatch auditorSt try { logger.Trace($"Auditor state from {pubKey.GetAccountId()} received by AlphaCatchup."); - if (Context.StateManager.State != State.Rising) + if (Context.NodesManager.CurrentNode.State != State.Rising) { logger.Warn($"Auditor state messages can be only handled when Alpha is in rising state. State sent by {((KeyPair)pubKey).AccountId}"); return; } - if (!applyDataTimer.Enabled) //start timer + if (!(applyDataTimer.Enabled || pubKey.Equals(Context.Settings.KeyPair))) //start timer applyDataTimer.Start(); if (!allAuditorStates.TryGetValue(pubKey, out var pendingAuditorBatch)) { pendingAuditorBatch = auditorState; allAuditorStates.Add(pubKey, auditorState); - applyDataTimer.Reset(); + if (applyDataTimer.Enabled) + applyDataTimer.Reset(); logger.Trace($"Auditor state from {pubKey.GetAccountId()} added."); } else if (!AddQuanta(pubKey, pendingAuditorBatch, auditorState)) //check if auditor sent all quanta already @@ -69,27 +70,13 @@ public async Task AddAuditorState(RawPubKey pubKey, CatchupQuantaBatch auditorSt int majority = Context.GetMajorityCount(), totalAuditorsCount = Context.GetTotalAuditorsCount(); - var completedStatesCount = allAuditorStates.Count(s => !s.Value.HasMore) + 1; //+1 is current server + var completedStatesCount = allAuditorStates.Count(s => !s.Value.HasMore); if (completedStatesCount == totalAuditorsCount) await TryApplyAuditorsData(); } catch (Exception exc) { - Context.StateManager.Failed(new Exception("Error on adding auditors state", exc)); - } - finally - { - semaphoreSlim.Release(); - } - } - - public void RemoveState(RawPubKey pubKey) - { - semaphoreSlim.Wait(); - try - { - allAuditorStates.Remove(pubKey); - validAuditorStates.Remove(pubKey); + Context.NodesManager.CurrentNode.Failed(new Exception("Error on adding auditors state", exc)); } finally { @@ -125,17 +112,18 @@ private async Task TryApplyAuditorsData() try { logger.Trace($"Try apply auditors data."); - if (Context.StateManager.State != State.Rising) + if (Context.NodesManager.CurrentNode.State != State.Rising) return; int majority = Context.GetMajorityCount(), totalAuditorsCount = Context.GetTotalAuditorsCount(); - if (Context.HasMajority(validAuditorStates.Count, false)) + if (Context.HasMajority(validAuditorStates.Count)) { await ApplyAuditorsData(); allAuditorStates.Clear(); validAuditorStates.Clear(); + Context.NodesManager.CurrentNode.Rised(); logger.Trace($"Alpha is risen."); } else @@ -147,7 +135,7 @@ private async Task TryApplyAuditorsData() } catch (Exception exc) { - Context.StateManager.Failed(new Exception("Error during raising.", exc)); + Context.NodesManager.CurrentNode.Failed(new Exception("Error during raising.", exc)); } finally { @@ -182,13 +170,9 @@ private async Task ApplyAuditorsData() var validQuanta = GetValidQuanta(); await ApplyQuanta(validQuanta); - - var alphaStateManager = Context.StateManager; - - alphaStateManager.Rised(); } - private async Task ApplyQuanta(List<(Quantum quantum, List signatures)> quanta) + private async Task ApplyQuanta(List<(Quantum quantum, List signatures)> quanta) { foreach (var quantumItem in quanta) { @@ -209,7 +193,7 @@ private async Task ApplyQuanta(List<(Quantum quantum, List signatures)> GetValidQuanta() + private List<(Quantum quantum, List signatures)> GetValidQuanta() { //group all quanta by their apex var quanta = allAuditorStates.Values @@ -217,7 +201,7 @@ private async Task ApplyQuanta(List<(Quantum quantum, List ((Quantum)q.Quantum).Apex) .OrderBy(q => q.Key); - var validQuanta = new List<(Quantum quantum, List signatures)>(); + var validQuanta = new List<(Quantum quantum, List signatures)>(); if (quanta.Count() == 0) return validQuanta; @@ -264,11 +248,11 @@ private async Task ApplyQuanta(List<(Quantum quantum, List signatures) GetQuantumData(ulong apex, List allQuanta, int alphaId, List auditors) + private (Quantum quantum, List signatures) GetQuantumData(ulong apex, List allQuanta, int alphaId, List auditors) { var payloadHash = default(byte[]); var quantum = default(Quantum); - var signatures = new Dictionary(); + var signatures = new Dictionary(); foreach (var currentItem in allQuanta) { diff --git a/Centaurus.Domain/Centaurus.Domain.csproj b/Centaurus.Domain/Centaurus.Domain.csproj index 5dbe7ebc..97cde304 100644 --- a/Centaurus.Domain/Centaurus.Domain.csproj +++ b/Centaurus.Domain/Centaurus.Domain.csproj @@ -1,48 +1,72 @@  - - netcoreapp3.1 - 0.1.21 - 0.1.21 - 0.1.21 - + + netcoreapp3.1 + 0.1.21 + 0.1.21 + 0.1.21 + - - - - - + + + + + - - - - + + + + - - - - - - - - - - + + + + + + + + + + - - - - + + + + - - - - - - - - - - + + + + + + + + + + + + + <_Parameter1>Centaurus.Test.Domain + + + + + + <_Parameter1>Centaurus.Test.Exchange.Analytics + + + + + + <_Parameter1>Centaurus.Test.Integration + + + + + + <_Parameter1>Centaurus.Test.Utils + + + \ No newline at end of file diff --git a/Centaurus.Domain/Contexts/ExecutionContext.Analytics.cs b/Centaurus.Domain/Contexts/ExecutionContext.Analytics.cs index 5346831b..a3809187 100644 --- a/Centaurus.Domain/Contexts/ExecutionContext.Analytics.cs +++ b/Centaurus.Domain/Contexts/ExecutionContext.Analytics.cs @@ -78,7 +78,7 @@ private void DisposeAnalyticsManager() private void AnalyticsManager_OnError(Exception exc) { - StateManager.Failed(new Exception("Analytics manager error.", exc)); + NodesManager.CurrentNode.Failed(new Exception("Analytics manager error.", exc)); } private void Exchange_OnUpdates(ExchangeUpdate updates) diff --git a/Centaurus.Domain/Contexts/ExecutionContext.cs b/Centaurus.Domain/Contexts/ExecutionContext.cs index a959c80b..590180f6 100644 --- a/Centaurus.Domain/Contexts/ExecutionContext.cs +++ b/Centaurus.Domain/Contexts/ExecutionContext.cs @@ -20,18 +20,34 @@ public partial class ExecutionContext : IDisposable { static Logger logger = LogManager.GetCurrentClassLogger(); + public ExecutionContext(Settings settings) + : this( + settings, + new PersistentStorageAbstraction(), + PaymentProvidersFactoryBase.Default, + OutgoingConnectionFactoryBase.Default) + { } + /// Application config /// Persistent storage object - public ExecutionContext(Settings settings, IPersistentStorage storage, PaymentProvidersFactoryBase paymentProviderFactory, OutgoingConnectionFactoryBase connectionFactory) + internal ExecutionContext( + Settings settings, + IPersistentStorage storage, + PaymentProvidersFactoryBase paymentProviderFactory, + OutgoingConnectionFactoryBase connectionFactory) { + DynamicSerializersInitializer.Init(); Settings = settings ?? throw new ArgumentNullException(nameof(settings)); PersistentStorage = storage ?? throw new ArgumentNullException(nameof(storage)); - PersistentStorage.Connect(GetAbsolutePath(Settings.ConnectionString)); + + OutgoingConnectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); PaymentProviderFactory = paymentProviderFactory ?? throw new ArgumentNullException(nameof(paymentProviderFactory)); + PersistentStorage.Connect(GetAbsolutePath(Settings.ConnectionString)); + RoleManager = new RoleManager((CentaurusNodeParticipationLevel)Settings.ParticipationLevel); ExtensionsManager = new ExtensionsManager(GetAbsolutePath(Settings.ExtensionsConfigFilePath)); @@ -41,7 +57,6 @@ public ExecutionContext(Settings settings, IPersistentStorage storage, PaymentPr QuantumProcessor = new QuantumProcessorsStorage(this); PendingUpdatesManager = new UpdatesManager(this); - PendingUpdatesManager.OnBatchSaved += PendingUpdatesManager_OnBatchSaved; MessageHandlers = new MessageHandlers(this); @@ -49,8 +64,6 @@ public ExecutionContext(Settings settings, IPersistentStorage storage, PaymentPr IncomingConnectionManager = new IncomingConnectionManager(this); - OutgoingConnectionManager = new OutgoingConnectionManager(this, connectionFactory); - SubscriptionsManager = new SubscriptionsManager(); InfoConnectionManager = new InfoConnectionManager(this); @@ -61,13 +74,12 @@ public ExecutionContext(Settings settings, IPersistentStorage storage, PaymentPr Catchup = new Catchup(this); - StateManager = new StateManager(this); - StateManager.StateChanged += AppState_StateChanged; - - DynamicSerializersInitializer.Init(); - var persistentData = DataProvider.GetPersistentData(); + StateNotifier = new StateNotifierWorker(this); + var currentState = persistentData == default ? State.WaitingForInit : State.Rising; + NodesManager = new NodesManager(this, currentState); + var lastApex = persistentData.snapshot?.Apex ?? 0; var lastHash = persistentData.snapshot?.LastHash ?? new byte[32]; @@ -80,64 +92,24 @@ public ExecutionContext(Settings settings, IPersistentStorage storage, PaymentPr //apply data, if presented in db if (persistentData != default) { - StateManager.Init(State.Rising); //apply snapshot if not null if (persistentData.snapshot != null) - Setup(persistentData.snapshot); + Init(persistentData.snapshot); if (persistentData.pendingQuanta != null) HandlePendingQuanta(persistentData.pendingQuanta); - if (!IsAlpha) - StateManager.Rised(); + NodesManager.CurrentNode.Rised(); } if (Constellation == null) - { - SetAuditorStates(); - //establish connection with genesis auditors - EstablishOutgoingConnections(); - } + SetNodes(); } - private void HandlePendingQuanta(List pendingQuanta) + private void HandlePendingQuanta(List pendingQuanta) { - foreach (var quantum in pendingQuanta) - { - try - { - //cache current payload hash - var persistentQuantumHash = quantum.Quantum.GetPayloadHash(); - - //handle quantum - var quantumProcessingItem = QuantumHandler.HandleAsync(quantum.Quantum, QuantumSignatureValidator.Validate(quantum.Quantum)); - - quantumProcessingItem.OnAcknowledged.Wait(); - - //verify that the pending quantum has current node signature - var currentNodeSignature = quantum.Signatures.FirstOrDefault(s => s.AuditorId == Constellation.GetAuditorId(Settings.KeyPair)) ?? throw new Exception($"Unable to get signature for quantum {quantum.Quantum.Apex}"); - - //verify the payload signature - if (!Settings.KeyPair.Verify(quantum.Quantum.GetPayloadHash(), currentNodeSignature.PayloadSignature.Data)) - throw new Exception($"Signature for the quantum {quantum.Quantum.Apex} is invalid."); - - //validate that quantum payload data is the same - if (!persistentQuantumHash.AsSpan().SequenceEqual(quantum.Quantum.GetPayloadHash())) - throw new Exception($"Payload hash for the quantum {quantum.Quantum.Apex} is not equal to persistent."); - - //add signatures - ResultManager.Add(new QuantumSignatures { Apex = quantum.Quantum.Apex, Signatures = quantum.Signatures }); - - if (quantum.Quantum.Apex % 1000 == 0) - logger.Info($"Pending quanta handling. Current apex: {quantum.Quantum.Apex}, last pending quanta: {pendingQuanta.Last().Quantum.Apex}"); - } - catch (AggregateException exc) - { - //unwrap aggregate exc - throw exc.GetBaseException(); - } - } + _ = Catchup.AddAuditorState(Settings.KeyPair, new CatchupQuantaBatch { Quanta = pendingQuanta, HasMore = false }); } private string GetAbsolutePath(string path) @@ -149,35 +121,20 @@ private string GetAbsolutePath(string path) : Path.GetFullPath(Path.Combine(Settings.CWD, path.Trim('.').Trim('\\').Trim('/'))); } - public void Setup(Snapshot snapshot) + public void Init(Snapshot snapshot) { - - if (Exchange != null) - Exchange.OnUpdates -= Exchange_OnUpdates; - - Constellation = snapshot.Settings; - - AuditorIds = Constellation.Auditors.ToDictionary(a => Constellation.GetAuditorId(a.PubKey), a => a.PubKey); - AuditorPubKeys = AuditorIds.ToDictionary(a => a.Value, a => a.Key); - AlphaId = AuditorPubKeys[Constellation.Alpha]; - - //update current auditors - SetAuditorStates(); - - SetRole(); + UpdateConstellationSettings(snapshot.ConstellationSettings); AccountStorage = new AccountStorage(snapshot.Accounts); - Exchange?.Dispose(); Exchange = Exchange.RestoreExchange(snapshot.Settings.Assets, snapshot.Orders, RoleManager.ParticipationLevel == CentaurusNodeParticipationLevel.Prime); + Exchange = Exchange.RestoreExchange(Constellation.Assets, snapshot.Orders, RoleManager.ParticipationLevel == CentaurusNodeParticipationLevel.Prime); SetupPaymentProviders(snapshot.Cursors); - DisposeAnalyticsManager(); - AnalyticsManager = new AnalyticsManager( PersistentStorage, DepthsSubscription.Precisions.ToList(), - Constellation.Assets.Where(a => !a.IsQuoteAsset).Select(a => a.Code).ToList(), //all but base asset + Constellation.Assets.Where(a => !a.IsQuoteAsset).Select(a => a.Code).ToList(), //all but quote asset snapshot.Orders.Select(o => o.Order.ToOrderInfo()).ToList() ); @@ -187,23 +144,33 @@ public void Setup(Snapshot snapshot) AnalyticsManager.OnError += AnalyticsManager_OnError; AnalyticsManager.OnUpdate += AnalyticsManager_OnUpdate; Exchange.OnUpdates += Exchange_OnUpdates; + } - PerformanceStatisticsManager?.Dispose(); PerformanceStatisticsManager = new PerformanceStatisticsManager(this); + public void UpdateConstellationSettings(ConstellationSettings constellationSettings) + { + Constellation = constellationSettings; - IncomingConnectionManager.CleanupAuditorConnections(); + AuditorIds = Constellation.Auditors.ToDictionary(a => Constellation.GetAuditorId(a.PubKey), a => a.PubKey); + AuditorPubKeys = AuditorIds.ToDictionary(a => a.Value, a => a.Key); + AlphaId = AuditorPubKeys[Constellation.Alpha]; - EstablishOutgoingConnections(); + //update current auditors + SetNodes(); + + SetRole(); + + IncomingConnectionManager.CleanupAuditorConnections(); } public void Complete() { - StateManager.Stopped(); + NodesManager.CurrentNode.Stopped(); //close all connections Task.WaitAll( IncomingConnectionManager.CloseAllConnections(), InfoConnectionManager.CloseAllConnections(), - Task.Factory.StartNew(OutgoingConnectionManager.CloseAllConnections) + Task.Factory.StartNew(() => NodesManager.ClearNodes()) ); PersistPendingQuanta(); @@ -224,21 +191,12 @@ private void PersistPendingQuanta() PendingUpdatesManager.PersistPendingQuanta(); } - private void EstablishOutgoingConnections() + private void SetNodes() { var auditors = Constellation != null - ? Constellation.Auditors.Select(a => new Settings.Auditor(a.PubKey, a.Address)).ToList() - : Settings.GenesisAuditors.ToList(); - OutgoingConnectionManager.Connect(auditors); - } - - - private void SetAuditorStates() - { - var auditors = Constellation != null - ? Constellation.Auditors.Select(a => a.PubKey).ToList() - : Settings.GenesisAuditors.Select(a => (RawPubKey)a.PubKey).ToList(); - StateManager.SetAuditors(auditors); + ? Constellation.Auditors.ToList() + : Settings.GenesisAuditors.Select(a => new Auditor { PubKey = a.PubKey, Address = a.Address }).ToList(); + NodesManager.SetAuditors(auditors); } private void SetRole() @@ -280,8 +238,8 @@ private void SetupPaymentProviders(Dictionary cursors) private void PaymentProvider_OnPaymentCommit(PaymentProviderBase paymentProvider, PaymentProvider.Models.DepositNotificationModel notification) { - if (!IsAlpha || StateManager.State != State.Ready) - throw new OperationCanceledException($"Current server is not ready to process deposits. Is Alpha: {IsAlpha}, State: {StateManager.State}"); + if (!IsAlpha || NodesManager.CurrentNode.State != State.Ready) + throw new OperationCanceledException($"Current server is not ready to process deposits. Is Alpha: {IsAlpha}, State: {NodesManager.CurrentNode.State}"); QuantumHandler.HandleAsync(new DepositQuantum { Source = notification.ToDomainModel() }, Task.FromResult(true)); } @@ -298,21 +256,18 @@ public void Dispose() ResultManager.Dispose(); DisposeAnalyticsManager(); - PerformanceStatisticsManager?.Dispose(); + PerformanceStatisticsManager.Dispose(); } private async void AppState_StateChanged(StateChangedEventArgs stateChangedEventArgs) { - var state = stateChangedEventArgs.State; var prevState = stateChangedEventArgs.PrevState; if (state != State.Ready && prevState == State.Ready) //close all connections (except auditors) await IncomingConnectionManager.CloseAllConnections(false); - } - - private void PendingUpdatesManager_OnBatchSaved(BatchSavedInfo batchInfo) - { - PerformanceStatisticsManager?.OnBatchSaved(batchInfo); + if (prevState == State.Rising && state == State.Running || state == State.Ready) + //after node successfully started, the pending quanta can be deleted + PersistentStorage.DeletePendingQuanta(); } public DataProvider DataProvider { get; } @@ -337,21 +292,23 @@ private void PendingUpdatesManager_OnBatchSaved(BatchSavedInfo batchInfo) public PaymentProvidersFactoryBase PaymentProviderFactory { get; } - public StateManager StateManager { get; } + internal NodesManager NodesManager { get; } + + internal StateNotifierWorker StateNotifier { get; } public QuantumHandler QuantumHandler { get; } public IncomingConnectionManager IncomingConnectionManager { get; } - public OutgoingConnectionManager OutgoingConnectionManager { get; } - public SubscriptionsManager SubscriptionsManager { get; } public InfoConnectionManager InfoConnectionManager { get; } - public SyncQuantaDataWorker SyncQuantaDataWorker { get; } - public ProxyWorker ProxyWorker { get; } - public Catchup Catchup { get; } + private SyncQuantaDataWorker SyncQuantaDataWorker { get; } + + internal ProxyWorker ProxyWorker { get; } + + internal Catchup Catchup { get; } public InfoCommandsHandlers InfoCommandsHandlers { get; } @@ -363,7 +320,7 @@ private void PendingUpdatesManager_OnBatchSaved(BatchSavedInfo batchInfo) public AccountStorage AccountStorage { get; private set; } - public PerformanceStatisticsManager PerformanceStatisticsManager { get; private set; } + private PerformanceStatisticsManager PerformanceStatisticsManager { get; } public HashSet AssetIds { get; private set; } @@ -375,6 +332,10 @@ private void PendingUpdatesManager_OnBatchSaved(BatchSavedInfo batchInfo) public Dictionary AuditorPubKeys { get; private set; } + public OutgoingConnectionFactoryBase OutgoingConnectionFactory { get; } + public byte AlphaId { get; private set; } + + public event Action OnComplete; } } \ No newline at end of file diff --git a/Centaurus.Domain/DataProvider/DataProvider.cs b/Centaurus.Domain/DataProvider/DataProvider.cs index 6724dcdb..d9405823 100644 --- a/Centaurus.Domain/DataProvider/DataProvider.cs +++ b/Centaurus.Domain/DataProvider/DataProvider.cs @@ -93,9 +93,9 @@ public ulong GetLastApex() return Context.PersistentStorage.GetLastApex(); } - public (Snapshot snapshot, List pendingQuanta) GetPersistentData() + public (Snapshot snapshot, List pendingQuanta) GetPersistentData() { - (Snapshot snapshot, List pendingQuanta) data = default; + (Snapshot snapshot, List pendingQuanta) data = default; var lastApex = GetLastApex(); if (lastApex > 0) data.snapshot = GetSnapshot(lastApex); @@ -247,7 +247,7 @@ public Snapshot GetSnapshot(ulong apex) Apex = apex, Accounts = accountStorage.GetAll().OrderBy(a => a.Pubkey.ToString()).ToList(), Orders = allOrders.OrderBy(o => o.OrderId).ToList(), - Settings = settings, + ConstellationSettings = settings, LastHash = lastQuantumData.ComputeHash(), Cursors = cursors }; @@ -259,12 +259,12 @@ public void SaveBatch(List updates) Context.PersistentStorage.SaveBatch(updates); } - public List LoadPendingQuanta() + public List LoadPendingQuanta() { var pendingQuantaModel = Context.PersistentStorage.LoadPendingQuanta(); if (pendingQuantaModel == null) return null; - return pendingQuantaModel.Quanta.Select(q => q.ToDomainModel()).ToList(); + return pendingQuantaModel.Quanta.Select(q => q.ToCatchupQuantaBatchItem()).ToList(); } /// diff --git a/Centaurus.Domain/DataProvider/PersistentModelExtensions/PendingQuantumPersistentModelExtensions.cs b/Centaurus.Domain/DataProvider/PersistentModelExtensions/PendingQuantumPersistentModelExtensions.cs index 7aa25d6b..21970432 100644 --- a/Centaurus.Domain/DataProvider/PersistentModelExtensions/PendingQuantumPersistentModelExtensions.cs +++ b/Centaurus.Domain/DataProvider/PersistentModelExtensions/PendingQuantumPersistentModelExtensions.cs @@ -7,28 +7,34 @@ namespace Centaurus.Domain { - public static class PendingQuantumPersistentModelExtensions + internal static class PendingQuantumPersistentModelExtensions { - public static PendingQuantum ToDomainModel(this PendingQuantumPersistentModel quantumModel) + public static CatchupQuantaBatchItem ToCatchupQuantaBatchItem(this PendingQuantumPersistentModel quantumModel) { - var quantum = new PendingQuantum + var quantum = new CatchupQuantaBatchItem { - Quantum = (Quantum)XdrConverter.Deserialize(quantumModel.RawQuantum), - Signatures = new List() + Quantum = XdrConverter.Deserialize(quantumModel.RawQuantum), + Signatures = new List() }; foreach (var signature in quantumModel.Signatures) { - quantum.Signatures.Add(new AuditorSignatureInternal - { - AuditorId = signature.AuditorId, - PayloadSignature = new TinySignature { Data = signature.PayloadSignature }, - TxSignature = signature.TxSignature, - TxSigner = signature.TxSigner - }); + var nodeSignature = signature.ToNodeSignature(); + quantum.Signatures.Add(nodeSignature); } return quantum; } + + public static NodeSignatureInternal ToNodeSignature(this SignatureModel signature) + { + return new NodeSignatureInternal + { + AuditorId = signature.AuditorId, + PayloadSignature = new TinySignature { Data = signature.PayloadSignature }, + TxSignature = signature.TxSignature, + TxSigner = signature.TxSigner + }; + } } } diff --git a/Centaurus.Domain/DataProvider/PersistentModelExtensions/SignatureModelExtensions.cs b/Centaurus.Domain/DataProvider/PersistentModelExtensions/SignatureModelExtensions.cs index fcf6249d..871df88d 100644 --- a/Centaurus.Domain/DataProvider/PersistentModelExtensions/SignatureModelExtensions.cs +++ b/Centaurus.Domain/DataProvider/PersistentModelExtensions/SignatureModelExtensions.cs @@ -8,7 +8,7 @@ namespace Centaurus.Domain { public static class SignatureModelExtensions { - public static SignatureModel ToPersistenModel(this AuditorSignatureInternal auditorSignature) + public static SignatureModel ToPersistenModel(this NodeSignatureInternal auditorSignature) { if (auditorSignature == null) throw new ArgumentNullException(nameof(auditorSignature)); @@ -22,12 +22,12 @@ public static SignatureModel ToPersistenModel(this AuditorSignatureInternal audi }; } - public static AuditorSignatureInternal ToDomainModel(this SignatureModel auditorSignature) + public static NodeSignatureInternal ToDomainModel(this SignatureModel auditorSignature) { if (auditorSignature == null) throw new ArgumentNullException(nameof(auditorSignature)); - return new AuditorSignatureInternal + return new NodeSignatureInternal { AuditorId = auditorSignature.AuditorId, PayloadSignature = new TinySignature { Data = auditorSignature.PayloadSignature }, diff --git a/Centaurus.Domain/Helpers/ConstellationQuantumExtesnions.cs b/Centaurus.Domain/Helpers/ConstellationQuantumExtesnions.cs index 7c6e3ea5..2b3bf6f0 100644 --- a/Centaurus.Domain/Helpers/ConstellationQuantumExtesnions.cs +++ b/Centaurus.Domain/Helpers/ConstellationQuantumExtesnions.cs @@ -16,7 +16,6 @@ public static void Validate(this ConstellationQuantum constellationQuantum, Exec if (context == null) throw new ArgumentNullException(nameof(context)); - if (constellationQuantum.RequestMessage == null || !(constellationQuantum.RequestMessage is ConstellationRequestMessage)) throw new Exception("Invalid message type."); diff --git a/Centaurus.Domain/Helpers/ConstellationSettingsExtensions.cs b/Centaurus.Domain/Helpers/ConstellationSettingsExtensions.cs index 2f36ffc1..cf55f761 100644 --- a/Centaurus.Domain/Helpers/ConstellationSettingsExtensions.cs +++ b/Centaurus.Domain/Helpers/ConstellationSettingsExtensions.cs @@ -26,7 +26,7 @@ public static Snapshot ToSnapshot(this ConstellationSettings settings, ulong ape Apex = apex, Accounts = accounts ?? throw new ArgumentNullException(nameof(accounts)), Orders = orders ?? throw new ArgumentNullException(nameof(orders)), - Settings = settings ?? throw new ArgumentNullException(nameof(settings)), + ConstellationSettings = settings ?? throw new ArgumentNullException(nameof(settings)), Cursors = cursors ?? throw new ArgumentNullException(nameof(cursors)), LastHash = quantumHash ?? throw new ArgumentNullException(nameof(quantumHash)) }; diff --git a/Centaurus.Domain/Helpers/ExecutionContextExtensions.cs b/Centaurus.Domain/Helpers/ExecutionContextExtensions.cs index 5f71765e..6ecddcdd 100644 --- a/Centaurus.Domain/Helpers/ExecutionContextExtensions.cs +++ b/Centaurus.Domain/Helpers/ExecutionContextExtensions.cs @@ -1,8 +1,10 @@ -using Centaurus.Models; +using Centaurus.Domain.Models; +using Centaurus.Models; using System; using System.Collections.Generic; using System.Linq; using System.Text; +using System.Threading.Tasks; namespace Centaurus.Domain { @@ -18,5 +20,47 @@ public static List GetAuditors(this ExecutionContext context) : context.Constellation.Auditors.Select(a => a.PubKey)) .ToList(); } + + public static ConstellationInfo GetInfo(this ExecutionContext context) + { + if (context == null) + throw new ArgumentNullException(nameof(context)); + + var currentNode = context.NodesManager.CurrentNode; + var info = new ConstellationInfo + { + State = currentNode.State, + Apex = context.QuantumHandler.CurrentApex, + PubKey = currentNode.AccountId + }; + + var constellationSettings = context.Constellation; + if (constellationSettings != null) + { + info.Providers = constellationSettings.Providers.ToArray(); + info.Auditors = constellationSettings.Auditors + .Select(a => new ConstellationInfo.Auditor { PubKey = a.PubKey.GetAccountId(), Address = a.Address }) + .ToArray(); + info.MinAccountBalance = constellationSettings.MinAccountBalance; + info.MinAllowedLotSize = constellationSettings.MinAllowedLotSize; + info.Assets = constellationSettings.Assets.ToArray(); + info.RequestRateLimits = constellationSettings.RequestRateLimits; + } + return info; + } + + public static async Task HandleConstellationQuantum(this ExecutionContext context, ConstellationMessageEnvelope constellationInitEnvelope) + { + if (context == null) + throw new ArgumentNullException(nameof(context)); + + var constellationQuantum = new ConstellationQuantum { RequestEnvelope = constellationInitEnvelope }; + + constellationQuantum.Validate(context); + + var quantumProcessingItem = context.QuantumHandler.HandleAsync(constellationQuantum, Task.FromResult(true)); + + await quantumProcessingItem.OnProcessed; + } } -} +} \ No newline at end of file diff --git a/Centaurus.Domain/Helpers/SyncCursorUpdateExtensions.cs b/Centaurus.Domain/Helpers/SyncCursorUpdateExtensions.cs index d5e497d1..6f89c23f 100644 --- a/Centaurus.Domain/Helpers/SyncCursorUpdateExtensions.cs +++ b/Centaurus.Domain/Helpers/SyncCursorUpdateExtensions.cs @@ -8,32 +8,17 @@ namespace Centaurus.Domain { public static class SyncCursorUpdateExtensions { - public static List ToDomainModel(this List syncCursors) + public static SyncCursorType ToDomainCursorType(this XdrSyncCursorType syncCursorType) { - if (syncCursors == null) - throw new ArgumentNullException(nameof(syncCursors)); - - var cursors = new List(); - - foreach (var cursorReset in syncCursors) + switch (syncCursorType) { - var cursorType = default(SyncCursorType); - switch (cursorReset.Type) - { - case XdrSyncCursorType.Quanta: - cursorType = SyncCursorType.Quanta; - break; - case XdrSyncCursorType.Signatures: - cursorType = SyncCursorType.Signatures; - break; - default: - throw new ArgumentNullException($"{cursorReset.Type} cursor type is not supported."); - } - var cursor = cursorReset.ClearCursor ? null : (ulong?)cursorReset.Cursor; - cursors.Add(new SyncCursorUpdate(default(DateTime), cursor, cursorType)); + case XdrSyncCursorType.Quanta: + return SyncCursorType.Quanta; + case XdrSyncCursorType.Signatures: + return SyncCursorType.Signatures; + default: + throw new ArgumentNullException($"{syncCursorType} cursor type is not supported."); } - - return cursors; } } } diff --git a/Centaurus.Domain/MessageHandlers/AuditorStatisticsMessageHandler.cs b/Centaurus.Domain/MessageHandlers/AuditorStatisticsMessageHandler.cs deleted file mode 100644 index 2465d6a5..00000000 --- a/Centaurus.Domain/MessageHandlers/AuditorStatisticsMessageHandler.cs +++ /dev/null @@ -1,32 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Text; -using System.Threading.Tasks; -using Centaurus.Models; - -namespace Centaurus.Domain -{ - public class AuditorPerfStatisticsMessageHandler : MessageHandlerBase - { - public AuditorPerfStatisticsMessageHandler(ExecutionContext context) - : base(context) - { - } - - public override string SupportedMessageType { get; } = typeof(AuditorPerfStatistics).Name; - - public override ConnectionState[] ValidConnectionStates => new[] { ConnectionState.Ready }; - - public override bool IsAuditorOnly => true; - - public override Task HandleMessage(ConnectionBase connection, IncomingMessage message) - { - var auditor = connection.PubKeyAddress; - var statistics = (AuditorPerfStatistics)message.Envelope.Message; - - Context.PerformanceStatisticsManager?.AddAuditorStatistics(auditor, statistics); - return Task.CompletedTask; - } - } -} \ No newline at end of file diff --git a/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchHandler.cs b/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchHandler.cs index f9bb057c..eeb465a0 100644 --- a/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchHandler.cs +++ b/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchHandler.cs @@ -6,7 +6,7 @@ namespace Centaurus.Domain { - public class CatchupQuantaBatchHandler: MessageHandlerBase + internal class CatchupQuantaBatchHandler: MessageHandlerBase { public CatchupQuantaBatchHandler(ExecutionContext context) :base(context) @@ -15,7 +15,7 @@ public CatchupQuantaBatchHandler(ExecutionContext context) public override string SupportedMessageType => typeof(CatchupQuantaBatch).Name; - public override Task HandleMessage(IncomingAuditorConnection connection, IncomingMessage message) + public override Task HandleMessage(IncomingNodeConnection connection, IncomingMessage message) { _ = Context.Catchup.AddAuditorState(connection.PubKey, (CatchupQuantaBatch)message.Envelope.Message); return Task.CompletedTask; diff --git a/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchRequestHandler.cs b/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchRequestHandler.cs index 89e2f2d1..a80e3008 100644 --- a/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchRequestHandler.cs +++ b/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchRequestHandler.cs @@ -8,7 +8,7 @@ namespace Centaurus.Domain { - public class CatchupQuantaBatchRequestHandler : MessageHandlerBase + internal class CatchupQuantaBatchRequestHandler : MessageHandlerBase { public CatchupQuantaBatchRequestHandler(ExecutionContext context) : base(context) @@ -45,9 +45,9 @@ private async Task SendQuanta(OutgoingConnection connection, CatchupQuantaBatchR { var apex = ((Quantum)quantum.Quantum).Apex; - var quantumSignatures = default(List); + var quantumSignatures = default(List); //if quantum was persisted, than it contains majority signatures already. Otherwise we need to obtain it from the result manager - if (apex > Context.PendingUpdatesManager.LastSavedApex) + if (apex > Context.PendingUpdatesManager.LastPersistedApex) Context.ResultManager.TryGetSignatures(apex, out quantumSignatures); else { @@ -55,11 +55,11 @@ private async Task SendQuanta(OutgoingConnection connection, CatchupQuantaBatchR if (signaturesBatch.TryGetValue(apex, out quantumSignatures)) quantumSignatures.Insert(0, quantum.AlphaSignature); else - quantumSignatures = new List { quantum.AlphaSignature }; + quantumSignatures = new List { quantum.AlphaSignature }; } if (quantumSignatures.Count < 1) { - Context.StateManager.Failed(new Exception($"Unable to find signatures for quantum {apex}")); + Context.NodesManager.CurrentNode.Failed(new Exception($"Unable to find signatures for quantum {apex}")); return; } batch.Quanta.Add(new CatchupQuantaBatchItem diff --git a/Centaurus.Domain/MessageHandlers/EffectsRequestMessageHandler.cs b/Centaurus.Domain/MessageHandlers/EffectsRequestMessageHandler.cs index 34300cb3..7318af1a 100644 --- a/Centaurus.Domain/MessageHandlers/EffectsRequestMessageHandler.cs +++ b/Centaurus.Domain/MessageHandlers/EffectsRequestMessageHandler.cs @@ -6,7 +6,7 @@ namespace Centaurus.Domain { - public class EffectsRequestMessageHandler : MessageHandlerBase + internal class EffectsRequestMessageHandler : MessageHandlerBase { static Logger logger = LogManager.GetCurrentClassLogger(); diff --git a/Centaurus.Domain/MessageHandlers/HandshakeRequestHandler.cs b/Centaurus.Domain/MessageHandlers/HandshakeRequestHandler.cs index cfa32848..fdc9e811 100644 --- a/Centaurus.Domain/MessageHandlers/HandshakeRequestHandler.cs +++ b/Centaurus.Domain/MessageHandlers/HandshakeRequestHandler.cs @@ -7,7 +7,7 @@ namespace Centaurus.Domain { - public class HandshakeRequestHandler : MessageHandlerBase + internal class HandshakeRequestHandler : MessageHandlerBase { public HandshakeRequestHandler(ExecutionContext context) : base(context) @@ -28,13 +28,12 @@ public override async Task HandleMessage(OutgoingConnection connection, Incoming var signaturesCursor = new SyncCursor { Type = XdrSyncCursorType.Signatures, - Cursor = Context.PendingUpdatesManager.LastSavedApex, - ClearCursor = Context.RoleManager.ParticipationLevel == CentaurusNodeParticipationLevel.Prime + Cursor = Context.PendingUpdatesManager.LastPersistedApex, + DisableSync = Context.RoleManager.ParticipationLevel == CentaurusNodeParticipationLevel.Prime }; - await connection.SendMessage(new AuditorHandshakeResponse + await connection.SendMessage(new HandshakeResponse { - HandshakeData = handshakeRequest.HandshakeData, - Cursors = new List { quantaCursor, signaturesCursor } + HandshakeData = handshakeRequest.HandshakeData }); //after sending auditor handshake the connection becomes ready diff --git a/Centaurus.Domain/MessageHandlers/HandshakeResponseHandlers/AuditorHandshakeResponseHandler.cs b/Centaurus.Domain/MessageHandlers/HandshakeResponseHandlers/AuditorHandshakeResponseHandler.cs deleted file mode 100644 index b111d0fb..00000000 --- a/Centaurus.Domain/MessageHandlers/HandshakeResponseHandlers/AuditorHandshakeResponseHandler.cs +++ /dev/null @@ -1,55 +0,0 @@ -using Centaurus.Domain.Quanta.Sync; -using Centaurus.Models; -using NLog; -using System; -using System.Collections.Generic; -using System.Net.WebSockets; -using System.Text; -using System.Threading.Tasks; - -namespace Centaurus.Domain -{ - public class AuditorHandshakeResponseHandler : HandshakeResponseHandlerBase - { - public AuditorHandshakeResponseHandler(ExecutionContext context) - : base(context) - { - } - - public override string SupportedMessageType { get; } = typeof(AuditorHandshakeResponse).Name; - - public override bool IsAuditorOnly => true; - - public override async Task HandleMessage(ConnectionBase connection, IncomingMessage message) - { - await base.HandleMessage(connection, message); - - var auditorHandshake = (AuditorHandshakeResponse)message.Envelope.Message; - - if (!(connection is IncomingAuditorConnection incomingAuditorConnection)) - throw new BadRequestException("Invalid message."); - - if (!incomingAuditorConnection.TryValidate(auditorHandshake.HandshakeData)) - throw new ConnectionCloseException(WebSocketCloseStatus.InvalidPayloadData, "Handshake data is invalid."); - - //send current state - await incomingAuditorConnection.SendMessage(new StateUpdateMessage - { - State = Context.StateManager.State - }.CreateEnvelope()); - - incomingAuditorConnection.SetSyncCursor(true, auditorHandshake.Cursors.ToDomainModel().ToArray()); - - Context.StateManager.SetAuditorState(connection.PubKey, auditorHandshake.State); - - //request quanta on rising - if (Context.StateManager.State == State.Rising) - { - await incomingAuditorConnection.SendMessage(new CatchupQuantaBatchRequest - { - LastKnownApex = Context.QuantumHandler.CurrentApex - }.CreateEnvelope()); - } - } - } -} \ No newline at end of file diff --git a/Centaurus.Domain/MessageHandlers/HandshakeResponseHandlers/HandshakeResponseHandler.cs b/Centaurus.Domain/MessageHandlers/HandshakeResponseHandlers/HandshakeResponseHandler.cs index f0fb6a4e..a93f597b 100644 --- a/Centaurus.Domain/MessageHandlers/HandshakeResponseHandlers/HandshakeResponseHandler.cs +++ b/Centaurus.Domain/MessageHandlers/HandshakeResponseHandlers/HandshakeResponseHandler.cs @@ -8,27 +8,63 @@ namespace Centaurus.Domain { - public class HandshakeResponseHandler : HandshakeResponseHandlerBase + internal class HandshakeResponseHandler : MessageHandlerBase { public HandshakeResponseHandler(ExecutionContext context) : base(context) { } + public override ConnectionState[] ValidConnectionStates { get; } = new ConnectionState[] { ConnectionState.Connected }; + public override string SupportedMessageType { get; } = typeof(HandshakeResponse).Name; - public override async Task HandleMessage(IncomingClientConnection connection, IncomingMessage message) + public override async Task HandleMessage(IncomingConnectionBase connection, IncomingMessage message) { - await base.HandleMessage(connection, message); + var handshakeResponse = message.Envelope.Message as HandshakeResponse; + if (!connection.TryValidate(handshakeResponse.HandshakeData)) + throw new ConnectionCloseException(WebSocketCloseStatus.InvalidPayloadData, "Handshake data is invalid."); + + switch (connection) + { + case IncomingClientConnection clientConnection: + await HandleClientHandshake(clientConnection, message.Envelope); + break; + case IncomingNodeConnection auditorConnection: + await HandleAuditorHandshake(auditorConnection); + break; + default: + throw new BadRequestException("Unknown connection."); + } + } - if (connection.Context.StateManager.State != State.Ready) + private async Task HandleClientHandshake(IncomingClientConnection clientConnection, MessageEnvelopeBase envelope) + { + //if Alpha is not ready, close connection + if (clientConnection.Context.NodesManager.CurrentNode.State != State.Ready) throw new ConnectionCloseException(WebSocketCloseStatus.ProtocolError, "Alpha is not in Ready state."); - if (connection.Account == null) + //if account not presented, throw UnauthorizedException + if (clientConnection.Account == null) throw new UnauthorizedException(); - var result = message.Envelope.CreateResult(ResultStatusCode.Success); - await connection.SendMessage(result); + //send success response + await clientConnection.SendMessage(envelope.CreateResult(ResultStatusCode.Success)); + } + + private async Task HandleAuditorHandshake(IncomingNodeConnection auditorConnection) + { + //register connection + auditorConnection.Node.RegisterIncomingConnection(auditorConnection); + + //request quanta on rising + if (Context.NodesManager.CurrentNode.State == State.Rising) + { + await auditorConnection.SendMessage(new CatchupQuantaBatchRequest + { + LastKnownApex = Context.QuantumHandler.CurrentApex + }.CreateEnvelope()); + } } } } diff --git a/Centaurus.Domain/MessageHandlers/HandshakeResponseHandlers/HandshakeResponseHandlerBase.cs b/Centaurus.Domain/MessageHandlers/HandshakeResponseHandlers/HandshakeResponseHandlerBase.cs deleted file mode 100644 index 79774780..00000000 --- a/Centaurus.Domain/MessageHandlers/HandshakeResponseHandlers/HandshakeResponseHandlerBase.cs +++ /dev/null @@ -1,30 +0,0 @@ -using Centaurus.Models; -using System; -using System.Collections.Generic; -using System.Net.WebSockets; -using System.Text; -using System.Threading.Tasks; - -namespace Centaurus.Domain -{ - public abstract class HandshakeResponseHandlerBase : MessageHandlerBase - where T : IncomingConnectionBase - { - protected HandshakeResponseHandlerBase(ExecutionContext context) - : base(context) - { - } - - public override ConnectionState[] ValidConnectionStates { get; } = new ConnectionState[] { ConnectionState.Connected }; - - - public override Task HandleMessage(T connection, IncomingMessage message) - { - var handshakeRequest = message.Envelope.Message as HandshakeResponseBase; - if (!connection.TryValidate(handshakeRequest.HandshakeData)) - throw new ConnectionCloseException(WebSocketCloseStatus.InvalidPayloadData, "Handshake data is invalid."); - - return Task.CompletedTask; - } - } -} diff --git a/Centaurus.Domain/MessageHandlers/MessageHandlerBase.cs b/Centaurus.Domain/MessageHandlers/MessageHandlerBase.cs index cc126496..9c93ba3b 100644 --- a/Centaurus.Domain/MessageHandlers/MessageHandlerBase.cs +++ b/Centaurus.Domain/MessageHandlers/MessageHandlerBase.cs @@ -7,7 +7,7 @@ namespace Centaurus.Domain { - public abstract class MessageHandlerBase : ContextualBase + internal abstract class MessageHandlerBase : ContextualBase { protected MessageHandlerBase(ExecutionContext context) : base(context) @@ -76,7 +76,7 @@ private void ValidateClient(ConnectionBase connection) public abstract Task HandleMessage(ConnectionBase connection, IncomingMessage message); } - public abstract class MessageHandlerBase : MessageHandlerBase + internal abstract class MessageHandlerBase : MessageHandlerBase where T: ConnectionBase { protected MessageHandlerBase(ExecutionContext context) diff --git a/Centaurus.Domain/MessageHandlers/Quanta/AccountDataRequestHandler.cs b/Centaurus.Domain/MessageHandlers/Quanta/AccountDataRequestHandler.cs index 792122b9..3abdc132 100644 --- a/Centaurus.Domain/MessageHandlers/Quanta/AccountDataRequestHandler.cs +++ b/Centaurus.Domain/MessageHandlers/Quanta/AccountDataRequestHandler.cs @@ -6,7 +6,7 @@ namespace Centaurus.Domain { - public class AccountDataRequestHandler : QuantumHandlerBase + internal class AccountDataRequestHandler : QuantumHandlerBase { public AccountDataRequestHandler(ExecutionContext context) : base(context) diff --git a/Centaurus.Domain/MessageHandlers/Quanta/OrderCancellationMessageHandler.cs b/Centaurus.Domain/MessageHandlers/Quanta/OrderCancellationMessageHandler.cs index 72dc18e7..75957c28 100644 --- a/Centaurus.Domain/MessageHandlers/Quanta/OrderCancellationMessageHandler.cs +++ b/Centaurus.Domain/MessageHandlers/Quanta/OrderCancellationMessageHandler.cs @@ -6,7 +6,7 @@ namespace Centaurus.Domain { - public class OrderCancellationMessageHandler : QuantumHandlerBase + internal class OrderCancellationMessageHandler : QuantumHandlerBase { public OrderCancellationMessageHandler(ExecutionContext context) : base(context) diff --git a/Centaurus.Domain/MessageHandlers/Quanta/OrderMessageHandler.cs b/Centaurus.Domain/MessageHandlers/Quanta/OrderMessageHandler.cs index fac28505..436a66b6 100644 --- a/Centaurus.Domain/MessageHandlers/Quanta/OrderMessageHandler.cs +++ b/Centaurus.Domain/MessageHandlers/Quanta/OrderMessageHandler.cs @@ -6,7 +6,7 @@ namespace Centaurus.Domain { - public class OrderMessageHandler : QuantumHandlerBase + internal class OrderMessageHandler : QuantumHandlerBase { public OrderMessageHandler(ExecutionContext context) : base(context) diff --git a/Centaurus.Domain/MessageHandlers/Quanta/PaymentMessageHandler.cs b/Centaurus.Domain/MessageHandlers/Quanta/PaymentMessageHandler.cs index 934e12b5..392346a4 100644 --- a/Centaurus.Domain/MessageHandlers/Quanta/PaymentMessageHandler.cs +++ b/Centaurus.Domain/MessageHandlers/Quanta/PaymentMessageHandler.cs @@ -5,7 +5,7 @@ namespace Centaurus.Domain { - public class PaymentMessageHandler : QuantumHandlerBase + internal class PaymentMessageHandler : QuantumHandlerBase { public PaymentMessageHandler(ExecutionContext context) : base(context) diff --git a/Centaurus.Domain/MessageHandlers/Quanta/QuantumHandlerBase.cs b/Centaurus.Domain/MessageHandlers/Quanta/QuantumHandlerBase.cs index 4794160c..59910915 100644 --- a/Centaurus.Domain/MessageHandlers/Quanta/QuantumHandlerBase.cs +++ b/Centaurus.Domain/MessageHandlers/Quanta/QuantumHandlerBase.cs @@ -10,7 +10,7 @@ namespace Centaurus.Domain /// /// This handler should handle all quantum requests /// - public abstract class QuantumHandlerBase : MessageHandlerBase + internal abstract class QuantumHandlerBase : MessageHandlerBase { protected QuantumHandlerBase(ExecutionContext context) : base(context) diff --git a/Centaurus.Domain/MessageHandlers/Quanta/WithdrawalMessageHandler.cs b/Centaurus.Domain/MessageHandlers/Quanta/WithdrawalMessageHandler.cs index b0a4ca0d..56f159d0 100644 --- a/Centaurus.Domain/MessageHandlers/Quanta/WithdrawalMessageHandler.cs +++ b/Centaurus.Domain/MessageHandlers/Quanta/WithdrawalMessageHandler.cs @@ -6,7 +6,7 @@ namespace Centaurus.Domain { - public class WithdrawalMessageHandler : QuantumHandlerBase + internal class WithdrawalMessageHandler : QuantumHandlerBase { public WithdrawalMessageHandler(ExecutionContext context) : base(context) diff --git a/Centaurus.Domain/MessageHandlers/QuantumMajoritySignaturesBatchHandler.cs b/Centaurus.Domain/MessageHandlers/QuantumMajoritySignaturesBatchHandler.cs index 20acb9da..f0a1d315 100644 --- a/Centaurus.Domain/MessageHandlers/QuantumMajoritySignaturesBatchHandler.cs +++ b/Centaurus.Domain/MessageHandlers/QuantumMajoritySignaturesBatchHandler.cs @@ -7,7 +7,7 @@ namespace Centaurus.Domain { - public class QuantumMajoritySignaturesBatchHandler : MessageHandlerBase + internal class QuantumMajoritySignaturesBatchHandler : MessageHandlerBase { static Logger logger = LogManager.GetCurrentClassLogger(); diff --git a/Centaurus.Domain/MessageHandlers/RequestQuantaBatchHandler.cs b/Centaurus.Domain/MessageHandlers/RequestQuantaBatchHandler.cs index 732ac59e..1c7c6ab6 100644 --- a/Centaurus.Domain/MessageHandlers/RequestQuantaBatchHandler.cs +++ b/Centaurus.Domain/MessageHandlers/RequestQuantaBatchHandler.cs @@ -6,7 +6,7 @@ namespace Centaurus.Domain { - public class RequestQuantaBatchHandler : MessageHandlerBase + internal class RequestQuantaBatchHandler : MessageHandlerBase { public RequestQuantaBatchHandler(ExecutionContext context) : base(context) @@ -16,7 +16,7 @@ public RequestQuantaBatchHandler(ExecutionContext context) public override string SupportedMessageType { get; } = typeof(RequestQuantaBatch).Name; - public override Task HandleMessage(IncomingAuditorConnection connection, IncomingMessage message) + public override Task HandleMessage(IncomingNodeConnection connection, IncomingMessage message) { var requests = (RequestQuantaBatch)message.Envelope.Message; if (Context.IsAlpha) diff --git a/Centaurus.Domain/MessageHandlers/ResultBatchHandler.cs b/Centaurus.Domain/MessageHandlers/ResultBatchHandler.cs index 0845ddb4..f8254364 100644 --- a/Centaurus.Domain/MessageHandlers/ResultBatchHandler.cs +++ b/Centaurus.Domain/MessageHandlers/ResultBatchHandler.cs @@ -4,7 +4,7 @@ namespace Centaurus.Domain { - public class ResultBatchHandler : MessageHandlerBase + internal class ResultBatchHandler : MessageHandlerBase { public ResultBatchHandler(ExecutionContext context) : base(context) @@ -24,7 +24,7 @@ public override Task HandleMessage(ConnectionBase connection, IncomingMessage me Context.ResultManager.Add(new QuantumSignatures { Apex = result.Apex, - Signatures = new List { result.Signature } + Signatures = new List { result.Signature } }); return Task.CompletedTask; } diff --git a/Centaurus.Domain/MessageHandlers/StateUpdateMessageHandler.cs b/Centaurus.Domain/MessageHandlers/StateMessageHandler.cs similarity index 52% rename from Centaurus.Domain/MessageHandlers/StateUpdateMessageHandler.cs rename to Centaurus.Domain/MessageHandlers/StateMessageHandler.cs index 32193df3..58b3f95e 100644 --- a/Centaurus.Domain/MessageHandlers/StateUpdateMessageHandler.cs +++ b/Centaurus.Domain/MessageHandlers/StateMessageHandler.cs @@ -3,21 +3,24 @@ namespace Centaurus.Domain { - public class StateUpdateMessageHandler : MessageHandlerBase + internal class StateMessageHandler : MessageHandlerBase { - public StateUpdateMessageHandler(ExecutionContext executionContext) + public StateMessageHandler(ExecutionContext executionContext) : base(executionContext) { } - public override string SupportedMessageType => typeof(StateUpdateMessage).Name; + public override string SupportedMessageType => typeof(StateMessage).Name; public override bool IsAuditorOnly => true; + public override ConnectionState[] ValidConnectionStates => new[] { ConnectionState.Ready }; + public override Task HandleMessage(ConnectionBase connection, IncomingMessage message) { - Context.StateManager.SetAuditorState(connection.PubKey, ((StateUpdateMessage)message.Envelope.Message).State); + var nodeConnection = (INodeConnection)connection; + nodeConnection.Node.Update((StateMessage)message.Envelope.Message); return Task.CompletedTask; } } diff --git a/Centaurus.Domain/MessageHandlers/SyncCursorResetHandler.cs b/Centaurus.Domain/MessageHandlers/SyncCursorResetHandler.cs index 91414acf..bf996264 100644 --- a/Centaurus.Domain/MessageHandlers/SyncCursorResetHandler.cs +++ b/Centaurus.Domain/MessageHandlers/SyncCursorResetHandler.cs @@ -9,7 +9,7 @@ namespace Centaurus.Domain.Handlers.AlphaHandlers { - public class SyncCursorResetHandler : MessageHandlerBase + internal class SyncCursorResetHandler : MessageHandlerBase { public SyncCursorResetHandler(ExecutionContext context) : base(context) @@ -25,11 +25,19 @@ public SyncCursorResetHandler(ExecutionContext context) ConnectionState.Ready }; - public override Task HandleMessage(IncomingAuditorConnection connection, IncomingMessage message) + public override Task HandleMessage(IncomingNodeConnection connection, IncomingMessage message) { var cursorResetRequest = (SyncCursorReset)message.Envelope.Message; - connection.SetSyncCursor(true, cursorResetRequest.SyncCursors.ToDomainModel().ToArray()); + foreach (var cursor in cursorResetRequest.Cursors) + { + var cursorType = cursor.Type.ToDomainCursorType(); + if (cursor.DisableSync) + connection.Node.DisableSync(cursorType); + else + connection.Node.SetCursor(cursorType, default, cursor.Cursor, true); + } + return Task.CompletedTask; } } diff --git a/Centaurus.Domain/MessageHandlers/AlphaQuantaBatchHandler.cs b/Centaurus.Domain/MessageHandlers/SyncQuantaBatchHandler.cs similarity index 85% rename from Centaurus.Domain/MessageHandlers/AlphaQuantaBatchHandler.cs rename to Centaurus.Domain/MessageHandlers/SyncQuantaBatchHandler.cs index 9df00182..0f3a5579 100644 --- a/Centaurus.Domain/MessageHandlers/AlphaQuantaBatchHandler.cs +++ b/Centaurus.Domain/MessageHandlers/SyncQuantaBatchHandler.cs @@ -8,11 +8,11 @@ namespace Centaurus.Domain { - public class AlphaQuantaBatchHandler : MessageHandlerBase + internal class SyncQuantaBatchHandler : MessageHandlerBase { static Logger logger = LogManager.GetCurrentClassLogger(); - public AlphaQuantaBatchHandler(ExecutionContext context) + public SyncQuantaBatchHandler(ExecutionContext context) : base(context) { } @@ -33,9 +33,6 @@ private async Task AddQuantaToQueue(OutgoingConnection connection, IncomingMessa var quantumHandler = Context.QuantumHandler; var quantaBatch = (SyncQuantaBatch)message.Envelope.Message; - //update alpha apex - Context.StateManager.UpdateAlphaApex(quantaBatch.LastKnownApex); - //get last known apex var lastKnownApex = quantumHandler.LastAddedQuantumApex; @@ -53,7 +50,7 @@ private async Task AddQuantaToQueue(OutgoingConnection connection, IncomingMessa { await connection.SendMessage(new SyncCursorReset { - SyncCursors = new List { + Cursors = new List { new SyncCursor { Type = XdrSyncCursorType.Quanta, Cursor = quantumHandler.LastAddedQuantumApex @@ -68,7 +65,7 @@ await connection.SendMessage(new SyncCursorReset Context.ResultManager.Add(new QuantumSignatures { Apex = quantum.Apex, - Signatures = new List { processedQuantum.AlphaSignature } + Signatures = new List { processedQuantum.AlphaSignature } }); lastKnownApex = quantum.Apex; } diff --git a/Centaurus.Domain/Nodes/Common/NodeApexes.cs b/Centaurus.Domain/Nodes/Common/NodeApexes.cs new file mode 100644 index 00000000..c93fcb3e --- /dev/null +++ b/Centaurus.Domain/Nodes/Common/NodeApexes.cs @@ -0,0 +1,29 @@ +namespace Centaurus.Domain.StateManagers +{ + internal class NodeApexes : ValueAggregation + { + public NodeApexes() + :base(20) + { + + } + + /// + /// Calculates avg quanta per second + /// + /// + public override int GetAvg() + { + var data = GetData(); + if (data.Count < 2) + return default; + var firstItem = data[0]; + var lastItem = data[data.Count - 1]; + var valueDiff = lastItem.Value - firstItem.Value; + if (valueDiff == 0) + return 0; + var timeDiff = (decimal)(lastItem.AddedAt - firstItem.AddedAt).TotalSeconds; + return (int)decimal.Divide(valueDiff, timeDiff); + } + } +} \ No newline at end of file diff --git a/Centaurus.Domain/Nodes/Common/NodeBase.cs b/Centaurus.Domain/Nodes/Common/NodeBase.cs new file mode 100644 index 00000000..04ad8573 --- /dev/null +++ b/Centaurus.Domain/Nodes/Common/NodeBase.cs @@ -0,0 +1,48 @@ +using Centaurus.Domain.Quanta.Sync; +using Centaurus.Domain.StateManagers; +using Centaurus.Models; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Centaurus.Domain +{ + public abstract class NodeBase: ContextualBase + { + public NodeBase(ExecutionContext context, RawPubKey rawPubKey) + :base(context) + { + PubKey = rawPubKey ?? throw new ArgumentNullException(nameof(rawPubKey)); + AccountId = rawPubKey.GetAccountId(); + } + + public RawPubKey PubKey { get; } + + public string AccountId { get; } + + public State State { get; protected set; } = State.Undefined; + + public ulong LastApex => apexes.LastValue; + + public int QuantaPerSecond => apexes.GetAvg(); + + public int QuantaQueueLength => quantaQueueLengths.GetAvg(); + + public ulong LastPersistedApex { get; protected set; } + + public DateTime UpdateDate { get; protected set; } + + protected virtual void SetApex(DateTime updateDate, ulong apex) + { + apexes.AddValue(updateDate, apex); + } + + protected virtual void SetQuantaQueueLength(DateTime updateDate, int queueLength) + { + quantaQueueLengths.AddValue(updateDate, queueLength); + } + + private NodeQuantaQueueLengths quantaQueueLengths = new NodeQuantaQueueLengths(); + private NodeApexes apexes = new NodeApexes(); + } +} \ No newline at end of file diff --git a/Centaurus.Domain/Nodes/Common/NodeExtensions.cs b/Centaurus.Domain/Nodes/Common/NodeExtensions.cs new file mode 100644 index 00000000..6aa7bc4e --- /dev/null +++ b/Centaurus.Domain/Nodes/Common/NodeExtensions.cs @@ -0,0 +1,49 @@ +using Centaurus.Domain.StateManagers; +using Centaurus.Models; +using System; + +namespace Centaurus.Domain +{ + internal static class NodeExtensions + { + public static bool IsRunning(this NodeBase node) + { + if (node == null) + throw new ArgumentNullException(nameof(node)); + + var currentState = node.State; + return currentState == State.Running || currentState == State.Ready || currentState == State.Chasing; + } + + public static bool IsWaitingForInit(this NodeBase node) + { + if (node == null) + throw new ArgumentNullException(nameof(node)); + + return node.State == State.WaitingForInit; + } + + public static bool IsReady(this NodeBase node) + { + if (node == null) + throw new ArgumentNullException(nameof(node)); + + return node.State == State.Ready; + } + + public static bool IsReadyToHandleQuanta(this NodeBase node) + { + if (node == null) + throw new ArgumentNullException(nameof(node)); + + return node.IsRunning() || node.IsWaitingForInit(); + } + + public static ConnectionBase GetConnection(this RemoteNode node) + { + if (node == null) + throw new ArgumentNullException(nameof(node)); + return (ConnectionBase)node.IncomingConnection ?? node.OutgoingConnection; + } + } +} diff --git a/Centaurus.Domain/Nodes/Common/NodeQuantaQueueLengths.cs b/Centaurus.Domain/Nodes/Common/NodeQuantaQueueLengths.cs new file mode 100644 index 00000000..ee9cb444 --- /dev/null +++ b/Centaurus.Domain/Nodes/Common/NodeQuantaQueueLengths.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Centaurus.Domain.StateManagers +{ + internal class NodeQuantaQueueLengths : ValueAggregation + { + public NodeQuantaQueueLengths() + :base(20) + { + + } + + public override int GetAvg() + { + var data = GetData(); + if (data.Count < 1) + return 0; + + return (int)data.Average(d => d.Value); + } + } +} diff --git a/Centaurus.Domain/Nodes/Common/StateChangedEventArgs.cs b/Centaurus.Domain/Nodes/Common/StateChangedEventArgs.cs new file mode 100644 index 00000000..8929c50f --- /dev/null +++ b/Centaurus.Domain/Nodes/Common/StateChangedEventArgs.cs @@ -0,0 +1,21 @@ +using Centaurus.Models; +using System; + +namespace Centaurus.Domain +{ + public class StateChangedEventArgs : EventArgs + { + public StateChangedEventArgs(NodeBase source, State state, State prevState) + { + Source = source ?? throw new ArgumentNullException(nameof(source)); + State = state; + PrevState = prevState; + } + + public NodeBase Source { get; } + + public State State { get; } + + public State PrevState { get; } + } +} \ No newline at end of file diff --git a/Centaurus.Domain/Nodes/Common/SyncCursorCollection.cs b/Centaurus.Domain/Nodes/Common/SyncCursorCollection.cs new file mode 100644 index 00000000..f585f7a0 --- /dev/null +++ b/Centaurus.Domain/Nodes/Common/SyncCursorCollection.cs @@ -0,0 +1,42 @@ +using Centaurus.Domain.Quanta.Sync; +using System.Collections.Generic; +using System.Linq; + +namespace Centaurus.Domain.StateManagers +{ + internal class SyncCursorCollection + { + public RemoteNodeCursor Get(SyncCursorType cursorType, bool createIfNotExist = false) + { + var cursor = default(RemoteNodeCursor); + lock (syncRoot) + if (!cursors.TryGetValue(cursorType, out cursor) && createIfNotExist) + { + cursor = new RemoteNodeCursor(cursorType); + cursors.Add(cursorType, cursor); + } + return cursor; + } + + public void Remove(SyncCursorType cursorType) + { + lock (syncRoot) + cursors.Remove(cursorType); + } + + public List GetActiveCursors() + { + lock (syncRoot) + return cursors.Values.Where(c => c.IsSyncEnabled).ToList(); + } + + public void Clear() + { + lock (syncRoot) + cursors.Clear(); + } + + private object syncRoot = new { }; + private Dictionary cursors = new Dictionary(); + } +} diff --git a/Centaurus.Domain/Nodes/Common/ValueAggregation.cs b/Centaurus.Domain/Nodes/Common/ValueAggregation.cs new file mode 100644 index 00000000..80dad63b --- /dev/null +++ b/Centaurus.Domain/Nodes/Common/ValueAggregation.cs @@ -0,0 +1,54 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Centaurus.Domain.StateManagers +{ + internal abstract class ValueAggregation + { + protected ValueAggregation(int maxItems) + { + if (maxItems < 2) + throw new ArgumentException("Data must contain at least two items to be able to calculate avg value.", nameof(maxItems)); + } + + private List> data = new List>(); + + private object syncRoot = new { }; + + public void AddValue(DateTime dateTime, T value) + { + lock (syncRoot) + { + data.Add(new Item { AddedAt = dateTime, Value = value }); + LastValue = value; + if (data.Count > 20) //remove old data + data.RemoveAt(0); + } + } + + public T LastValue { get; private set; } + + public void Clear() + { + lock (syncRoot) + data.Clear(); + } + + public abstract int GetAvg(); + + protected List> GetData() + { + lock (syncRoot) + return data.ToList(); + } + + protected struct Item + { + public DateTime AddedAt { get; set; } + + public T Value { get; set; } + } + } +} diff --git a/Centaurus.Domain/Nodes/CurrentNode/CurrentNode.cs b/Centaurus.Domain/Nodes/CurrentNode/CurrentNode.cs new file mode 100644 index 00000000..b4d7064a --- /dev/null +++ b/Centaurus.Domain/Nodes/CurrentNode/CurrentNode.cs @@ -0,0 +1,128 @@ +using Centaurus.Models; +using NLog; +using System; + +namespace Centaurus.Domain.StateManagers +{ + internal class CurrentNode : NodeBase + { + public CurrentNode(ExecutionContext context, RawPubKey rawPubKey, State initState) + :base(context, rawPubKey) + { + State = initState; + } + + public event Action StateChanged; + + public void Init(State state) + { + lock (syncRoot) + { + if (State != State.WaitingForInit) + throw new InvalidOperationException("Context is already initialized."); + + if (state == State.WaitingForInit || state == State.Undefined) + throw new InvalidOperationException($"Init state cannot be {state}."); + + UpdateState(state); + } + } + + public void Stopped() + { + lock (syncRoot) + SetState(State.Stopped); + } + + public void Failed(Exception exc) + { + lock (syncRoot) + SetState(State.Failed, exc); + } + + public void Rised() + { + lock (syncRoot) + UpdateState(State.Running); + } + + public void RefreshState() + { + UpdateState(State); + } + + public void UpdateData(ulong currentApex, ulong lastPersistedApex, int quantaQueueLenght, DateTime updateDate) + { + lock (syncRoot) + { + SetApex(updateDate, currentApex); + SetQuantaQueueLength(updateDate, quantaQueueLenght); + LastPersistedApex = lastPersistedApex; + UpdateDelay(); + UpdateDate = updateDate; + } + } + + /// + /// Updates current node delay + /// + private void UpdateDelay() + { + if (Context.IsAlpha) + return; + lock (syncRoot) + { + var isDelayed = State.Chasing == State; + var currentApex = Context.QuantumHandler.CurrentApex; + var syncApex = Context.NodesManager.SyncSource.LastApex; + if (isDelayed) + { + if (syncApex <= currentApex || syncApex - currentApex < RunningDelayTreshold) + UpdateState(State.Running); + logger.Info($"Current node delay is {syncApex - currentApex}"); + } + else + { + if (syncApex > currentApex && syncApex - currentApex > ChasingDelayTreshold) + { + logger.Info($"Current node delay is {syncApex - currentApex}"); + UpdateState(State.Chasing); + } + } + } + } + + private const ulong ChasingDelayTreshold = 50_000; + private const ulong RunningDelayTreshold = 10_000; + private object syncRoot = new { }; + private static Logger logger = LogManager.GetCurrentClassLogger(); + + private void UpdateState(State state) + { + lock (syncRoot) + { + if (state == State.Running && Context.NodesManager.IsMajorityReady) + SetState(State.Ready); + else if (state == State.Ready && Context.NodesManager.IsMajorityReady) + SetState(State.Running); + else + SetState(state); + } + } + + private void SetState(State state, Exception exc = null) + { + if (exc != null) + logger.Error(exc); + if (State != state) + { + logger.Info($"State update: new state: {state}, prev state: {State}"); + var stateArgs = new StateChangedEventArgs(this, state, State); + State = state; + + StateChanged?.Invoke(stateArgs); + UpdateDate = DateTime.UtcNow; + } + } + } +} diff --git a/Centaurus.Domain/Nodes/Managers/NodesManager.cs b/Centaurus.Domain/Nodes/Managers/NodesManager.cs new file mode 100644 index 00000000..f001a0de --- /dev/null +++ b/Centaurus.Domain/Nodes/Managers/NodesManager.cs @@ -0,0 +1,207 @@ +using Centaurus.Domain.StateManagers; +using Centaurus.Models; +using NLog; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Centaurus.Domain +{ + internal class NodesManager : ContextualBase + { + static Logger logger = LogManager.GetCurrentClassLogger(); + public NodesManager(ExecutionContext context, State currentNodeInitState) + : base(context) + { + nodes = new RemoteNodesCollection(); + nodes.OnAdd += Nodes_OnAdd; + nodes.OnRemove += Nodes_OnRemove; + CurrentNode = new CurrentNode(Context, Context.Settings.KeyPair, currentNodeInitState); + } + + public RemoteNode SyncSource { get; private set; } + + public CurrentNode CurrentNode { get; } + + public bool IsMajorityReady { get; private set; } + + public List GetRemoteNodes() + { + return nodes.GetAllNodes(); + } + + //TODO: handle address switch + public void SetAuditors(List auditors) + { + var currentAuditorKey = (RawPubKey)Context.Settings.KeyPair; + var ueHttps = Context.Settings.UseSecureConnection; + var newNodes = new Dictionary>(); + foreach (var auditor in auditors) + { + var pubKey = auditor.PubKey; + if (pubKey.Equals(currentAuditorKey)) + continue; + + UriHelper.TryCreateWsConnection(auditor.Address, ueHttps, out var uri); + newNodes.Add(pubKey, new Lazy(() => new RemoteNode(Context, pubKey, uri))); + } + nodes.SetNodes(newNodes); + } + + public void EnqueueResult(AuditorResult result) + { + foreach (var node in GetRemoteNodes()) + node.EnqueueResult(result); + } + + public void EnqueueMessage(MessageEnvelopeBase message) + { + foreach (var node in GetRemoteNodes()) + { + node.EnqueueMessage(message); + } + } + + public bool TryGetNode(RawPubKey rawPubKey, out RemoteNode node) + { + return nodes.TryGetNode(rawPubKey, out node); + } + + private void Nodes_OnAdd(RemoteNode node) + { + node.OnStateUpdated += Node_OnStateUpdated; + node.OnLastApexUpdated += Node_OnLastApexUpdated; + node.Connect(); + CalcMajorityReadiness(); + } + + private void Nodes_OnRemove(RemoteNode node) + { + node.OnStateUpdated -= Node_OnStateUpdated; + node.OnLastApexUpdated -= Node_OnLastApexUpdated; + node.CloseOugoingConnection(); + CalcMajorityReadiness(); + } + + private void Node_OnLastApexUpdated(RemoteNode node) + { + //update current sync node + if (!Context.IsAlpha && node.LastApex > 0) + { + var syncSourceLastApex = SyncSource.LastApex; + if (SyncSource == null || + syncSourceLastApex < node.LastApex //if the current node ahead + && node.LastApex - syncSourceLastApex > 1000 //and the apexes difference greater than 1000 + && (DateTime.UtcNow - SyncSourceUpdateDate) > TimeSpan.FromSeconds(1)) //and the last sync source update was later than second ago + { + SetSyncSource(node); + } + } + } + + private void Node_OnStateUpdated(RemoteNode node) + { + CalcMajorityReadiness(); + } + + private DateTime SyncSourceUpdateDate; + private object syncSourceSyncRoot = new { }; + + private readonly RemoteNodesCollection nodes; + + private void SetSyncSource(RemoteNode auditorState) + { + lock (syncSourceSyncRoot) + { + ClearCurrentSyncCursor(); + + SyncSource = auditorState; + SyncSourceUpdateDate = DateTime.UtcNow; + + SetCurrentSyncCursor(); + } + } + + private void ClearCurrentSyncCursor() + { + if (SyncSource != null) + Context.Notify(SyncSource.PubKey, new SyncCursorReset + { + Cursors = new List { + new SyncCursor { Type = XdrSyncCursorType.Quanta, DisableSync = true }, + new SyncCursor { Type = XdrSyncCursorType.Signatures, DisableSync = true }, + } + }.CreateEnvelope()); + } + + private void SetCurrentSyncCursor() + { + if (SyncSource != null) + Context.Notify(SyncSource.PubKey, new SyncCursorReset + { + Cursors = new List { + new SyncCursor { Type = XdrSyncCursorType.Quanta, Cursor = Context.QuantumHandler.LastAddedQuantumApex }, + new SyncCursor { Type = XdrSyncCursorType.Signatures, Cursor = Context.PendingUpdatesManager.LastPersistedApex }, + } + }.CreateEnvelope()); + } + + private void CalcMajorityReadiness() + { + var majorityReadiness = false; + //if Prime node than it must be connected with other nodes + if (Context.RoleManager.ParticipationLevel == CentaurusNodeParticipationLevel.Prime) + { + majorityReadiness = GetReadinessForPrimeNode(); + } + else + majorityReadiness = GetReadinessForAuditorNode(); + + TrySetReadiness(majorityReadiness); + } + + private void TrySetReadiness(bool majorityReadiness) + { + if (majorityReadiness != IsMajorityReady) + { + IsMajorityReady = majorityReadiness; + //update current node state + CurrentNode.RefreshState(); + } + } + + private bool GetReadinessForPrimeNode() + { + //if current server is Alpha, than ignore Alpha validation + var isAlphaReady = Context.IsAlpha; + var connectedCount = 0; + foreach (var node in nodes.GetAllNodes()) + { + if (!(node.State == State.Ready || node.State == State.Running)) + continue; + if (Context.Constellation.Alpha.Equals(node.PubKey)) + isAlphaReady = true; + connectedCount++; + } + return isAlphaReady && Context.HasMajority(connectedCount, false); + } + + internal void ClearNodes() + { + nodes.Clear(); + } + + private bool GetReadinessForAuditorNode() + { + foreach (var node in nodes.GetAllNodes()) + { + if (!node.PubKey.Equals(Context.Constellation.Alpha)) + continue; + //if auditor doesn't have connections with another auditors, we only need to verify alpha's state + return node.State == State.Ready || node.State == State.Running; + } + return false; + } + } +} \ No newline at end of file diff --git a/Centaurus.Domain/Nodes/Managers/StateNotifierWorker.cs b/Centaurus.Domain/Nodes/Managers/StateNotifierWorker.cs new file mode 100644 index 00000000..97be30c2 --- /dev/null +++ b/Centaurus.Domain/Nodes/Managers/StateNotifierWorker.cs @@ -0,0 +1,60 @@ +using Centaurus.Models; +using NLog; +using System; +using System.Timers; + +namespace Centaurus.Domain +{ + + internal class StateNotifierWorker : ContextualBase + { + private Logger logger = LogManager.GetCurrentClassLogger(); + + public StateNotifierWorker(ExecutionContext context) + : base(context) + { + InitBroadcastTimer(); + } + + private Timer broadcastTimer; + + private void InitBroadcastTimer() + { + broadcastTimer = new Timer(); + broadcastTimer.AutoReset = false; + broadcastTimer.Interval = TimeSpan.FromSeconds(1).TotalMilliseconds; + broadcastTimer.Elapsed += BroadcastTimer_Elapsed; + broadcastTimer.Start(); + } + + private void BroadcastTimer_Elapsed(object sender, ElapsedEventArgs e) + { + try + { + var currentNode = Context.NodesManager.CurrentNode; + var currentApex = Context.QuantumHandler.CurrentApex; + var lastPersistedApex = Context.PendingUpdatesManager.LastPersistedApex; + var quantaQueueLenght = Context.QuantumHandler.QuantaQueueLenght; + var updateDate = DateTime.UtcNow; + var updateMessage = new StateMessage + { + State = currentNode.State, + CurrentApex = currentApex, + LastPersistedApex = lastPersistedApex, + QuantaQueueLength = quantaQueueLenght, + UpdateDate = updateDate + }; + currentNode.UpdateData(currentApex, lastPersistedApex, quantaQueueLenght, updateDate); + Context.NotifyAuditors(updateMessage.CreateEnvelope()); + } + catch (Exception exc) + { + logger.Error(exc, "Error on the state broadcasting."); + } + finally + { + broadcastTimer.Start(); + } + } + } +} \ No newline at end of file diff --git a/Centaurus.Domain/Nodes/RemoteNode/RemoteNode.cs b/Centaurus.Domain/Nodes/RemoteNode/RemoteNode.cs new file mode 100644 index 00000000..cc6867ed --- /dev/null +++ b/Centaurus.Domain/Nodes/RemoteNode/RemoteNode.cs @@ -0,0 +1,142 @@ +using Centaurus.Domain.Quanta.Sync; +using Centaurus.Models; +using System; +using System.Collections.Generic; + +namespace Centaurus.Domain.StateManagers +{ + internal partial class RemoteNode : NodeBase + { + public RemoteNode(ExecutionContext context, RawPubKey rawPubKey, Uri address) + : base(context, rawPubKey) + { + Address = address; + } + + private Uri address; + public Uri Address + { + get => address; + private set + { + if (value == address) + return; + + address = value; + + if (atomicConnection != null) + { + atomicConnection.Shutdown(); + atomicConnection = null; + } + if (value != null) + atomicConnection = new RemoteNodeConnection(this); + } + } + + private RemoteNodeConnection atomicConnection; + + public event Action OnStateUpdated; + + public event Action OnLastApexUpdated; + + public void RegisterIncomingConnection(IncomingNodeConnection connection) + { + if (connection == null) + throw new ArgumentNullException(nameof(connection)); + + IncomingConnection = connection; + } + + public void RemoveIncomingConnection() + { + IncomingConnection = null; + if (this.GetConnection() == null) + { + State = State.Undefined; + cursors.Clear(); + } + } + + public void Connect() + { + atomicConnection?.Run(); + } + + public void CloseOugoingConnection() + { + if (atomicConnection != null) + { + atomicConnection.Shutdown(); + atomicConnection = null; + } + } + + public void EnqueueResult(AuditorResult result) + { + atomicConnection?.OutgoingResultsStorage.EnqueueResult(result); + } + + public void EnqueueMessage(MessageEnvelopeBase message) + { + atomicConnection?.OutgoingMessageStorage.EnqueueMessage(message); + } + + public IncomingNodeConnection IncomingConnection { get; private set; } + + public OutgoingConnection OutgoingConnection => atomicConnection?.Connection; + + public List GetActiveCursors() + { + return cursors.GetActiveCursors(); + } + + public void SetCursor(SyncCursorType cursorType, DateTime timeToken, ulong currentCursor, bool force = false) + { + var cursor = cursors.Get(cursorType, true); + cursor.SetCursor(currentCursor, timeToken, force); + } + + public void DisableSync(SyncCursorType cursorType) + { + var cursor = cursors.Get(cursorType); + cursor?.DisableSync(); + } + + public void Update(StateMessage stateMessage) + { + lock (syncRoot) + { + //skip old data + if (UpdateDate > stateMessage.UpdateDate) + return; + SetState(stateMessage); + SetApex(stateMessage.UpdateDate, stateMessage.CurrentApex); + SetQuantaQueueLength(stateMessage.UpdateDate, stateMessage.QuantaQueueLength); + LastPersistedApex = stateMessage.LastPersistedApex; + UpdateDate = stateMessage.UpdateDate; + } + } + + protected override void SetApex(DateTime updateDate, ulong apex) + { + var lastApex = LastApex; + base.SetApex(updateDate, apex); + if (lastApex != LastApex) + OnLastApexUpdated?.Invoke(this); + } + + private void SetState(StateMessage stateMessage) + { + if (State != stateMessage.State) + { + State = stateMessage.State; + OnStateUpdated?.Invoke(this); + } + } + + private object syncRoot = new { }; + + private SyncCursorCollection cursors = new SyncCursorCollection(); + } +} \ No newline at end of file diff --git a/Centaurus.Domain/Nodes/RemoteNode/RemoteNodeConnection.cs b/Centaurus.Domain/Nodes/RemoteNode/RemoteNodeConnection.cs new file mode 100644 index 00000000..dc72e327 --- /dev/null +++ b/Centaurus.Domain/Nodes/RemoteNode/RemoteNodeConnection.cs @@ -0,0 +1,117 @@ +using Centaurus.Models; +using NLog; +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Web; + +namespace Centaurus.Domain.StateManagers +{ + internal class RemoteNodeConnection : ContextualBase + { + public RemoteNodeConnection(RemoteNode node) + : base(node.Context) + { + Node = node ?? throw new ArgumentNullException(nameof(node)); + + OutgoingMessageStorage = new OutgoingMessageStorage(); + OutgoingResultsStorage = new OutgoingResultsStorage(OutgoingMessageStorage); + } + + public RemoteNode Node { get; } + + public OutgoingMessageStorage OutgoingMessageStorage { get; } + + public OutgoingResultsStorage OutgoingResultsStorage { get; } + + public void Run() + { + try + { + ConnectToAuditor(); + } + catch (Exception exc) + { + logger.Error(exc); + throw; + } + } + + public void Shutdown() + { + lock (syncRoot) + { + isAborted = true; + Connection?.CloseConnection(); + OutgoingResultsStorage.Dispose(); + } + } + + public RawPubKey PubKey => Node.PubKey; + + public Uri Address => Node.Address; + + private void ConnectToAuditor() + { + Task.Factory.StartNew(async () => + { + Uri connectionUri = GetConnectionUri(); + var connectionAttempts = 0; + var listenTask = default(Task); + while (!isAborted) + { + //TODO: remove this condition after refactoring result message broadcasting + //wait while all pending quanta will be handled + if (Context.NodesManager.CurrentNode.State == State.Rising) + { + Thread.Sleep(1000); + continue; + } + lock (syncRoot) + { + if (!isAborted) + Connection = new OutgoingConnection(Context, Node, OutgoingMessageStorage, Context.OutgoingConnectionFactory.GetConnection()); + } + try + { + await Connection.EstablishConnection(connectionUri); + await Connection.Listen(); + } + catch (Exception exc) + { + + if (!(exc is OperationCanceledException) && connectionAttempts % 100 == 0) + logger.Error(exc, $"Unable establish connection with {connectionUri} after {connectionAttempts} attempts. Retry in 1000ms"); + Thread.Sleep(1000); + connectionAttempts++; + } + finally + { + Connection.Dispose(); + Connection = null; + } + } + }); + } + + private Uri GetConnectionUri() + { + var uriBuilder = new UriBuilder(Address); + var query = HttpUtility.ParseQueryString(uriBuilder.Query); + query[WebSocketConstants.PubkeyParamName] = Context.Settings.KeyPair.AccountId; + uriBuilder.Query = query.ToString(); + + var connectionUri = uriBuilder.Uri; + return connectionUri; + } + + private bool isAborted = false; + + public OutgoingConnection Connection { get; private set; } + + private static Logger logger = LogManager.GetCurrentClassLogger(); + private object syncRoot = new { }; + } +} diff --git a/Centaurus.Domain/Nodes/RemoteNode/RemoteNodeCursor.cs b/Centaurus.Domain/Nodes/RemoteNode/RemoteNodeCursor.cs new file mode 100644 index 00000000..afc1a064 --- /dev/null +++ b/Centaurus.Domain/Nodes/RemoteNode/RemoteNodeCursor.cs @@ -0,0 +1,54 @@ +using Centaurus.Domain.Quanta.Sync; +using System; + +namespace Centaurus.Domain +{ + internal class RemoteNodeCursor + { + private object syncRoot = new { }; + + public RemoteNodeCursor(SyncCursorType cursorType) + { + CursorType = cursorType; + } + + public SyncCursorType CursorType { get; } + + public ulong Cursor { get; private set; } + + public DateTime UpdateDate { get; private set; } + + /// + /// Disable data sync + /// + public void DisableSync() + { + lock (syncRoot) + { + IsSyncEnabled = false; + UpdateDate = DateTime.UtcNow; + } + } + + /// + /// + /// + /// Time token to validate that node didn't requested new cursor + /// New cursor value + /// Set true to skip time token validation + public void SetCursor(ulong cursor, DateTime timeToken, bool force = false) + { + //set new cursor + lock (syncRoot) + { + if (!(force || timeToken == UpdateDate)) + return; + Cursor = cursor; + IsSyncEnabled = true; + UpdateDate = DateTime.UtcNow; + } + } + + public bool IsSyncEnabled { get; private set; } + } +} \ No newline at end of file diff --git a/Centaurus.Domain/Nodes/RemoteNode/RemoteNodesCollection.cs b/Centaurus.Domain/Nodes/RemoteNode/RemoteNodesCollection.cs new file mode 100644 index 00000000..78431ded --- /dev/null +++ b/Centaurus.Domain/Nodes/RemoteNode/RemoteNodesCollection.cs @@ -0,0 +1,75 @@ +using Centaurus.Models; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Centaurus.Domain.StateManagers +{ + internal class RemoteNodesCollection + { + private object syncRoot = new { }; + private Dictionary nodes = new Dictionary(); + + public void SetNodes(Dictionary> nodeItems) + { + lock (syncRoot) + { + //copy current auditors + var oldNodes = nodes.ToDictionary(a => a.Key, a => a.Value); + nodes.Clear(); + foreach (var nodeItem in nodeItems) + { + var pubKey = nodeItem.Key; + var node = nodeItem.Value; + //if the auditor node already presented, re-add it. Otherwise create and add the new instance. + if (oldNodes.Remove(pubKey, out var auditorState)) + { + nodes.Add(pubKey, auditorState); + continue; + } + nodes.Add(pubKey, node.Value); + OnAdd?.Invoke(node.Value); + } + //all nodes that are not presented in new auditors list, should be removed + NodesRemoved(oldNodes.Values.ToList()); + } + } + + public event Action OnAdd; + + public event Action OnRemove; + + public List GetAllNodes() + { + lock (syncRoot) + { + return nodes.Values.ToList(); + } + } + + public bool TryGetNode(RawPubKey pubKey, out RemoteNode node) + { + lock (syncRoot) + return nodes.TryGetValue(pubKey, out node); + } + + internal void Clear() + { + lock (syncRoot) + { + var removedNodes = nodes.Values.ToList(); + nodes.Clear(); + NodesRemoved(removedNodes); + } + } + + private void NodesRemoved(List removedNodes) + { + foreach (var removedNode in removedNodes) + { + OnRemove?.Invoke(removedNode); + } + } + } +} \ No newline at end of file diff --git a/Centaurus.Domain/Quanta/Handlers/QuantumHandler.cs b/Centaurus.Domain/Quanta/Handlers/QuantumHandler.cs index 117076c0..768b8f3f 100644 --- a/Centaurus.Domain/Quanta/Handlers/QuantumHandler.cs +++ b/Centaurus.Domain/Quanta/Handlers/QuantumHandler.cs @@ -63,6 +63,14 @@ private async Task RunQuantumWorker() awaitedQuanta.TryDequeue(out handlingItem); if (handlingItem != null) { + //after Alpha switch, quanta queue can contain requests that should be send to new Alpha server + if (!Context.IsAlpha && handlingItem.Quantum.Apex == 0) + { + //if the quantum is not client request, than new Alpha will generate it, so we can skip it + if (handlingItem.Quantum is RequestQuantumBase request) + Context.ProxyWorker.AddRequestsToQueue(request.RequestEnvelope); + continue; + } await HandleItem(handlingItem); if (Context.IsAlpha && QuantaThrottlingManager.Current.IsThrottlingEnabled) Thread.Sleep(QuantaThrottlingManager.Current.SleepTime); @@ -79,7 +87,7 @@ private async Task RunQuantumWorker() while (awaitedQuanta.TryDequeue(out var item)) item.SetException(new Exception("Cancelled.")); } - Context.StateManager.Failed(new Exception("Quantum worker failed.", exc)); + Context.NodesManager.CurrentNode.Failed(new Exception("Quantum worker failed.", exc)); return; } } @@ -140,7 +148,7 @@ void ValidateQuantum(QuantumProcessingItem processingItem) if (!quantum.PrevHash.AsSpan().SequenceEqual(LastQuantumHash)) throw new Exception($"Quantum previous hash doesn't equal to last quantum hash."); } - if (!Context.IsAlpha || Context.StateManager.State == State.Rising) + if (!Context.IsAlpha || Context.NodesManager.CurrentNode.State == State.Rising) ValidateRequestQuantum(processingItem); } @@ -182,9 +190,6 @@ async Task ProcessQuantum(QuantumProcessingItem processingItem) CurrentApex = quantum.Apex; LastQuantumHash = processingItem.QuantumHash; - if (CurrentApex % 1000 == 0) - Context.StateManager.UpdateDelay(); - ProcessResult(processingItem); if (Context.IsAlpha) diff --git a/Centaurus.Domain/Quanta/Processors/AlphaUpdateProcessor.cs b/Centaurus.Domain/Quanta/Processors/AlphaUpdateProcessor.cs new file mode 100644 index 00000000..5fb0d25f --- /dev/null +++ b/Centaurus.Domain/Quanta/Processors/AlphaUpdateProcessor.cs @@ -0,0 +1,58 @@ +using Centaurus.Domain.Models; +using Centaurus.Models; +using Centaurus.PaymentProvider; +using Centaurus.Xdr; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Centaurus.Domain +{ + public class AlphaUpdateProcessor : QuantumProcessorBase + { + public AlphaUpdateProcessor(ExecutionContext context) + : base(context) + { + + } + + public override string SupportedMessageType { get; } = typeof(AlphaUpdate).Name; + + public override Task Process(QuantumProcessingItem processingItem) + { + var alphaUpdate = (AlphaUpdate)((ConstellationQuantum)processingItem.Quantum).RequestMessage; + + //make copy of current settings + var newConstellationSettings = XdrConverter.Deserialize(XdrConverter.Serialize(Context.Constellation)); + + newConstellationSettings.Apex = processingItem.Apex; + newConstellationSettings.Alpha = alphaUpdate.Alpha; + + processingItem.AddConstellationUpdate(Context.Constellation, Context.Constellation); + + Context.UpdateConstellationSettings(newConstellationSettings); + + return Task.FromResult((QuantumResultMessageBase)processingItem.Quantum.CreateEnvelope().CreateResult(ResultStatusCode.Success)); + } + + public override Task Validate(QuantumProcessingItem processingItem) + { + var currentState = Context.NodesManager.CurrentNode.State; + if (currentState == State.Undefined || currentState == State.WaitingForInit) + throw new InvalidOperationException($"Constellation is not initialized yet."); + + ((ConstellationQuantum)processingItem.Quantum).Validate(Context); + + var alphaUpdate = (AlphaUpdate)((ConstellationQuantum)processingItem.Quantum).RequestMessage; + + if (alphaUpdate.Alpha.Equals(Context.Constellation.Alpha)) + throw new InvalidOperationException($"{Context.Constellation.Alpha.GetAccountId()} is Alpha already."); + + if (!Context.Constellation.Auditors.Any(a => a.PubKey.Equals(Context.Constellation.Alpha))) + throw new InvalidOperationException($"{Context.Constellation.Alpha.GetAccountId()} is not an auditor."); + + return Task.CompletedTask; + } + } +} diff --git a/Centaurus.Domain/Quanta/Processors/ConstellationUpdateProcessor.cs b/Centaurus.Domain/Quanta/Processors/ConstellationUpdateProcessor.cs index 36e95058..c35643ed 100644 --- a/Centaurus.Domain/Quanta/Processors/ConstellationUpdateProcessor.cs +++ b/Centaurus.Domain/Quanta/Processors/ConstellationUpdateProcessor.cs @@ -33,11 +33,12 @@ public override Task Process(QuantumProcessingItem pro GetCursors(settings.Providers), processingItem.Quantum.ComputeHash() ); - Context.Setup(updateSnapshot); + Context.Init(updateSnapshot); + var currentNode = Context.NodesManager.CurrentNode; //if state is undefined, than we need to init it - if (Context.StateManager.State == State.Undefined) - Context.StateManager.Init(State.Running); + if (currentNode.State == State.WaitingForInit) + currentNode.Init(State.Running); return Task.FromResult((QuantumResultMessageBase)processingItem.Quantum.CreateEnvelope().CreateResult(ResultStatusCode.Success)); } diff --git a/Centaurus.Domain/Quanta/Sync/ApexItemsBatch.cs b/Centaurus.Domain/Quanta/Sync/ApexItemsBatch.cs index e8d26aa1..ac9ee31c 100644 --- a/Centaurus.Domain/Quanta/Sync/ApexItemsBatch.cs +++ b/Centaurus.Domain/Quanta/Sync/ApexItemsBatch.cs @@ -1,4 +1,5 @@ using Centaurus.Models; +using NLog; using System; using System.Collections.Generic; using System.Diagnostics; @@ -11,6 +12,7 @@ namespace Centaurus.Domain.Quanta.Sync public partial class ApexItemsBatch: ContextualBase where T : IApex { + static Logger logger = LogManager.GetCurrentClassLogger(); public ApexItemsBatch(ExecutionContext context, ulong start, int size, int portionSize, List initData) :base(context) { @@ -100,9 +102,8 @@ private bool TryProcessFirstOutrunItem() { AddToData(nextItem, item); if (!outrunData.Remove(nextItem)) - { - Console.WriteLine("Unable to remove."); - } + logger.Error($"Unable to remove item from outrun data. {typeof(T).Name} batch, id: {Start}."); + return true; } } diff --git a/Centaurus.Domain/Quanta/Sync/ApexItemsBatchPortion.cs b/Centaurus.Domain/Quanta/Sync/ApexItemsBatchPortion.cs index 364cd82e..6f7102c5 100644 --- a/Centaurus.Domain/Quanta/Sync/ApexItemsBatchPortion.cs +++ b/Centaurus.Domain/Quanta/Sync/ApexItemsBatchPortion.cs @@ -53,8 +53,7 @@ protected SyncPortion GetBatchData() var items = quantaBatch.GetItems(Start, Size, true); var batch = new SyncQuantaBatch { - Quanta = items, - LastKnownApex = source.Context.QuantumHandler.CurrentApex + Quanta = items }; return new SyncPortion(batch.CreateEnvelope().ToByteArray(), items.Last().Apex); } diff --git a/Centaurus.Domain/Quanta/Sync/CursorGroup.cs b/Centaurus.Domain/Quanta/Sync/CursorGroup.cs new file mode 100644 index 00000000..3841c173 --- /dev/null +++ b/Centaurus.Domain/Quanta/Sync/CursorGroup.cs @@ -0,0 +1,43 @@ +using Centaurus.Domain.Quanta.Sync; +using Centaurus.Domain.StateManagers; +using System; +using System.Collections.Generic; + +namespace Centaurus.Domain +{ + internal partial class SyncQuantaDataWorker + { + internal class CursorGroup + { + public CursorGroup(SyncCursorType cursorType, ulong batchId) + { + CursorType = cursorType; + BatchId = batchId; + } + + public SyncCursorType CursorType { get; } + + public ulong BatchId { get; } + + public DateTime LastUpdate { get; set; } + + public List Nodes { get; } = new List(); + } + + internal class NodeCursorData + { + public NodeCursorData(RemoteNode node, DateTime timeToken, ulong cursor) + { + Node = node; + TimeToken = timeToken; + Cursor = cursor; + } + + public RemoteNode Node { get; } + + public DateTime TimeToken { get; } + + public ulong Cursor { get; } + } + } +} \ No newline at end of file diff --git a/Centaurus.Domain/Quanta/Sync/ProxyWorker.cs b/Centaurus.Domain/Quanta/Sync/ProxyWorker.cs index a11f158c..df739e7a 100644 --- a/Centaurus.Domain/Quanta/Sync/ProxyWorker.cs +++ b/Centaurus.Domain/Quanta/Sync/ProxyWorker.cs @@ -9,7 +9,7 @@ namespace Centaurus.Domain { - public class ProxyWorker : ContextualBase, IDisposable + internal class ProxyWorker : ContextualBase, IDisposable { static Logger logger = LogManager.GetCurrentClassLogger(); public ProxyWorker(ExecutionContext context) @@ -46,7 +46,12 @@ private void Run() hasQuantaToSend = requestsToSend != null; if (hasQuantaToSend) { - await Context.OutgoingConnectionManager.SendTo(Context.Constellation.Alpha, new RequestQuantaBatch { Requests = requestsToSend }.CreateEnvelope()); + if (!Context.NodesManager.TryGetNode(Context.Constellation.Alpha, out var node)) + throw new Exception($"Unable to get Alpha node."); + var connection = node.GetConnection(); + if (connection == null) + throw new Exception($"Alpha node is not connected."); + await connection.SendMessage(new RequestQuantaBatch { Requests = requestsToSend }.CreateEnvelope()); } } catch (Exception exc) diff --git a/Centaurus.Domain/Quanta/Sync/SyncCursorUpdate.cs b/Centaurus.Domain/Quanta/Sync/SyncCursorUpdate.cs index 90517197..9b046b39 100644 --- a/Centaurus.Domain/Quanta/Sync/SyncCursorUpdate.cs +++ b/Centaurus.Domain/Quanta/Sync/SyncCursorUpdate.cs @@ -1,4 +1,5 @@ -using System; +using Centaurus.Domain.StateManagers; +using System; namespace Centaurus.Domain.Quanta.Sync { @@ -8,31 +9,31 @@ public enum SyncCursorType Signatures = 1 } - public class SyncCursorUpdate + internal class SyncCursorUpdate { - public SyncCursorUpdate(DateTime timeToken, ulong? newCursor, SyncCursorType cursorType) + public SyncCursorUpdate(DateTime timeToken, ulong newCursor, SyncCursorType cursorType) { NewCursor = newCursor; CursorType = cursorType; TimeToken = timeToken; } - public ulong? NewCursor { get; } + public ulong NewCursor { get; } public SyncCursorType CursorType { get; } public DateTime TimeToken { get; } } - public class AuditorSyncCursorUpdate + internal class NodeSyncCursorUpdate { - public AuditorSyncCursorUpdate(IncomingAuditorConnection connection, SyncCursorUpdate syncCursorUpdate) + public NodeSyncCursorUpdate(RemoteNode node, SyncCursorUpdate syncCursorUpdate) { - Connection = connection ?? throw new ArgumentNullException(nameof(connection)); + Node = node ?? throw new ArgumentNullException(nameof(node)); CursorUpdate = syncCursorUpdate ?? throw new ArgumentNullException(nameof(syncCursorUpdate)); } - public IncomingAuditorConnection Connection { get; } + public RemoteNode Node { get; } public SyncCursorUpdate CursorUpdate { get; set; } } diff --git a/Centaurus.Domain/Quanta/Sync/SyncQuantaDataWorker.cs b/Centaurus.Domain/Quanta/Sync/SyncQuantaDataWorker.cs index b0abd3f9..54101b60 100644 --- a/Centaurus.Domain/Quanta/Sync/SyncQuantaDataWorker.cs +++ b/Centaurus.Domain/Quanta/Sync/SyncQuantaDataWorker.cs @@ -1,4 +1,5 @@ using Centaurus.Domain.Quanta.Sync; +using Centaurus.Domain.StateManagers; using Centaurus.Models; using NLog; using System; @@ -9,7 +10,7 @@ namespace Centaurus.Domain { - public class SyncQuantaDataWorker : ContextualBase, IDisposable + internal partial class SyncQuantaDataWorker : ContextualBase, IDisposable { static Logger logger = LogManager.GetCurrentClassLogger(); @@ -21,112 +22,140 @@ public SyncQuantaDataWorker(ExecutionContext context) private CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); - - private bool IsAuditorReadyToHandleQuanta(IncomingAuditorConnection auditor) + private bool GetIsCurrentNodeReady() { - return auditor.ConnectionState == ConnectionState.Ready //connection is validated - && (auditor.AuditorState.IsRunning || auditor.AuditorState.IsWaitingForInit); //auditor is ready to handle quanta + var currentNode = Context.NodesManager.CurrentNode; + return currentNode.State == State.Running || currentNode.State == State.Ready; } - private bool IsCurrentNodeReady => Context.StateManager.State != State.Rising - && Context.StateManager.State != State.Undefined - && Context.StateManager.State != State.Failed; - - private List GetCursors() + private List GetCursors() { - var auditorConnections = Context.IncomingConnectionManager.GetAuditorConnections(); - - var quantaCursors = new Dictionary(); - var signaturesCursors = new Dictionary(); - foreach (var connection in auditorConnections) + var nodes = Context.NodesManager.GetRemoteNodes(); + var cursors = new Dictionary(); + foreach (var node in nodes) { - var (quantaCursor, signaturesCursor, timeToken) = connection.GetCursors(); - SetCursors(quantaCursors, connection, SyncCursorType.Quanta, quantaCursor, timeToken); - SetCursors(signaturesCursors, connection, SyncCursorType.Signatures, signaturesCursor, timeToken); + if (!node.IsReadyToHandleQuanta()) + continue; + var nodeCursors = node.GetActiveCursors(); + SetNodeCursors(cursors, node, nodeCursors); } - return quantaCursors.Values.Union(signaturesCursors.Values).ToList(); + return cursors.Values.ToList(); } - private void SetCursors(Dictionary cursors, IncomingAuditorConnection connection, SyncCursorType cursorType, ulong? cursor, DateTime timeToken) + private void SetNodeCursors(Dictionary cursors, RemoteNode node, List nodeCursors) { - if (cursor.HasValue && cursor.Value < Context.QuantumHandler.CurrentApex) + foreach (var cursor in nodeCursors) + SetCursors(cursors, node, cursor.CursorType, cursor.Cursor, cursor.UpdateDate); + } + + private void SetCursors(Dictionary cursors, RemoteNode node, SyncCursorType cursorType, ulong cursor, DateTime timeToken) + { + var cursorToLookup = cursor + 1; + var batchId = cursorToLookup - cursorToLookup % (ulong)Context.SyncStorage.PortionSize; + var groupId = $"{cursorType}-{batchId}"; + if (!cursors.TryGetValue(groupId, out var currentCursorGroup)) { - var cursorToLookup = cursor.Value + 1; - var quantaCursor = cursorToLookup - cursorToLookup % (ulong)Context.SyncStorage.PortionSize; - if (!cursors.TryGetValue(cursorToLookup, out var currentCursorGroup)) - { - currentCursorGroup = new AuditorCursorGroup(cursorType, quantaCursor); - cursors.Add(cursorToLookup, currentCursorGroup); - } - currentCursorGroup.Auditors.Add(new AuditorCursorGroup.AuditorCursorData(connection, timeToken, cursor.Value)); - if (currentCursorGroup.LastUpdate == default || currentCursorGroup.LastUpdate > timeToken) - currentCursorGroup.LastUpdate = timeToken; + currentCursorGroup = new CursorGroup(cursorType, batchId); + cursors.Add(groupId, currentCursorGroup); } + currentCursorGroup.Nodes.Add(new NodeCursorData(node, timeToken, cursor)); + if (currentCursorGroup.LastUpdate == default || currentCursorGroup.LastUpdate > timeToken) + currentCursorGroup.LastUpdate = timeToken; } const int ForceTimeOut = 100; - private List> SendQuantaData(List cursorGroups) + private List> SendQuantaData(List cursorGroups) { - var sendingQuantaTasks = new List>(); + var sendingQuantaTasks = new List>(); + foreach (var cursorGroup in cursorGroups) { - var currentCursor = cursorGroup.Cursor; - if (currentCursor == Context.QuantumHandler.CurrentApex) - continue; - if (currentCursor > Context.QuantumHandler.CurrentApex && IsCurrentNodeReady) - { - var message = $"Auditors {string.Join(',', cursorGroup.Auditors.Select(a => a.Connection.PubKeyAddress))} is above current constellation state."; - if (cursorGroup.Auditors.Count >= Context.GetMajorityCount() - 1) //-1 is current server - logger.Error(message); - else - logger.Info(message); - continue; - } + var cursorSendingTasks = ProcessCursorGroup(cursorGroup); + if (cursorSendingTasks.Count > 0) + sendingQuantaTasks.AddRange(cursorSendingTasks); + } + return sendingQuantaTasks; + } - var force = (DateTime.UtcNow - cursorGroup.LastUpdate).TotalMilliseconds > ForceTimeOut; + private List> ProcessCursorGroup(CursorGroup cursorGroup) + { + if (!(ValidateCursorGroup(cursorGroup) && TryGetBatch(cursorGroup, out var batch))) + return null; - var batch = default(SyncPortion); - var cursorType = cursorGroup.CursorType; - switch (cursorType) - { - case SyncCursorType.Quanta: - batch = Context.SyncStorage.GetQuanta(currentCursor, force); - break; - case SyncCursorType.Signatures: - batch = Context.SyncStorage.GetSignatures(currentCursor, force); - break; - default: - throw new NotImplementedException($"{cursorType} cursor type is not supported."); - } + var currentCursor = cursorGroup.BatchId; + var cursorType = cursorGroup.CursorType; - if (batch == null) - continue; + var lastBatchApex = batch.LastDataApex; - var lastBatchApex = batch.LastDataApex; + var sendingQuantaTasks = new List>(); + foreach (var node in cursorGroup.Nodes) + { + var sendMessageTask = SendSingleBatch(batch, currentCursor, cursorType, node); + if (sendMessageTask != null) + sendingQuantaTasks.Add(sendMessageTask); + } + return sendingQuantaTasks; + } - foreach (var auditorConnection in cursorGroup.Auditors) + private static Task SendSingleBatch(SyncPortion batch, ulong currentCursor, SyncCursorType cursorType, NodeCursorData currentAuditor) + { + var connection = currentAuditor.Node.GetConnection(); + if (currentAuditor.Cursor < batch.LastDataApex || connection == null) + return null; + var sendMessageTask = connection.SendMessage(batch.Data.AsMemory()); + return sendMessageTask.ContinueWith(t => + { + if (!t.IsFaulted) { - var currentAuditor = auditorConnection; - if (IsAuditorReadyToHandleQuanta(currentAuditor.Connection) && auditorConnection.Cursor < lastBatchApex) - sendingQuantaTasks.Add( - currentAuditor.Connection.SendMessage(batch.Data.AsMemory()) - .ContinueWith(t => - { - if (t.IsFaulted) - { - logger.Error(t.Exception, $"Unable to send quanta data to {currentAuditor.Connection.PubKeyAddress}. Cursor: {currentCursor}; CurrentApex: {Context.QuantumHandler.CurrentApex}"); - return null; - } - return new AuditorSyncCursorUpdate(currentAuditor.Connection, - new SyncCursorUpdate(currentAuditor.TimeToken, (ulong?)lastBatchApex, cursorType) - ); - }) - ); + HandleFaultedSendTask(t, currentCursor, batch, cursorType, currentAuditor); + return null; } + return new NodeSyncCursorUpdate(currentAuditor.Node, + new SyncCursorUpdate(currentAuditor.TimeToken, batch.LastDataApex, cursorType) + ); + }); + } + + private bool TryGetBatch(CursorGroup cursorGroup, out SyncPortion batch) + { + var force = (DateTime.UtcNow - cursorGroup.LastUpdate).TotalMilliseconds > ForceTimeOut; + + switch (cursorGroup.CursorType) + { + case SyncCursorType.Quanta: + batch = Context.SyncStorage.GetQuanta(cursorGroup.BatchId, force); + break; + case SyncCursorType.Signatures: + batch = Context.SyncStorage.GetSignatures(cursorGroup.BatchId, force); + break; + default: + throw new NotImplementedException($"{cursorGroup.CursorType} cursor type is not supported."); } - return sendingQuantaTasks; + return batch != null; + } + + private bool ValidateCursorGroup(CursorGroup cursorGroup) + { + var currentCursor = cursorGroup.BatchId; + if (currentCursor == Context.QuantumHandler.CurrentApex) + return false; + if (currentCursor > Context.QuantumHandler.CurrentApex && GetIsCurrentNodeReady()) + { + var message = $"Auditors {string.Join(',', cursorGroup.Nodes.Select(a => a.Node.AccountId))} is above current constellation state."; + if (cursorGroup.Nodes.Count >= Context.GetMajorityCount() - 1) //-1 is current server + logger.Error(message); + else + logger.Info(message); + return false; + } + return true; + } + + private static void HandleFaultedSendTask(Task t, ulong currentCursor, SyncPortion batch, SyncCursorType cursorType, NodeCursorData currentAuditor) + { + logger.Error(t.Exception, $"Unable to send quanta data to {currentAuditor.Node.AccountId}. CursorType: {cursorType}; Cursor: {currentCursor}; CurrentApex: {batch.LastDataApex}"); } private async Task SyncQuantaData() @@ -137,75 +166,59 @@ private async Task SyncQuantaData() if (!hasPendingQuanta) Thread.Sleep(50); - if (!Context.IsAlpha || !IsCurrentNodeReady) + if (!Context.IsAlpha || !GetIsCurrentNodeReady()) { hasPendingQuanta = false; continue; } - try - { - var cursors = GetCursors(); - - var sendingQuantaTasks = SendQuantaData(cursors); - - if (sendingQuantaTasks.Count > 0) - { - hasPendingQuanta = true; - - var cursorUpdates = await Task.WhenAll(sendingQuantaTasks); - var auditorUpdates = cursorUpdates.Where(u => u != null).GroupBy(u => u.Connection); - foreach (var connection in auditorUpdates) - connection.Key.SetSyncCursor(false, connection.Select(u => u.CursorUpdate).ToArray()); - } - } - catch (Exception exc) - { - if (exc is ObjectDisposedException - || exc.GetBaseException() is ObjectDisposedException) - throw; - logger.Error(exc, "Error on quanta data sync."); - } + hasPendingQuanta = await TrySendData(); } } - public void Dispose() + private async Task TrySendData() { - cancellationTokenSource.Cancel(); - cancellationTokenSource.Dispose(); - } + var hasPendingQuanta = false; + try + { + var cursors = GetCursors(); - class AuditorCursorGroup - { - public AuditorCursorGroup(SyncCursorType cursorType, ulong cursor) + var sendingQuantaTasks = SendQuantaData(cursors); + hasPendingQuanta = await HandleSendingTasks(sendingQuantaTasks); + } + catch (Exception exc) { - CursorType = cursorType; - Cursor = cursor; + if (exc is ObjectDisposedException + || exc.GetBaseException() is ObjectDisposedException) + throw; + logger.Error(exc, "Error on quanta data sync."); } - public SyncCursorType CursorType { get; } - - public ulong Cursor { get; } - - public DateTime LastUpdate { get; set; } + return hasPendingQuanta; + } - public List Auditors { get; } = new List(); + private static async Task HandleSendingTasks(List> sendingQuantaTasks) + { + var hasPendingQuanta = false; + if (sendingQuantaTasks.Count < 1) + return false; - public class AuditorCursorData + var cursorUpdates = await Task.WhenAll(sendingQuantaTasks); + foreach (var cursorUpdate in cursorUpdates) { - public AuditorCursorData(IncomingAuditorConnection connection, DateTime timeToken, ulong cursor) - { - Connection = connection; - TimeToken = timeToken; - Cursor = cursor; - } - - public IncomingAuditorConnection Connection { get; } + if (cursorUpdate == null) + continue; + cursorUpdate.Node.SetCursor(cursorUpdate.CursorUpdate.CursorType, cursorUpdate.CursorUpdate.TimeToken, cursorUpdate.CursorUpdate.NewCursor); + hasPendingQuanta = true; + } - public DateTime TimeToken { get; } + return hasPendingQuanta; + } - public ulong Cursor { get; } - } + public void Dispose() + { + cancellationTokenSource.Cancel(); + cancellationTokenSource.Dispose(); } } } \ No newline at end of file diff --git a/Centaurus.Domain/Quanta/Sync/SyncStorage.cs b/Centaurus.Domain/Quanta/Sync/SyncStorage.cs index b482e061..9c85378a 100644 --- a/Centaurus.Domain/Quanta/Sync/SyncStorage.cs +++ b/Centaurus.Domain/Quanta/Sync/SyncStorage.cs @@ -44,7 +44,7 @@ public SyncPortion GetQuanta(ulong from, bool force) { var quantaBatchStart = GetBatchApexStart(from + 1); var batch = GetBatch(quantaBatchStart); - + return batch.Quanta.GetData(from, force); } diff --git a/Centaurus.Domain/ResultManagers/ResultManager.cs b/Centaurus.Domain/ResultManagers/ResultManager.cs index 7622d4b7..1acc4296 100644 --- a/Centaurus.Domain/ResultManagers/ResultManager.cs +++ b/Centaurus.Domain/ResultManagers/ResultManager.cs @@ -87,7 +87,7 @@ private void ProcessResults() } catch (Exception exc) { - Context.StateManager.Failed(new Exception($"Error on processing result for apex {result.Apex}", exc)); + Context.NodesManager.CurrentNode.Failed(new Exception($"Error on processing result for apex {result.Apex}", exc)); return; } }); @@ -106,7 +106,7 @@ private void ProcessAuditorResults() } catch (Exception exc) { - Context.StateManager.Failed(new Exception($"Error on auditor result for apex {result.Apex}", exc)); + Context.NodesManager.CurrentNode.Failed(new Exception($"Error on auditor result for apex {result.Apex}", exc)); return; } }); @@ -115,7 +115,7 @@ private void ProcessAuditorResults() private void AddInternal(QuantumSignatures signaturesMessage) { //auditor can send delayed results - if (signaturesMessage.Apex <= Context.PendingUpdatesManager.LastSavedApex) + if (signaturesMessage.Apex <= Context.PendingUpdatesManager.LastPersistedApex) return; //add the signature to the aggregate var aggregate = GetAggregate(signaturesMessage.Apex); @@ -146,7 +146,7 @@ private void AddInternal(QuantumProcessingItem processingResult) GetAggregate(processingResult.Apex).Add(processingResult); } - public bool TryGetSignatures(ulong apex, out List signatures) + public bool TryGetSignatures(ulong apex, out List signatures) { TryGetAggregate(apex, out var aggregate); signatures = aggregate?.GetSignatures(); @@ -185,12 +185,12 @@ private void UpdateCache() var _currentBatchIds = currentBatchIds.ToList(); foreach (var batchId in _currentBatchIds) { - if (batchId == currentBatchId || batchId + batchSize > Context.PendingUpdatesManager.LastSavedApex) + if (batchId == currentBatchId || batchId + batchSize > Context.PendingUpdatesManager.LastPersistedApex) break; currentBatchIds.Remove(batchId); if (!resultCache.TryGetValue(batchId, out var batch)) { - Context.StateManager.Failed(new Exception($"Result batch {batchId} is not found.")); + Context.NodesManager.CurrentNode.Failed(new Exception($"Result batch {batchId} is not found.")); return; } var memoryCacheOption = new MemoryCacheEntryOptions { SlidingExpiration = TimeSpan.FromSeconds(15) }; @@ -243,16 +243,16 @@ public Aggregate(ulong apex, ResultManager manager) private bool IsFinalized; private object syncRoot = new { }; private HashSet processedAuditors = new HashSet(); - private List signatures = new List(); - private List outrunResults = new List(); + private List signatures = new List(); + private List outrunResults = new List(); - public List GetSignatures() + public List GetSignatures() { lock (syncRoot) return signatures.ToList(); } - public void Add(AuditorSignatureInternal signature) + public void Add(NodeSignatureInternal signature) { var majorityResult = MajorityResult.Unknown; lock (syncRoot) @@ -293,7 +293,7 @@ public void Add(AuditorSignatureInternal signature) $" is unreachable. Results received count is {processedAuditors.Count}," + $" valid results count is {votesCount}. The constellation collapsed."); ProcessingItem.SetException(exc); - Manager.Context.StateManager.Failed(exc); + Manager.Context.NodesManager.CurrentNode.Failed(exc); } IsFinalized = true; @@ -310,12 +310,12 @@ public void Add(QuantumProcessingItem quantumProcessingItem) //mark as acknowledged ProcessingItem.Acknowledged(); //send result to auditors - Manager.Context.OutgoingConnectionManager.EnqueueResult(new AuditorResult { Apex = Apex, Signature = signature }); + Manager.Context.NodesManager.EnqueueResult(new AuditorResult { Apex = Apex, Signature = signature }); ProcessOutrunSignatures(this); } } - private void AddValidSignature(AuditorSignatureInternal signature) + private void AddValidSignature(NodeSignatureInternal signature) { processedAuditors.Add(signature.AuditorId); //skip if processed or current auditor already sent the result @@ -346,7 +346,7 @@ private bool IsSignedByAlpha() return signatures.Any(s => s.AuditorId == ProcessingItem.AlphaId); } - private AuditorSignatureInternal AddCurrentNodeSignature() + private NodeSignatureInternal AddCurrentNodeSignature() { //get current node signature var signature = GetSignature(); @@ -355,9 +355,9 @@ private AuditorSignatureInternal AddCurrentNodeSignature() return signature; } - private AuditorSignatureInternal GetSignature() + private NodeSignatureInternal GetSignature() { - var currentAuditorSignature = new AuditorSignatureInternal + var currentAuditorSignature = new NodeSignatureInternal { AuditorId = ProcessingItem.CurrentAuditorId, PayloadSignature = ProcessingItem.PayloadHash.Sign(Manager.Context.Settings.KeyPair) diff --git a/Centaurus.Domain/StateManagers/StateManager.cs b/Centaurus.Domain/StateManagers/StateManager.cs deleted file mode 100644 index 321fe3f9..00000000 --- a/Centaurus.Domain/StateManagers/StateManager.cs +++ /dev/null @@ -1,272 +0,0 @@ -using Centaurus.Models; -using NLog; -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; - -namespace Centaurus.Domain -{ - public class StateChangedEventArgs : EventArgs - { - public StateChangedEventArgs(State state, State prevState) - { - State = state; - PrevState = prevState; - } - - public State State { get; } - - public State PrevState { get; } - } - - public class StateManager : ContextualBase - { - private Logger logger = LogManager.GetCurrentClassLogger(); - - public StateManager(ExecutionContext context) - : base(context) - { - } - - public State State { get; private set; } - - public event Action StateChanged; - - - public void Init(State state) - { - lock (syncRoot) - { - if (State != State.Undefined) - throw new InvalidOperationException("Context is already initialized."); - - if (state == State.Undefined) - throw new InvalidOperationException($"Init state cannot be {state}."); - - UpdateState(state); - } - } - - public void Stopped() - { - lock (syncRoot) - SetState(State.Stopped); - } - - public void Failed(Exception exc) - { - lock (syncRoot) - SetState(State.Failed, exc); - } - - public int ConnectedAuditorsCount - { - get - { - lock (statesSyncRoot) - return auditors.Count(a => a.Value.CurrentState.HasValue); - } - } - - public AuditorState GetAuditorState(RawPubKey pubKey) - { - lock (statesSyncRoot) - { - if (!auditors.TryGetValue(pubKey, out var auditorState)) - throw new UnauthorizedException($"{pubKey.GetAccountId()} is not auditor."); - return auditorState; - } - } - - public List ConnectedAuditors - { - get - { - lock (statesSyncRoot) - return auditors.Keys.ToList(); - } - } - - public void Rised() - { - lock (syncRoot) - UpdateState(State.Running); - //after node successfully started, the pending quanta can be deleted - Context.PersistentStorage.DeletePendingQuanta(); - } - - public void SetAuditors(List auditorPubkeys) - { - lock (statesSyncRoot) - { - var currentAuditors = auditors; - auditors.Clear(); - var currentAuditorKey = (RawPubKey)Context.Settings.KeyPair; - foreach (var auditor in auditorPubkeys) - { - if (auditor.Equals(currentAuditorKey)) - continue; //skip current - //if auditor's state already presented, add it. Otherwise create and add new state instance. - if (currentAuditors.TryGetValue(auditor, out var auditorState)) - auditors.Add(auditor, auditorState); - else - auditors.Add(auditor, new AuditorState()); - } - //update current node state - UpdateState(); - } - } - - public void SetAuditorState(RawPubKey auditorPubKey, State state) - { - lock (statesSyncRoot) - { - logger.Trace($"Auditor's {auditorPubKey.GetAccountId()} state {state} received."); - if (!auditors.TryGetValue(auditorPubKey, out var currentState)) - throw new Exception($"{auditorPubKey.GetAccountId()} is not auditor."); - - if (currentState.CurrentState == state) - return; //state didn't change - - currentState.CurrentState = state; - logger.Trace($"Auditor's {auditorPubKey.GetAccountId()} state {state} set."); - //update current node state - UpdateState(); - } - } - - public void RemoveAuditorState(RawPubKey auditorPubKey) - { - lock (statesSyncRoot) - { - if (auditors.TryGetValue(auditorPubKey, out var currentState)) - { - currentState.CurrentState = null; - //remove state from catchup if presented - Context.Catchup.RemoveState(auditorPubKey); - //update current node state - UpdateState(); - } - else - throw new Exception($"{auditorPubKey.GetAccountId()} is not auditor."); - } - } - - /// - /// Updates last processed by alpha apex - /// - /// - public void UpdateAlphaApex(ulong constellationApex) - { - lock (syncRoot) - { - AlphaApex = constellationApex; - } - } - - /// - /// Updates current node delay - /// - public void UpdateDelay() - { - if (Context.IsAlpha) - return; - lock (syncRoot) - { - var isDelayed = State.Chasing == State; - var currentApex = Context.QuantumHandler.CurrentApex; - if (isDelayed) - { - if (AlphaApex <= currentApex || AlphaApex - currentApex < RunningDelayTreshold) - UpdateState(State.Running); - logger.Info($"Current node delay is {AlphaApex - currentApex}"); - } - else - { - if (AlphaApex > currentApex && AlphaApex - currentApex > ChasingDelayTreshold) - { - logger.Info($"Current node delay is {AlphaApex - currentApex}"); - UpdateState(State.Chasing); - } - } - } - } - - private const ulong ChasingDelayTreshold = 50_000; - private const ulong RunningDelayTreshold = 10_000; - private ulong AlphaApex; - private object syncRoot = new { }; - private object statesSyncRoot = new { }; - private Dictionary auditors = new Dictionary(); - - private void UpdateState(State? state = null) - { - if (!state.HasValue) //state can be null if trying update on auditor's state change - state = State; - if (state == State.Running && IsConstellationReady()) - SetState(State.Ready); - else if (state == State.Ready && !IsConstellationReady()) - SetState(State.Running); - else - SetState(state.Value); - } - - private bool IsConstellationReady() - { - lock (statesSyncRoot) - { - //if Prime node than it must be connected with other nodes - if (Context.RoleManager.ParticipationLevel == CentaurusNodeParticipationLevel.Prime) - { - //if current server is - var isAlphaReady = Context.IsAlpha && (State == State.Ready || State == State.Running); - var connectedCount = 0; - foreach (var auditorState in auditors) - { - if (!(auditorState.Value.CurrentState == State.Ready || auditorState.Value.CurrentState == State.Running)) - continue; - if (Context.Constellation.Alpha.Equals(auditorState.Key)) - isAlphaReady = true; - connectedCount++; - } - return isAlphaReady && Context.HasMajority(connectedCount, false); - } - else - //if auditor doesn't have connections with another auditors, we only need to verify alpha's state - return auditors.TryGetValue(Context.Constellation.Alpha, out var alphaState) && (alphaState.CurrentState == State.Ready || alphaState.CurrentState == State.Running); - } - } - - private void SetState(State state, Exception exc = null) - { - if (exc != null) - logger.Error(exc); - if (State != state) - { - logger.Info($"State update: new state: {state}, prev state: {State}"); - var stateArgs = new StateChangedEventArgs(state, State); - State = state; - - StateChanged?.Invoke(stateArgs); - var updateMessage = new StateUpdateMessage { State = State } - .CreateEnvelope() - .Sign(Context.Settings.KeyPair); - Context.NotifyAuditors(updateMessage); - logger.Info($"State update {state} sent."); - } - } - - - public class AuditorState - { - public State? CurrentState { get; set; } - - public bool IsRunning => CurrentState == State.Running || CurrentState == State.Ready || CurrentState == State.Chasing; - - public bool IsWaitingForInit => CurrentState == State.Undefined; - - public bool IsReady => CurrentState == State.Ready; - } - } -} \ No newline at end of file diff --git a/Centaurus.Domain/Statistics/BatchSavedInfoExtensions.cs b/Centaurus.Domain/Statistics/BatchSavedInfoExtensions.cs deleted file mode 100644 index fd866cfa..00000000 --- a/Centaurus.Domain/Statistics/BatchSavedInfoExtensions.cs +++ /dev/null @@ -1,37 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace Centaurus.Domain -{ - public static class BatchSavedInfoExtensions - { - public static Centaurus.Models.BatchSavedInfo ToBatchSavedInfoModel(this BatchSavedInfo batchSavedInfo) - { - if (batchSavedInfo == null) - throw new ArgumentNullException(nameof(batchSavedInfo)); - - return new Centaurus.Models.BatchSavedInfo - { - EffectsCount = batchSavedInfo.EffectsCount, - QuantaCount = batchSavedInfo.QuantaCount, - ElapsedMilliseconds = batchSavedInfo.ElapsedMilliseconds, - SavedAt = batchSavedInfo.SavedAt.Ticks - }; - } - - public static BatchSavedInfo FromModel(this Centaurus.Models.BatchSavedInfo batchSavedInfo) - { - if (batchSavedInfo == null) - throw new ArgumentNullException(nameof(batchSavedInfo)); - - return new BatchSavedInfo - { - EffectsCount = batchSavedInfo.EffectsCount, - QuantaCount = batchSavedInfo.QuantaCount, - ElapsedMilliseconds = batchSavedInfo.ElapsedMilliseconds, - SavedAt = new DateTime(batchSavedInfo.SavedAt, DateTimeKind.Utc) - }; - } - } -} diff --git a/Centaurus.Domain/Statistics/PerformanceStatistics.cs b/Centaurus.Domain/Statistics/PerformanceStatistics.cs index bcac5230..a560daad 100644 --- a/Centaurus.Domain/Statistics/PerformanceStatistics.cs +++ b/Centaurus.Domain/Statistics/PerformanceStatistics.cs @@ -4,55 +4,29 @@ namespace Centaurus.Domain { - public abstract class PerformanceStatistics + public class PerformanceStatistics { + public string PublicKey { get; set; } + + public ulong Apex { get; set; } + + public ulong PersistedApex { get; set; } + public int QuantaPerSecond { get; set; } public int QuantaQueueLength { get; set; } - public List BatchInfos { get; set; } - public int Throttling { get; set; } public int State { get; set; } public DateTime UpdateDate { get; set; } - } - public class AlphaPerformanceStatistics : PerformanceStatistics - { - public List AuditorStatistics { get; set; } + public long ApexDiff { get; set; } - public AlphaPerformanceStatistics Clone() + public PerformanceStatistics Clone() { - return (AlphaPerformanceStatistics)MemberwiseClone(); + return (PerformanceStatistics)MemberwiseClone(); } } - - public class AuditorPerformanceStatistics : PerformanceStatistics - { - public string Auditor { get; set; } - - public int Delay { get; set; } = -1; - - public AuditorPerformanceStatistics Clone() - { - return (AuditorPerformanceStatistics)MemberwiseClone(); - } - } - - public class BatchSavedInfo - { - public int QuantaCount { get; set; } - - public int EffectsCount { get; set; } - - public long ElapsedMilliseconds { get; set; } - - public DateTime SavedAt { get; set; } - - public ulong FromApex { get; internal set; } - - public ulong ToApex { get; internal set; } - } } diff --git a/Centaurus.Domain/Statistics/PerformanceStatisticsExtensions.cs b/Centaurus.Domain/Statistics/PerformanceStatisticsExtensions.cs deleted file mode 100644 index 0fcb97cc..00000000 --- a/Centaurus.Domain/Statistics/PerformanceStatisticsExtensions.cs +++ /dev/null @@ -1,38 +0,0 @@ -using Centaurus.Models; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -namespace Centaurus.Domain -{ - public static class PerformanceStatisticsExtensions - { - public static AuditorPerformanceStatistics FromModel(this AuditorPerfStatistics statistics, string auditor) - { - if (statistics == null) - throw new ArgumentNullException(nameof(statistics)); - return new AuditorPerformanceStatistics - { - Auditor = auditor, - QuantaPerSecond = statistics.QuantaPerSecond, - QuantaQueueLength = statistics.QuantaQueueLength, - BatchInfos = statistics.BatchInfos.Select(b => b.FromModel()).ToList(), - UpdateDate = new DateTime(statistics.UpdateDate, DateTimeKind.Utc), - State = (int)statistics.State - }; - } - - public static AuditorPerfStatistics ToModel(this PerformanceStatistics statistics) - { - return new AuditorPerfStatistics - { - QuantaPerSecond = statistics.QuantaPerSecond, - QuantaQueueLength = statistics.QuantaQueueLength, - BatchInfos = statistics.BatchInfos.Select(b => b.ToBatchSavedInfoModel()).ToList(), - UpdateDate = statistics.UpdateDate.Ticks, - State = (State)statistics.State - }; - } - } -} diff --git a/Centaurus.Domain/Statistics/PerformanceStatisticsManager.cs b/Centaurus.Domain/Statistics/PerformanceStatisticsManager.cs index 859e7ee7..9f0098d1 100644 --- a/Centaurus.Domain/Statistics/PerformanceStatisticsManager.cs +++ b/Centaurus.Domain/Statistics/PerformanceStatisticsManager.cs @@ -9,166 +9,86 @@ namespace Centaurus.Domain { - public class PerformanceStatisticsManager : ContextualBase, IDisposable + internal class PerformanceStatisticsManager : ContextualBase, IDisposable { const int updateInterval = 1000; - static Logger logger = LogManager.GetCurrentClassLogger(); - public PerformanceStatisticsManager(ExecutionContext context) : base(context) { - updateTimer = new System.Timers.Timer(); + updateTimer = new Timer(); updateTimer.Interval = updateInterval; updateTimer.AutoReset = false; updateTimer.Elapsed += UpdateTimer_Elapsed; updateTimer.Start(); } - public void OnBatchSaved(BatchSavedInfo batchInfo) - { - lock (syncRoot) - { - LastBatchInfos.Add(batchInfo); - if (LastBatchInfos.Count > 20) - LastBatchInfos.RemoveAt(0); - } - } - - public void AddAuditorStatistics(string accountId, AuditorPerfStatistics message) - { - lock (auditorsStatistics) - auditorsStatistics[accountId] = message.FromModel(accountId); - } - - public void RemoveAuditorStatistics(string accountId) - { - lock (auditorsStatistics) - auditorsStatistics.Remove(accountId); - } - public void Dispose() { lock (syncRoot) { updateTimer?.Stop(); updateTimer?.Dispose(); - updateTimer = null; } } - List RecentApexes = new List(); - List LastBatchInfos = new List(); - List LastQuantaQueueLengths = new List(); - - object syncRoot = new { }; - Timer updateTimer; + private object syncRoot = new { }; + private readonly Timer updateTimer; - void UpdateTimer_Elapsed(object sender, ElapsedEventArgs e) + private void UpdateTimer_Elapsed(object sender, ElapsedEventArgs e) { - if (Context.StateManager.State == State.Running - || Context.StateManager.State == State.Ready - || Context.StateManager.State == State.Chasing) + if (Context.NodesManager.CurrentNode.State != State.Undefined) { lock (syncRoot) - CollectStatistics(); + SendToSubscribers(); } updateTimer?.Start(); } - Dictionary auditorsStatistics { get; set; } = new Dictionary(); - List GetAuditorsStatistics() + private List GetStatistics() { - lock (auditorsStatistics) - { - var auditorConnections = Context.IncomingConnectionManager.GetAuditorConnections(); - foreach (var auditorConnection in auditorConnections) - { - var accountId = auditorConnection?.PubKeyAddress; - if (!auditorsStatistics.TryGetValue(accountId, out var statistics)) - continue; - var auditorApex = auditorConnection.CurrentQuantaCursor; - if (auditorApex >= 0) - statistics.Delay = (int)(Context.QuantumHandler.CurrentApex - auditorApex); - } - return auditorsStatistics.Values.ToList(); - } + var statistics = GetRemoteNodesStatistics(); + statistics.Insert(0, GetNodeStatistics(Context.NodesManager.CurrentNode)); + return statistics; } - void CollectStatistics() + private List GetRemoteNodesStatistics() { - try - { - var current = new AuditorPerfStatistics - { - BatchInfos = GetBatchInfos().Select(b => b.ToBatchSavedInfoModel()).ToList(), - QuantaPerSecond = GetItemsPerSecond(), - QuantaQueueLength = GetQuantaAvgLength(), - UpdateDate = DateTime.UtcNow.Ticks, - State = Context.StateManager.State - }; - AddAuditorStatistics(Context.Settings.KeyPair.AccountId, current); - - Notifier.NotifyAuditors(Context, current.CreateEnvelope()); - SendToSubscribers(); - } - catch (Exception exc) + var nodes = Context.NodesManager.GetRemoteNodes(); + var statistics = new List(); + foreach (var node in nodes) { - logger.Error(exc); + var nodeStatistics = GetNodeStatistics(node); + statistics.Add(nodeStatistics); } + return statistics; } - void SendToSubscribers() + private PerformanceStatistics GetNodeStatistics(NodeBase node) { - if (!Context.SubscriptionsManager.TryGetSubscription(PerformanceStatisticsSubscription.SubscriptionName, out var subscription)) - return; - var statistics = new AlphaPerformanceStatistics + var accountId = node.PubKey.GetAccountId(); + var lastApex = node.LastApex; + var nodeStatistics = new PerformanceStatistics { - QuantaPerSecond = GetItemsPerSecond(), - QuantaQueueLength = GetQuantaAvgLength(), - Throttling = GetThrottling(), - AuditorStatistics = GetAuditorsStatistics(), - BatchInfos = GetBatchInfos(), + PublicKey = accountId, + Apex = lastApex, + PersistedApex = node.LastPersistedApex, + QuantaPerSecond = node.QuantaPerSecond, + QuantaQueueLength = node.QuantaQueueLength, + //Throttling = GetThrottling(), UpdateDate = DateTime.UtcNow, - State = (int)Context.StateManager.State + State = (int)node.State, + ApexDiff = (long)(Context.QuantumHandler.CurrentApex - node.LastApex) }; - Context.InfoConnectionManager.SendSubscriptionUpdate(subscription, PerformanceStatisticsUpdate.Generate(statistics, PerformanceStatisticsSubscription.SubscriptionName)); - } - - int GetQuantaAvgLength() - { - LastQuantaQueueLengths.Add(Context.QuantumHandler.QuantaQueueLenght); - if (LastQuantaQueueLengths.Count > 20) - LastQuantaQueueLengths.RemoveAt(0); - return (int)Math.Floor(decimal.Divide(LastQuantaQueueLengths.Sum(), LastQuantaQueueLengths.Count)); - } - - int GetThrottling() - { - return QuantaThrottlingManager.Current.IsThrottlingEnabled ? QuantaThrottlingManager.Current.MaxItemsPerSecond : 0; - } - - int GetItemsPerSecond() - { - RecentApexes.Add(new Apex { UpdatedAt = DateTime.UtcNow, CurrentApex = Context.QuantumHandler.CurrentApex }); - if (RecentApexes.Count > 20) - RecentApexes.RemoveAt(0); - - if (RecentApexes.Count < 2) - return 0; - var lastItem = RecentApexes.Last(); - var firstItem = RecentApexes.First(); - var timeDiff = (decimal)(lastItem.UpdatedAt - firstItem.UpdatedAt).TotalMilliseconds; - return (int)(decimal.Divide(lastItem.CurrentApex - firstItem.CurrentApex, timeDiff) * 1000); + return nodeStatistics; } - List GetBatchInfos() => LastBatchInfos; - - class Apex + void SendToSubscribers() { - public ulong CurrentApex { get; set; } + if (!Context.SubscriptionsManager.TryGetSubscription(PerformanceStatisticsSubscription.SubscriptionName, out var subscription)) + return; - public DateTime UpdatedAt { get; set; } + Context.InfoConnectionManager.SendSubscriptionUpdate(subscription, PerformanceStatisticsUpdate.Generate(GetStatistics(), PerformanceStatisticsSubscription.SubscriptionName)); } } } diff --git a/Centaurus.Domain/UpdatesManager/BatchSavedInfo.cs b/Centaurus.Domain/UpdatesManager/BatchSavedInfo.cs new file mode 100644 index 00000000..852cc59a --- /dev/null +++ b/Centaurus.Domain/UpdatesManager/BatchSavedInfo.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Centaurus.Domain +{ + public class BatchSavedInfo + { + public int QuantaCount { get; set; } + + public int EffectsCount { get; set; } + + public long ElapsedMilliseconds { get; set; } + + public DateTime SavedAt { get; set; } + + public ulong FromApex { get; internal set; } + + public ulong ToApex { get; internal set; } + } +} diff --git a/Centaurus.Domain/UpdatesManager/UpdatesManager.cs b/Centaurus.Domain/UpdatesManager/UpdatesManager.cs index ba97dc63..7e9e938b 100644 --- a/Centaurus.Domain/UpdatesManager/UpdatesManager.cs +++ b/Centaurus.Domain/UpdatesManager/UpdatesManager.cs @@ -22,7 +22,7 @@ public UpdatesManager(ExecutionContext context) { InitTimers(); - LastSavedApex = Context.DataProvider.GetLastApex(); + LastPersistedApex = Context.DataProvider.GetLastApex(); pendingUpdates = new UpdatesContainer(context); RegisterUpdates(pendingUpdates); @@ -50,9 +50,9 @@ public void UpdateBatch(bool force = false) } catch (Exception exc) { - if (Context.StateManager.State != State.Failed) + if (Context.NodesManager.CurrentNode.State != State.Failed) { - Context.StateManager.Failed(new Exception("Batch update failed.", exc)); + Context.NodesManager.CurrentNode.Failed(new Exception("Batch update failed.", exc)); } } finally @@ -63,7 +63,7 @@ public void UpdateBatch(bool force = false) } //last saved quantum apex - public ulong LastSavedApex { get; private set; } + public ulong LastPersistedApex { get; private set; } public void ApplyUpdates() { lock (applyUpdatesSyncRoot) @@ -80,7 +80,7 @@ public void ApplyUpdates() Context.DataProvider.SaveBatch(updates.GetUpdates()); sw.Stop(); - LastSavedApex = updates.LastApex; + LastPersistedApex = updates.LastApex; var batchInfo = new BatchSavedInfo { @@ -99,9 +99,9 @@ public void ApplyUpdates() catch (Exception exc) { updates = null; - if (Context.StateManager.State != State.Failed) + if (Context.NodesManager.CurrentNode.State != State.Failed) { - Context.StateManager.Failed(new Exception("Saving failed.", exc)); + Context.NodesManager.CurrentNode.Failed(new Exception("Saving failed.", exc)); } } } @@ -113,7 +113,7 @@ public void PersistPendingQuanta() lock (applyUpdatesSyncRoot) { //if node stopped during rising than we don't need to persist pending quanta, it's already in db - if (Context.StateManager.State == State.Rising) + if (Context.NodesManager.CurrentNode.State == State.Rising) return; var pendingQuanta = new List(); var hasMoreQuanta = true; @@ -216,9 +216,9 @@ private void BatchUpdateTimer_Elapsed(object sender, ElapsedEventArgs e) { try { - if (Context.StateManager?.State == State.Running - || Context.StateManager?.State == State.Ready - || Context.StateManager?.State == State.Chasing) + if (Context.NodesManager.CurrentNode.State == State.Running + || Context.NodesManager.CurrentNode.State == State.Ready + || Context.NodesManager.CurrentNode.State == State.Chasing) { UpdateBatch(); } diff --git a/Centaurus.Domain/WebSockets/Centaurus/ConnectionBase.cs b/Centaurus.Domain/WebSockets/Centaurus/ConnectionBase.cs index 64d33e9d..49a2f77a 100644 --- a/Centaurus.Domain/WebSockets/Centaurus/ConnectionBase.cs +++ b/Centaurus.Domain/WebSockets/Centaurus/ConnectionBase.cs @@ -59,7 +59,7 @@ public ConnectionBase(Domain.ExecutionContext context, KeyPair pubKey, WebSocket protected XdrBufferFactory.RentedBuffer incommingBuffer; protected XdrBufferFactory.RentedBuffer outgoingBuffer; - public bool IsAuditor => this is IAuditorConnection; + public abstract bool IsAuditor { get; } ConnectionState connectionState; /// @@ -147,7 +147,8 @@ public virtual async Task SendMessage(MessageEnvelopeBase envelope) Context.ExtensionsManager.BeforeSendMessage(this, envelope); - if (!(envelope.Message is AuditorPerfStatistics)) + var isStateMessage = envelope.Message is StateMessage; + if (!isStateMessage) logger.Trace($"Connection {PubKeyAddress}, about to send {envelope.Message.GetMessageType()} message."); using (var writer = new XdrBufferWriter(outgoingBuffer.Buffer)) @@ -157,7 +158,7 @@ public virtual async Task SendMessage(MessageEnvelopeBase envelope) await SendRawMessageInternal(outgoingBuffer.Buffer.AsMemory(0, writer.Length)); - if (!(envelope.Message is AuditorPerfStatistics)) + if (!isStateMessage) logger.Trace($"Connection {PubKeyAddress}, message {envelope.Message.GetMessageType()} sent. Size: {writer.Length}"); } } @@ -244,11 +245,12 @@ public async Task Listen() var reader = new XdrBufferReader(incommingBuffer.Buffer, incommingBuffer.Length); envelope = XdrConverter.Deserialize(reader); - if (!(envelope.Message is AuditorPerfStatistics)) + var isStateMessage = envelope.Message is StateMessage; + if (!isStateMessage) logger.Trace($"Connection {PubKeyAddress}, message {envelope.Message.GetMessageType()} received."); if (!await HandleMessage(envelope)) throw new UnexpectedMessageException($"No handler registered for message type {envelope.Message.GetMessageType()}."); - if (!(envelope.Message is AuditorPerfStatistics)) + if (!isStateMessage) logger.Trace($"Connection {PubKeyAddress}, message {envelope.Message.GetMessageType()} handled."); } catch (BaseClientException exc) diff --git a/Centaurus.Domain/WebSockets/Centaurus/IAuditorConnection.cs b/Centaurus.Domain/WebSockets/Centaurus/IAuditorConnection.cs deleted file mode 100644 index a8f3dd9e..00000000 --- a/Centaurus.Domain/WebSockets/Centaurus/IAuditorConnection.cs +++ /dev/null @@ -1,15 +0,0 @@ -using Centaurus.Models; -using System; -using System.Collections.Generic; -using System.Text; -using static Centaurus.Domain.StateManager; - -namespace Centaurus.Domain -{ - public interface IAuditorConnection - { - RawPubKey PubKey { get; } - - AuditorState AuditorState { get; } - } -} diff --git a/Centaurus.Domain/WebSockets/Centaurus/INodeConnection.cs b/Centaurus.Domain/WebSockets/Centaurus/INodeConnection.cs new file mode 100644 index 00000000..d2758b77 --- /dev/null +++ b/Centaurus.Domain/WebSockets/Centaurus/INodeConnection.cs @@ -0,0 +1,12 @@ +using Centaurus.Domain.StateManagers; +using Centaurus.Models; + +namespace Centaurus.Domain +{ + internal interface INodeConnection + { + RawPubKey PubKey { get; } + + RemoteNode Node { get; } + } +} diff --git a/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingAuditorConnection.cs b/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingAuditorConnection.cs deleted file mode 100644 index e8982fcd..00000000 --- a/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingAuditorConnection.cs +++ /dev/null @@ -1,75 +0,0 @@ -using Centaurus.Domain; -using Centaurus.Domain.Quanta.Sync; -using NLog; -using System; -using System.Net.WebSockets; -using static Centaurus.Domain.StateManager; - -namespace Centaurus -{ - public class IncomingAuditorConnection : IncomingConnectionBase, IAuditorConnection - { - static Logger logger = LogManager.GetCurrentClassLogger(); - - public IncomingAuditorConnection(ExecutionContext context, KeyPair keyPair, WebSocket webSocket, string ip) - : base(context, keyPair, webSocket, ip) - { - AuditorState = Context.StateManager.GetAuditorState(keyPair); - SendHandshake(); - } - - const int AuditorBufferSize = 50 * 1024 * 1024; - - protected override int inBufferSize => AuditorBufferSize; - - protected override int outBufferSize => AuditorBufferSize; - - public AuditorState AuditorState { get; } - - private object syncRoot = new { }; - - public ulong? CurrentQuantaCursor { get; private set; } - public ulong? CurrentSignaturesCursor { get; private set; } - public DateTime LastCursorUpdateDate { get; private set; } - - /// - /// - /// - /// Force means that auditor requested cursor reset - /// - public void SetSyncCursor(bool force, params SyncCursorUpdate[] cursorUpdates) - { - //set new quantum and signature cursors - lock (syncRoot) - { - if (cursorUpdates.Length == 0) - return; - foreach (var cursorUpdate in cursorUpdates) - { - if (!(force || cursorUpdate.TimeToken == LastCursorUpdateDate)) - continue; - switch (cursorUpdate.CursorType) - { - case SyncCursorType.Quanta: - CurrentQuantaCursor = cursorUpdate.NewCursor; - break; - case SyncCursorType.Signatures: - CurrentSignaturesCursor = cursorUpdate.NewCursor; - break; - default: - throw new NotImplementedException($"{cursorUpdate.CursorType} cursor type is not supported."); - } - } - LastCursorUpdateDate = DateTime.UtcNow; - if (force) - logger.Info($"Connection {PubKeyAddress}, cursors reset. Quanta cursor: {CurrentQuantaCursor}, signatures cursor: {CurrentSignaturesCursor}"); - } - } - - public (ulong? quantaCursor, ulong? signaturesCursor, DateTime timeToken) GetCursors() - { - lock (syncRoot) - return (CurrentQuantaCursor, CurrentSignaturesCursor, LastCursorUpdateDate); - } - } -} \ No newline at end of file diff --git a/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingClientConnection.cs b/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingClientConnection.cs index 65e6d521..53ecb8ba 100644 --- a/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingClientConnection.cs +++ b/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingClientConnection.cs @@ -6,7 +6,7 @@ namespace Centaurus { - public class IncomingClientConnection : IncomingConnectionBase + internal class IncomingClientConnection : IncomingConnectionBase { public IncomingClientConnection(ExecutionContext context, KeyPair keyPair, WebSocket webSocket, string ip) : base(context, keyPair, webSocket, ip) @@ -17,5 +17,7 @@ public IncomingClientConnection(ExecutionContext context, KeyPair keyPair, WebSo } public Account Account { get; } + + public override bool IsAuditor => false; } } \ No newline at end of file diff --git a/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingConnectionBase.cs b/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingConnectionBase.cs index 2c948a03..6697e66b 100644 --- a/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingConnectionBase.cs +++ b/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingConnectionBase.cs @@ -7,7 +7,7 @@ namespace Centaurus.Domain { - public class IncomingConnectionBase : ConnectionBase + internal abstract class IncomingConnectionBase : ConnectionBase { public IncomingConnectionBase(ExecutionContext context, KeyPair keyPair, WebSocket webSocket, string ip) : base(context, keyPair, webSocket) diff --git a/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingConnectionManager.cs b/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingConnectionManager.cs index 26e51013..4e07a05f 100644 --- a/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingConnectionManager.cs +++ b/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingConnectionManager.cs @@ -7,6 +7,9 @@ using System.Net.WebSockets; using System.Threading.Tasks; using System.Linq; +using Centaurus.Domain.WebSockets; +using Centaurus.Domain.StateManagers; +using System.Threading; namespace Centaurus.Domain { @@ -17,53 +20,31 @@ public class IncomingConnectionManager : ContextualBase { static Logger logger = LogManager.GetCurrentClassLogger(); - public IncomingConnectionManager(ExecutionContext context) + internal IncomingConnectionManager(ExecutionContext context) : base(context) { } - /// - /// Gets the connection by the account public key - /// - /// Account public key - /// Current account connection - /// True if connection is found, otherwise false - public bool TryGetConnection(RawPubKey pubKey, out IncomingConnectionBase connection) - { - return connections.TryGetValue(pubKey, out connection); - } - - - /// - /// Gets all auditor connections - /// - /// The list of current auditor connections - public List GetAuditorConnections() - { - var auditorConnections = new List(); - var auditors = Context.GetAuditors(); - for (var i = 0; i < auditors.Count; i++) - { - if (TryGetConnection(auditors[i], out IncomingConnectionBase auditorConnection)) - auditorConnections.Add((IncomingAuditorConnection)auditorConnection); - } - return auditorConnections; - } - /// /// Registers new client websocket connection /// /// New websocket connection public async Task OnNewConnection(WebSocket webSocket, RawPubKey rawPubKey, string ip) { - Context.ExtensionsManager.BeforeNewConnection(webSocket, ip); if (webSocket == null) throw new ArgumentNullException(nameof(webSocket)); + if (!ValidStates.Contains(Context.NodesManager.CurrentNode.State)) + { + await webSocket.CloseAsync(WebSocketCloseStatus.ProtocolError, "Server is not ready.", CancellationToken.None); + return; + } + + Context.ExtensionsManager.BeforeNewConnection(webSocket, ip); + var connection = default(IncomingConnectionBase); - var auditors = Context.GetAuditors(); - if (auditors.Any(a => a.Equals(rawPubKey))) - connection = new IncomingAuditorConnection(Context, rawPubKey, webSocket, ip); + if (Context.NodesManager.TryGetNode(rawPubKey, out var node)) + connection = new IncomingNodeConnection(Context, webSocket, ip, node); else connection = new IncomingClientConnection(Context, rawPubKey, webSocket, ip); using (connection) @@ -73,12 +54,17 @@ public async Task OnNewConnection(WebSocket webSocket, RawPubKey rawPubKey, stri } } + internal bool TryGetConnection(RawPubKey account, out IncomingConnectionBase connection) + { + return connections.TryGetConnection(account, out connection); + } + /// /// Closes all connection /// - public async Task CloseAllConnections(bool includingAuditors = true) + internal async Task CloseAllConnections(bool includingAuditors = true) { - var pubkeys = connections.Keys.ToArray(); + var pubkeys = connections.GetAllPubKeys(); foreach (var pk in pubkeys) try { @@ -86,7 +72,7 @@ public async Task CloseAllConnections(bool includingAuditors = true) if (!includingAuditors && Context.Constellation.Auditors.Any(a => a.PubKey.Equals(pk)) || !connections.TryRemove(pk, out var connection)) continue; - await UnsubscribeAndClose(connection); + await RemoveConnection(connection); } catch (Exception e) @@ -95,23 +81,24 @@ public async Task CloseAllConnections(bool includingAuditors = true) } } - public void CleanupAuditorConnections() + internal void CleanupAuditorConnections() { - var irrelevantAuditors = Context.StateManager.ConnectedAuditors + var irrelevantAuditors = Context.NodesManager.GetRemoteNodes() .Where(ca => !Context.Constellation.Auditors.Any(a => a.PubKey.Equals(ca))) + .Select(ca => ca.PubKey) .ToList(); foreach (var pk in irrelevantAuditors) try { - if (connections.TryRemove(pk, out var connection)) + if (!connections.TryRemove(pk, out var connection)) continue; UnsubscribeAndClose(connection) .ContinueWith(t => { //we need to drop it, to prevent sending private constellation data if (t.IsFaulted) - Context.StateManager.Failed(new Exception("Unable to drop irrelevant auditor connection.")); + Context.NodesManager.CurrentNode.Failed(new Exception("Unable to drop irrelevant auditor connection.")); }); } catch (Exception e) @@ -122,7 +109,9 @@ public void CleanupAuditorConnections() #region Private members - ConcurrentDictionary connections = new ConcurrentDictionary(); + IncomingConnectionsCollection connections = new IncomingConnectionsCollection(); + + private static State[] ValidStates = new State[] { State.Rising, State.Running, State.Ready }; void Subscribe(IncomingConnectionBase connection) { @@ -143,15 +132,8 @@ async Task UnsubscribeAndClose(IncomingConnectionBase connection) void AddConnection(IncomingConnectionBase connection) { - lock (connection) - { - connections.AddOrUpdate(connection.PubKey, connection, (key, oldConnection) => - { - RemoveConnection(oldConnection); - return connection; - }); - logger.Trace($"{connection.PubKey} is connected."); - } + connections.Add(connection); + logger.Trace($"{connection.PubKey} is connected."); } void OnConnectionStateChanged((ConnectionBase connection, ConnectionState prev, ConnectionState current) args) @@ -167,25 +149,22 @@ void OnConnectionStateChanged((ConnectionBase connection, ConnectionState prev, case ConnectionState.Closed: //if prev connection wasn't validated than the validated connection could be dropped if (args.prev == ConnectionState.Ready) - RemoveConnection(connection); + _ = RemoveConnection(connection); break; default: break; } } - void RemoveConnection(IncomingConnectionBase connection) + async Task RemoveConnection(IncomingConnectionBase connection) { - lock (connection) - { - _ = UnsubscribeAndClose(connection); + await UnsubscribeAndClose(connection); - connections.TryRemove(connection.PubKey, out _); - if (connection.IsAuditor) - { - Context.StateManager.RemoveAuditorState(connection.PubKey); - Context.PerformanceStatisticsManager.RemoveAuditorStatistics(connection.PubKeyAddress); - } + connections.TryRemove(connection); + if (connection.IsAuditor) + { + var nodeConnection = (IncomingNodeConnection)connection; + nodeConnection.Node.RemoveIncomingConnection(); } } diff --git a/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingConnectionsCollection.cs b/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingConnectionsCollection.cs new file mode 100644 index 00000000..d27721b7 --- /dev/null +++ b/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingConnectionsCollection.cs @@ -0,0 +1,54 @@ +using Centaurus.Models; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Centaurus.Domain.WebSockets +{ + internal class IncomingConnectionsCollection + { + ConcurrentDictionary connections = new ConcurrentDictionary(); + + /// + /// Gets the connection by the account public key + /// + /// Account public key + /// Current account connection + /// True if connection is found, otherwise false + public bool TryGetConnection(RawPubKey pubKey, out IncomingConnectionBase connection) + { + return connections.TryGetValue(pubKey, out connection); + } + + public void Add(IncomingConnectionBase connection) + { + connections.AddOrUpdate(connection.PubKey, connection, (key, oldConnection) => + { + TryRemove(oldConnection); + return connection; + }); + } + + public bool TryRemove(IncomingConnectionBase connection) + { + return TryRemove(connection.PubKey, out _); + } + + public bool TryRemove(RawPubKey pubKey, out IncomingConnectionBase connection) + { + return connections.TryRemove(pubKey, out connection); + } + + public List GetAll() + { + return connections.Values.ToList(); + } + + internal List GetAllPubKeys() + { + return connections.Keys.ToList(); + } + } +} diff --git a/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingNodeConnection.cs b/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingNodeConnection.cs new file mode 100644 index 00000000..676a0e0c --- /dev/null +++ b/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingNodeConnection.cs @@ -0,0 +1,27 @@ +using Centaurus.Domain; +using Centaurus.Domain.StateManagers; +using System; +using System.Net.WebSockets; + +namespace Centaurus +{ + internal class IncomingNodeConnection : IncomingConnectionBase, INodeConnection + { + public IncomingNodeConnection(ExecutionContext context, WebSocket webSocket, string ip, RemoteNode node) + : base(context, node?.PubKey, webSocket, ip) + { + Node = node ?? throw new ArgumentNullException(nameof(node)); + SendHandshake(); + } + + const int AuditorBufferSize = 50 * 1024 * 1024; + + protected override int inBufferSize => AuditorBufferSize; + + protected override int outBufferSize => AuditorBufferSize; + + public RemoteNode Node { get; } + + public override bool IsAuditor => true; + } +} \ No newline at end of file diff --git a/Centaurus.Domain/WebSockets/Centaurus/Notifier.cs b/Centaurus.Domain/WebSockets/Centaurus/Notifier.cs index 88b016ed..a07fc480 100644 --- a/Centaurus.Domain/WebSockets/Centaurus/Notifier.cs +++ b/Centaurus.Domain/WebSockets/Centaurus/Notifier.cs @@ -38,30 +38,8 @@ public static void NotifyAuditors(this ExecutionContext context, MessageEnvelope if (context == null) throw new ArgumentNullException(nameof(context)); context.ExtensionsManager.BeforeNotifyAuditors(envelope); - if (context.RoleManager.ParticipationLevel == CentaurusNodeParticipationLevel.Prime) //if Prime node than send message to all incoming connection - { - var auditors = context.IncomingConnectionManager.GetAuditorConnections(); - foreach (var auditor in auditors) - try - { - _ = auditor.SendMessage(envelope); - } - catch { } - } - else //notify all connected auditors - NotifyConnectedAuditors(context, envelope); - } - - static void NotifyConnectedAuditors(ExecutionContext context, MessageEnvelopeBase envelope) - { - try - { - _ = context.OutgoingConnectionManager.SendToAll(envelope); - } - catch (Exception exc) - { - logger.Error(exc, "Exception occurred during broadcasting message to auditors."); - } + foreach (var node in context.NodesManager.GetRemoteNodes()) + node.GetConnection()?.SendMessage(envelope); } } } diff --git a/Centaurus.Domain/WebSockets/Centaurus/Outgoing/OutgoingConnection.cs b/Centaurus.Domain/WebSockets/Centaurus/Outgoing/OutgoingConnection.cs index 65678e59..7ee5dfe9 100644 --- a/Centaurus.Domain/WebSockets/Centaurus/Outgoing/OutgoingConnection.cs +++ b/Centaurus.Domain/WebSockets/Centaurus/Outgoing/OutgoingConnection.cs @@ -1,26 +1,27 @@ using Centaurus.Client; +using Centaurus.Domain.StateManagers; using Centaurus.Models; using NLog; using System; using System.Linq; using System.Threading; using System.Threading.Tasks; -using static Centaurus.Domain.StateManager; +using static Centaurus.Domain.StateNotifierWorker; namespace Centaurus.Domain { - public class OutgoingConnection : ConnectionBase, IAuditorConnection + internal class OutgoingConnection : ConnectionBase, INodeConnection { static Logger logger = LogManager.GetCurrentClassLogger(); private readonly OutgoingConnectionWrapperBase connection; private readonly OutgoingMessageStorage outgoingMessageStorage; - public OutgoingConnection(ExecutionContext context, KeyPair keyPair, OutgoingMessageStorage outgoingMessageStorage, OutgoingConnectionWrapperBase connection) - : base(context, keyPair, connection.WebSocket) + public OutgoingConnection(ExecutionContext context, RemoteNode node, OutgoingMessageStorage outgoingMessageStorage, OutgoingConnectionWrapperBase connection) + : base(context, node?.PubKey, connection.WebSocket) { this.connection = connection ?? throw new ArgumentNullException(nameof(connection)); this.outgoingMessageStorage = outgoingMessageStorage ?? throw new ArgumentNullException(nameof(outgoingMessageStorage)); - AuditorState = Context.StateManager.GetAuditorState(keyPair); + Node = node; //we know that we connect to auditor so we can set connection state to validated immediately ConnectionState = ConnectionState.Ready; } @@ -30,7 +31,9 @@ public OutgoingConnection(ExecutionContext context, KeyPair keyPair, OutgoingMes protected override int inBufferSize => BufferSize; protected override int outBufferSize => BufferSize; - public AuditorState AuditorState { get; } + public RemoteNode Node { get; } + + public override bool IsAuditor => throw new NotImplementedException(); public async Task EstablishConnection(Uri uri) { @@ -50,7 +53,7 @@ private void ProcessOutgoingMessageQueue() try { if (!cancellationToken.IsCancellationRequested - && AuditorState.IsRunning + && Node.IsRunning() && outgoingMessageStorage.TryPeek(out MessageEnvelopeBase message) ) { diff --git a/Centaurus.Domain/WebSockets/Centaurus/Outgoing/OutgoingConnectionManager.cs b/Centaurus.Domain/WebSockets/Centaurus/Outgoing/OutgoingConnectionManager.cs deleted file mode 100644 index 5fc3d7c5..00000000 --- a/Centaurus.Domain/WebSockets/Centaurus/Outgoing/OutgoingConnectionManager.cs +++ /dev/null @@ -1,220 +0,0 @@ -using Centaurus.Client; -using Centaurus.Models; -using NLog; -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using System.Web; - -namespace Centaurus.Domain -{ - public class OutgoingConnectionManager : ContextualBase - { - static Logger logger = LogManager.GetCurrentClassLogger(); - - private readonly Dictionary connections = new Dictionary(); - - private OutgoingConnectionFactoryBase connectionFactory; - - public OutgoingConnectionManager(ExecutionContext context, OutgoingConnectionFactoryBase connectionFactory) - : base(context) - { - this.connectionFactory = connectionFactory ?? throw new ArgumentNullException(); - } - - public void Connect(List auditors) - { - if (auditors == null) - throw new ArgumentNullException(nameof(auditors)); - try - { - CloseInvalidConnections(auditors); - - foreach (var auditor in auditors) - { - if (!auditor.IsPrime //not prime server - || connections.ContainsKey(auditor.PubKey) //already connected - || auditor.PubKey.Equals(Context.Settings.KeyPair)) //current server - continue; - - var connection = new AtomicConnection(this, auditor.PubKey, GetAuditorUri(auditor, Context.Settings.UseSecureConnection)); - connections.Add(auditor.PubKey, connection); - connection.Run(); - } - } - catch (Exception exc) - { - if (Context.StateManager != null) - Context.StateManager.Failed(exc); - } - } - - public async Task SendToAll(MessageEnvelopeBase message) - { - var sendTasks = new List(); - foreach (var c in connections) - { - sendTasks.Add(c.Value.SendMessage(message)); - } - await Task.WhenAll(sendTasks); - } - - public async Task SendTo(RawPubKey rawPubKey, MessageEnvelopeBase message) - { - if (!connections.TryGetValue(rawPubKey, out var connection)) - throw new Exception($"Connection {rawPubKey} is not found."); - await connection.SendMessage(message); - } - - public void CloseAllConnections() - { - foreach (var connection in connections.Values) - connection.Shutdown(); - connections.Clear(); - } - - private void CloseInvalidConnections(List auditors) - { - var connectionsToDrop = new List(); - foreach (var connection in connections.Values) - { - var auditor = auditors.FirstOrDefault(a => a.PubKey.Equals(connection.PubKey)); - if (auditor == null //not in address book - || !auditor.IsPrime //not prime server - || GetAuditorUri(auditor, Context.Settings.UseSecureConnection) != connection.Address) //address changed - connectionsToDrop.Add(connection.PubKey); - } - foreach (var connectionKey in connectionsToDrop) - { - if (!connections.Remove(connectionKey, out var connection)) - throw new Exception($"Unable to drop connection {connectionKey}"); - connection.Shutdown(); - } - } - - public static Uri GetAuditorUri(Settings.Auditor auditor, bool useSecureConnection) - { - return new Uri(auditor.GetWsConnection(useSecureConnection), WebSocketConstants.CentaurusWebSocketEndPoint); - } - - public void EnqueueResult(AuditorResult result) - { - foreach (var connection in connections) - connection.Value.OutgoingResultsStorage.EnqueueResult(result); - } - - public void EnqueueMessage(MessageEnvelopeBase message) - { - foreach (var connection in connections) - { - connection.Value.OutgoingMessageStorage.EnqueueMessage(message); - } - } - - class AtomicConnection : ContextualBase - { - - public AtomicConnection(OutgoingConnectionManager connectionManager, RawPubKey keyPair, Uri address) - : base(connectionManager?.Context) - { - PubKey = keyPair ?? throw new ArgumentNullException(nameof(keyPair)); - Address = address ?? throw new ArgumentNullException(nameof(address)); - - this.connectionManager = connectionManager ?? throw new ArgumentNullException(nameof(connectionManager)); - OutgoingMessageStorage = new OutgoingMessageStorage(); - OutgoingResultsStorage = new OutgoingResultsStorage(OutgoingMessageStorage); - } - - public OutgoingMessageStorage OutgoingMessageStorage { get; } - - public OutgoingResultsStorage OutgoingResultsStorage { get; } - - public void Run() - { - try - { - ConnectToAuditor(); - } - catch (Exception exc) - { - logger.Error(exc); - throw; - } - } - - public void Shutdown() - { - lock (syncRoot) - { - isAborted = true; - auditor?.CloseConnection(); - OutgoingResultsStorage.Dispose(); - } - } - - public async Task SendMessage(MessageEnvelopeBase message) - { - if (auditor != null) - await auditor.SendMessage(message); - } - - public RawPubKey PubKey { get; } - - public Uri Address { get; } - - private void ConnectToAuditor() - { - Task.Factory.StartNew(async () => - { - var uriBuilder = new UriBuilder(Address); - var query = HttpUtility.ParseQueryString(uriBuilder.Query); - query[WebSocketConstants.PubkeyParamName] = Context.Settings.KeyPair.AccountId; - uriBuilder.Query = query.ToString(); - - var connectionUri = uriBuilder.Uri; - var connectionAttempts = 0; - while (!isAborted) - { - if (Context.StateManager.State == State.Rising) //wait while all pending quanta will be handled - { - Thread.Sleep(1000); - continue; - } - auditor = new OutgoingConnection(Context, PubKey, OutgoingMessageStorage, connectionManager.connectionFactory.GetConnection()); - try - { - await auditor.EstablishConnection(connectionUri); - await auditor.Listen(); - } - catch (Exception exc) - { - - if (!(exc is OperationCanceledException) && connectionAttempts % 100 == 0) - logger.Error(exc, $"Unable establish connection with {connectionUri} after {connectionAttempts} attempts. Retry in 1000ms"); - Thread.Sleep(1000); - connectionAttempts++; - } - finally - { - auditor.Dispose(); - auditor = null; - } - } - }); - } - - private readonly object syncRoot = new { }; - - private OutgoingConnectionManager connectionManager; - private bool isAborted = false; - - private OutgoingConnection auditor; - - private Logger logger = LogManager.GetCurrentClassLogger(); - } - } -} diff --git a/Centaurus.Domain/WebSockets/Info/Messages/Subscriptions/PerformanceStatisticsUpdate.cs b/Centaurus.Domain/WebSockets/Info/Messages/Subscriptions/PerformanceStatisticsUpdate.cs index 5b89e326..5bc932eb 100644 --- a/Centaurus.Domain/WebSockets/Info/Messages/Subscriptions/PerformanceStatisticsUpdate.cs +++ b/Centaurus.Domain/WebSockets/Info/Messages/Subscriptions/PerformanceStatisticsUpdate.cs @@ -8,63 +8,21 @@ namespace Centaurus.Domain { public class PerformanceStatisticsUpdate : SubscriptionUpdateBase { - public int QuantaPerSecond { get; set; } - - public int Throttling { get; set; } - - public int QuantaQueueLength { get; private set; } - - public List BatchSavedInfos { get; set; } - - public List AuditorStatistics { get; private set; } + public List AuditorStatistics { get; private set; } public override SubscriptionUpdateBase GetUpdateForDate(DateTime lastUpdateDate) { - var batches = BatchSavedInfos.Where(b => b.SavedAt > lastUpdateDate).ToList(); - - var statistics = new List(); - foreach (var auditorStatistic in AuditorStatistics) - { - if (auditorStatistic.UpdateDate < lastUpdateDate) - continue; - var auditorBatches = auditorStatistic.BatchInfos.Where(b => b.SavedAt > lastUpdateDate).ToList(); - if (auditorBatches.Count == auditorStatistic.BatchInfos.Count) - statistics.Add(auditorStatistic); - else - { - var statistic = auditorStatistic.Clone(); - statistic.BatchInfos = auditorBatches; - statistics.Add(statistic); - } - } - - //all data are new for the client - if (batches.Count == BatchSavedInfos.Count - && statistics.Sum(s => s.BatchInfos.Count) == AuditorStatistics.Sum(s => s.BatchInfos.Count)) - return this; - - return new PerformanceStatisticsUpdate - { - QuantaPerSecond = QuantaPerSecond, - Throttling = Throttling, - BatchSavedInfos = batches, //send only new data - QuantaQueueLength = QuantaQueueLength, - AuditorStatistics = statistics, - ChannelName = ChannelName, - UpdateDate = UpdateDate - }; + if (UpdateDate < lastUpdateDate) + return null; + return this; } - public static PerformanceStatisticsUpdate Generate(AlphaPerformanceStatistics update, string channelName) + public static PerformanceStatisticsUpdate Generate(List update, string channelName) { return new PerformanceStatisticsUpdate { - QuantaPerSecond = update.QuantaPerSecond, - Throttling = update.Throttling, - BatchSavedInfos = update.BatchInfos, + AuditorStatistics = update, ChannelName = channelName, - QuantaQueueLength = update.QuantaQueueLength, - AuditorStatistics = update.AuditorStatistics, UpdateDate = DateTime.UtcNow }; } diff --git a/Centaurus.Domain/exchange/OrderMatcher.cs b/Centaurus.Domain/exchange/OrderMatcher.cs index c38c6112..661d873c 100644 --- a/Centaurus.Domain/exchange/OrderMatcher.cs +++ b/Centaurus.Domain/exchange/OrderMatcher.cs @@ -152,7 +152,7 @@ private bool PlaceReminderOrder() /// /// Single orders match descriptor. /// - internal class OrderMatch + class OrderMatch { /// /// Create new instance of order match. diff --git a/Centaurus.Models/Common/AuditorSignatureBase.cs b/Centaurus.Models/Common/AuditorSignatureBase.cs index 378b2257..e23e40ed 100644 --- a/Centaurus.Models/Common/AuditorSignatureBase.cs +++ b/Centaurus.Models/Common/AuditorSignatureBase.cs @@ -6,7 +6,7 @@ namespace Centaurus.Models { [XdrContract] - [XdrUnion(0, typeof(AuditorSignatureInternal))] + [XdrUnion(0, typeof(NodeSignatureInternal))] [XdrUnion(1, typeof(AuditorSignature))] public class AuditorSignatureBase { @@ -17,7 +17,7 @@ public class AuditorSignatureBase public TinySignature PayloadSignature { get; set; } } - public class AuditorSignatureInternal: AuditorSignatureBase + public class NodeSignatureInternal: AuditorSignatureBase { [XdrField(0, Optional = true)] diff --git a/Centaurus.Models/Common/State.cs b/Centaurus.Models/Common/State.cs index 9c0f6ddb..37caee58 100644 --- a/Centaurus.Models/Common/State.cs +++ b/Centaurus.Models/Common/State.cs @@ -7,21 +7,20 @@ namespace Centaurus.Models public enum State { Undefined = 0, + WaitingForInit = 1, + Rising = 2, /// /// It has started, but not yet ready. If Alpha, then it waits for the majority to connect. If the Auditor, then it waits for a handshake /// - Running = 1, - /// - /// Ready to process quanta - /// - Ready = 2, - - Rising = 3, - + Running = 3, /// /// Auditor is delayed /// Chasing = 4, + /// + /// Ready to process quanta + /// + Ready = 5, Failed = 10, diff --git a/Centaurus.Models/InternalMessages/AlphaUpdate.cs b/Centaurus.Models/InternalMessages/AlphaUpdate.cs index 5959a75d..90fd1c80 100644 --- a/Centaurus.Models/InternalMessages/AlphaUpdate.cs +++ b/Centaurus.Models/InternalMessages/AlphaUpdate.cs @@ -1,13 +1,6 @@ -using Centaurus.Xdr; -using System; -using System.Collections.Generic; -using System.Text; - -namespace Centaurus.Models +namespace Centaurus.Models { - public abstract class AlphaUpdate: ConstellationRequestMessage + public class AlphaUpdate: AlphaUpdateBase { - [XdrField(0)] - public RawPubKey Alpha { get; set; } } } diff --git a/Centaurus.Models/InternalMessages/AlphaUpdateBase.cs b/Centaurus.Models/InternalMessages/AlphaUpdateBase.cs new file mode 100644 index 00000000..9cee4ae0 --- /dev/null +++ b/Centaurus.Models/InternalMessages/AlphaUpdateBase.cs @@ -0,0 +1,10 @@ +using Centaurus.Xdr; + +namespace Centaurus.Models +{ + public abstract class AlphaUpdateBase: ConstellationRequestMessage + { + [XdrField(0)] + public RawPubKey Alpha { get; set; } + } +} diff --git a/Centaurus.Models/InternalMessages/CatchupQuantaBatch.cs b/Centaurus.Models/InternalMessages/CatchupQuantaBatch.cs index 4cc83689..e5598cd9 100644 --- a/Centaurus.Models/InternalMessages/CatchupQuantaBatch.cs +++ b/Centaurus.Models/InternalMessages/CatchupQuantaBatch.cs @@ -22,6 +22,6 @@ public class CatchupQuantaBatchItem public Message Quantum { get; set; } [XdrField(1)] - public List Signatures { get; set; } + public List Signatures { get; set; } } } diff --git a/Centaurus.Models/InternalMessages/ConstellationUpdate.cs b/Centaurus.Models/InternalMessages/ConstellationUpdate.cs index 823c95c7..af4cd5ac 100644 --- a/Centaurus.Models/InternalMessages/ConstellationUpdate.cs +++ b/Centaurus.Models/InternalMessages/ConstellationUpdate.cs @@ -5,7 +5,7 @@ namespace Centaurus.Models { - public class ConstellationUpdate : AlphaUpdate + public class ConstellationUpdate : AlphaUpdateBase { [XdrField(0)] public List Providers { get; set; } diff --git a/Centaurus.Models/InternalMessages/PendingQuantum.cs b/Centaurus.Models/InternalMessages/PendingQuantum.cs deleted file mode 100644 index a7f5a2d9..00000000 --- a/Centaurus.Models/InternalMessages/PendingQuantum.cs +++ /dev/null @@ -1,14 +0,0 @@ -using Centaurus.Xdr; -using System; -using System.Collections.Generic; -using System.Text; - -namespace Centaurus.Models -{ - public class PendingQuantum - { - public Quantum Quantum { get; set; } - - public List Signatures { get; set; } - } -} diff --git a/Centaurus.Models/InternalMessages/PerformanceStatistics.cs b/Centaurus.Models/InternalMessages/PerformanceStatistics.cs deleted file mode 100644 index 18d32fcb..00000000 --- a/Centaurus.Models/InternalMessages/PerformanceStatistics.cs +++ /dev/null @@ -1,41 +0,0 @@ -using Centaurus.Xdr; -using System; -using System.Collections.Generic; -using System.Text; - -namespace Centaurus.Models -{ - public class AuditorPerfStatistics : Message - { - [XdrField(0)] - public int QuantaPerSecond { get; set; } - - [XdrField(1)] - public int QuantaQueueLength { get; set; } - - [XdrField(2)] - public List BatchInfos { get; set; } - - [XdrField(3)] - public long UpdateDate { get; set; } - - [XdrField(4)] - public State State { get; set; } - } - - [XdrContract] - public class BatchSavedInfo - { - [XdrField(0)] - public int QuantaCount { get; set; } - - [XdrField(1)] - public int EffectsCount { get; set; } - - [XdrField(2)] - public long ElapsedMilliseconds { get; set; } - - [XdrField(3)] - public long SavedAt { get; set; } - } -} diff --git a/Centaurus.Models/InternalMessages/QuantumMajoritySignaturesBatch.cs b/Centaurus.Models/InternalMessages/QuantumMajoritySignaturesBatch.cs index 156caa92..ec586e39 100644 --- a/Centaurus.Models/InternalMessages/QuantumMajoritySignaturesBatch.cs +++ b/Centaurus.Models/InternalMessages/QuantumMajoritySignaturesBatch.cs @@ -22,6 +22,6 @@ public class QuantumSignatures: IApex /// Contains all quantum's signatures except Alpha /// [XdrField(1)] - public List Signatures { get; set; } + public List Signatures { get; set; } } } diff --git a/Centaurus.Models/InternalMessages/StateMessage.cs b/Centaurus.Models/InternalMessages/StateMessage.cs index 3612b25d..9c9a5ec9 100644 --- a/Centaurus.Models/InternalMessages/StateMessage.cs +++ b/Centaurus.Models/InternalMessages/StateMessage.cs @@ -1,10 +1,23 @@ using Centaurus.Xdr; +using System; namespace Centaurus.Models { - public class StateUpdateMessage : Message + public class StateMessage : Message { [XdrField(0)] + public ulong CurrentApex { get; set; } + + [XdrField(1)] + public ulong LastPersistedApex { get; set; } + + [XdrField(2)] + public int QuantaQueueLength { get; set; } + + [XdrField(3)] public State State { get; set; } + + [XdrField(4)] + public DateTime UpdateDate { get; set; } } } \ No newline at end of file diff --git a/Centaurus.Models/InternalMessages/SyncCursor.cs b/Centaurus.Models/InternalMessages/SyncCursor.cs index 1afb0a3c..eb32c525 100644 --- a/Centaurus.Models/InternalMessages/SyncCursor.cs +++ b/Centaurus.Models/InternalMessages/SyncCursor.cs @@ -19,7 +19,7 @@ public class SyncCursor /// Set to false, to cancel sync. /// [XdrField(2)] - public bool ClearCursor { get; set; } + public bool DisableSync { get; set; } } public enum XdrSyncCursorType diff --git a/Centaurus.Models/InternalMessages/SyncCursorReset.cs b/Centaurus.Models/InternalMessages/SyncCursorReset.cs index f9db18b1..93c1fa59 100644 --- a/Centaurus.Models/InternalMessages/SyncCursorReset.cs +++ b/Centaurus.Models/InternalMessages/SyncCursorReset.cs @@ -9,6 +9,6 @@ namespace Centaurus.Models public class SyncCursorReset : Message { [XdrField(0)] - public List SyncCursors { get; set; } = new List(); + public List Cursors { get; set; } = new List(); } } \ No newline at end of file diff --git a/Centaurus.Models/InternalMessages/SyncQuantaBatch.cs b/Centaurus.Models/InternalMessages/SyncQuantaBatch.cs index 9f235fbd..1f3edb8e 100644 --- a/Centaurus.Models/InternalMessages/SyncQuantaBatch.cs +++ b/Centaurus.Models/InternalMessages/SyncQuantaBatch.cs @@ -10,9 +10,6 @@ public class SyncQuantaBatch : Message { [XdrField(0)] public List Quanta { get; set; } - - [XdrField(2)] - public ulong LastKnownApex { get; set; } } [XdrContract] @@ -23,7 +20,7 @@ public class SyncQuantaBatchItem : IApex public Message Quantum { get; set; } [XdrField(1)] - public AuditorSignatureInternal AlphaSignature { get; set; } + public NodeSignatureInternal AlphaSignature { get; set; } public ulong Apex => ((Quantum)Quantum).Apex; } diff --git a/Centaurus.Models/Messages/HandshakeResponse.cs b/Centaurus.Models/Messages/HandshakeResponse.cs index 13febbf2..90939a7b 100644 --- a/Centaurus.Models/Messages/HandshakeResponse.cs +++ b/Centaurus.Models/Messages/HandshakeResponse.cs @@ -3,25 +3,11 @@ namespace Centaurus.Models { - public abstract class HandshakeResponseBase : Message + public class HandshakeResponse : Message { [XdrField(0)] public HandshakeData HandshakeData { get; set; } public override long MessageId => HandshakeData?.Data.GetInt64Fingerprint() ?? 0; } - - public class HandshakeResponse : HandshakeResponseBase - { - } - - public class AuditorHandshakeResponse : HandshakeResponseBase - { - - [XdrField(0)] - public State State { get; set; } - - [XdrField(1)] - public List Cursors { get; set; } - } } diff --git a/Centaurus.Models/Messages/Message.cs b/Centaurus.Models/Messages/Message.cs index b4ef6609..8994894a 100644 --- a/Centaurus.Models/Messages/Message.cs +++ b/Centaurus.Models/Messages/Message.cs @@ -6,7 +6,6 @@ namespace Centaurus.Models [XdrContract] [XdrUnion((int)MessageTypes.HandshakeRequest, typeof(HandshakeRequest))] [XdrUnion((int)MessageTypes.HandshakeResponse, typeof(HandshakeResponse))] - [XdrUnion((int)MessageTypes.AuditorHandshakeResponse, typeof(AuditorHandshakeResponse))] [XdrUnion((int)MessageTypes.WithdrawalRequest, typeof(WithdrawalRequest))] [XdrUnion((int)MessageTypes.AccountDataRequest, typeof(AccountDataRequest))] [XdrUnion((int)MessageTypes.AccountDataRequestQuantum, typeof(AccountDataRequestQuantum))] @@ -20,15 +19,15 @@ namespace Centaurus.Models [XdrUnion((int)MessageTypes.RequestQuantum, typeof(RequestQuantum))] [XdrUnion((int)MessageTypes.WithdrawalRequestQuantum, typeof(WithdrawalRequestQuantum))] [XdrUnion((int)MessageTypes.DepositQuantum, typeof(DepositQuantum))] - [XdrUnion((int)MessageTypes.AuditorPerfStatistics, typeof(AuditorPerfStatistics))] - [XdrUnion((int)MessageTypes.StateUpdate, typeof(StateUpdateMessage))] + [XdrUnion((int)MessageTypes.StateUpdate, typeof(StateMessage))] [XdrUnion((int)MessageTypes.AuditorSignaturesBatch, typeof(AuditorSignaturesBatch))] + [XdrUnion((int)MessageTypes.AlphaUpdate, typeof(AlphaUpdate))] [XdrUnion((int)MessageTypes.ConstellationUpdate, typeof(ConstellationUpdate))] [XdrUnion((int)MessageTypes.ConstellationQuantum, typeof(ConstellationQuantum))] [XdrUnion((int)MessageTypes.SyncCursorReset, typeof(SyncCursorReset))] + [XdrUnion((int)MessageTypes.QuantumMajoritySignaturesBatch, typeof(QuantumMajoritySignaturesBatch))] [XdrUnion((int)MessageTypes.SyncQuantaBatch, typeof(SyncQuantaBatch))] [XdrUnion((int)MessageTypes.RequestQuantaBatch, typeof(RequestQuantaBatch))] - [XdrUnion((int)MessageTypes.QuantumMajoritySignaturesBatch, typeof(QuantumMajoritySignaturesBatch))] [XdrUnion((int)MessageTypes.CatchupQuantaBatchRequest, typeof(CatchupQuantaBatchRequest))] [XdrUnion((int)MessageTypes.CatchupQuantaBatch, typeof(CatchupQuantaBatch))] [XdrUnion((int)MessageTypes.QuantumInfoRequest, typeof(QuantumInfoRequest))] diff --git a/Centaurus.Models/Messages/MessageTypes.cs b/Centaurus.Models/Messages/MessageTypes.cs index d5c86b53..da59a596 100644 --- a/Centaurus.Models/Messages/MessageTypes.cs +++ b/Centaurus.Models/Messages/MessageTypes.cs @@ -76,14 +76,6 @@ public enum MessageTypes /// EffectsNotification = 106, /// - /// Auditor's performance statistics. - /// - AuditorPerfStatistics = 107, - /// - /// Internal message. Contains quantum message envelope and effects signatures. - /// - PendingQuantum = 108, - /// /// Internal message. Contains new Auditor state. /// StateUpdate = 109, @@ -96,6 +88,10 @@ public enum MessageTypes /// QuantumInfoResponse = 152, /// + /// Contains new Alpha server public key. + /// + AlphaUpdate = 200, + /// /// Update quorum, add/remove auditors, apply new settings etc. /// ConstellationUpdate = 201, @@ -106,11 +102,15 @@ public enum MessageTypes /// /// Contains last saved Apex /// - CatchupQuantaBatchRequest = 211, + CatchupQuantaBatchRequest = 210, /// /// Contains batch of quanta, that was requested by Alpha /// - CatchupQuantaBatch = 212, + CatchupQuantaBatch = 211, + /// + /// Contains last apex processed by an auditor + /// + ApexUpdate = 212, /// /// Contains cursors for quanta and signatures /// diff --git a/Centaurus.Models/Results/AuditorResult.cs b/Centaurus.Models/Results/AuditorResult.cs index e303d1ae..c1885611 100644 --- a/Centaurus.Models/Results/AuditorResult.cs +++ b/Centaurus.Models/Results/AuditorResult.cs @@ -12,6 +12,6 @@ public class AuditorResult public ulong Apex { get; set; } [XdrField(1)] - public AuditorSignatureInternal Signature { get; set; } + public NodeSignatureInternal Signature { get; set; } } } diff --git a/Centaurus.Test.Domain/AlphaMessageHandlersTests.cs b/Centaurus.Test.Domain/AlphaMessageHandlersTests.cs index 21da0dda..1de67ba4 100644 --- a/Centaurus.Test.Domain/AlphaMessageHandlersTests.cs +++ b/Centaurus.Test.Domain/AlphaMessageHandlersTests.cs @@ -10,7 +10,7 @@ namespace Centaurus.Test { - public class AlphaMessageHandlersTests : BaseMessageHandlerTests + internal class AlphaMessageHandlersTests : BaseMessageHandlerTests { [OneTimeSetUp] public async Task Setup() @@ -32,8 +32,8 @@ public void TearDown() { new object[] { TestEnvironment.Client1KeyPair, State.Rising, typeof(ConnectionCloseException) }, new object[] { TestEnvironment.Client1KeyPair, State.Ready, null }, - new object[] { TestEnvironment.Auditor1KeyPair, State.Rising, typeof(UnauthorizedException) }, - new object[] { TestEnvironment.Auditor1KeyPair, State.Ready, typeof(UnauthorizedException) } + new object[] { TestEnvironment.Auditor1KeyPair, State.Rising, null}, + new object[] { TestEnvironment.Auditor1KeyPair, State.Ready, null } }; [Test] @@ -64,42 +64,6 @@ public async Task HandshakeTest(KeyPair clientKeyPair, State alphaState, Type ex else Assert.ThrowsAsync(expectedException, async () => await context.MessageHandlers.HandleMessage(connection, inMessage)); } - static object[] AuditorHandshakeTestCases = - { - new object[] { TestEnvironment.Client1KeyPair, State.Rising, typeof(UnauthorizedException) }, - new object[] { TestEnvironment.Client1KeyPair, State.Ready, typeof(UnauthorizedException) }, - new object[] { TestEnvironment.Auditor1KeyPair, State.Rising, null }, - new object[] { TestEnvironment.Auditor1KeyPair, State.Ready, null } - }; - - [Test] - [TestCaseSource(nameof(AuditorHandshakeTestCases))] - public async Task AuditorHandshakeTest(KeyPair clientKeyPair, State alphaState, Type expectedException) - { - context.SetState(alphaState); - - var connection = GetIncomingConnection(context, clientKeyPair); - - var handshakeData = (HandshakeData)typeof(IncomingConnectionBase) - .GetField("handshakeData", BindingFlags.Instance | BindingFlags.NonPublic) - .GetValue(connection); - - var message = new AuditorHandshakeResponse { HandshakeData = handshakeData, Cursors = new List(), State = State.Running }; - var envelope = message.CreateEnvelope(); - envelope.Sign(clientKeyPair); - - using var writer = new XdrBufferWriter(); - var inMessage = envelope.ToIncomingMessage(writer); - if (expectedException == null) - { - var isHandled = await context.MessageHandlers.HandleMessage(connection, inMessage); - - Assert.IsTrue(isHandled); - Assert.AreEqual(connection.ConnectionState, ConnectionState.Ready); - } - else - Assert.ThrowsAsync(expectedException, async () => await context.MessageHandlers.HandleMessage(connection, inMessage)); - } [Test] public void HandshakeInvalidDataTest() @@ -120,30 +84,6 @@ public void HandshakeInvalidDataTest() Assert.ThrowsAsync(async () => await context.MessageHandlers.HandleMessage(clientConnection, inMessage)); } - [Test] - public async Task AuditorPerfStatisticsTest() - { - var auditorPerf = new AuditorPerfStatistics - { - BatchInfos = new List(), - QuantaPerSecond = 1, - QuantaQueueLength = 2, - UpdateDate = DateTime.UtcNow.Ticks - }; - - var envelope = auditorPerf.CreateEnvelope(); - envelope.Sign(TestEnvironment.Auditor1KeyPair); - - - var auditorConnection = GetIncomingConnection(context, TestEnvironment.Auditor1KeyPair.PublicKey, ConnectionState.Ready); - - using var writer = new XdrBufferWriter(); - var inMessage = envelope.ToIncomingMessage(writer); - - await AssertMessageHandling(auditorConnection, inMessage, null); - - } - static object[] QuantaBatchTestCases = { new object[] { TestEnvironment.Client1KeyPair, ConnectionState.Ready, typeof(UnauthorizedException) }, @@ -193,7 +133,7 @@ public async Task QuantumMajoritySignaturesBatchTest(KeyPair clientKeyPair, Conn await AssertMessageHandling(clientConnection, inMessage, excpectedException); if (excpectedException == null) - Assert.AreEqual(context.StateManager.State, State.Running); + Assert.AreEqual(context.NodesManager.CurrentNode.State, State.Running); } static object[] OrderTestCases = diff --git a/Centaurus.Test.Domain/AuditorMessageHandlersTests.cs b/Centaurus.Test.Domain/AuditorMessageHandlersTests.cs index 10b2d2a9..ace98ca6 100644 --- a/Centaurus.Test.Domain/AuditorMessageHandlersTests.cs +++ b/Centaurus.Test.Domain/AuditorMessageHandlersTests.cs @@ -8,7 +8,7 @@ namespace Centaurus.Test { - public class AuditorMessageHandlersTests : BaseMessageHandlerTests + internal class AuditorMessageHandlersTests : BaseMessageHandlerTests { private ExecutionContext context; diff --git a/Centaurus.Test.Domain/BaseMessageHandlerTests.cs b/Centaurus.Test.Domain/BaseMessageHandlerTests.cs index a2a9905b..f5ac7eda 100644 --- a/Centaurus.Test.Domain/BaseMessageHandlerTests.cs +++ b/Centaurus.Test.Domain/BaseMessageHandlerTests.cs @@ -9,7 +9,7 @@ namespace Centaurus.Test { - public abstract class BaseMessageHandlerTests + internal abstract class BaseMessageHandlerTests { protected async Task AssertMessageHandling(T connection, IncomingMessage message, Type excpectedException = null) where T : ConnectionBase @@ -30,8 +30,8 @@ protected async Task AssertMessageHandling(T connection, IncomingMessage mess protected IncomingConnectionBase GetIncomingConnection(ExecutionContext context, RawPubKey pubKey, ConnectionState? state = null) { var connection = default(IncomingConnectionBase); - if (context.Constellation.Auditors.Any(a => a.PubKey.Equals(pubKey))) - connection = new IncomingAuditorConnection(context, pubKey, new DummyWebSocket(), "127.0.0.1"); + if (context.NodesManager.TryGetNode(pubKey, out var node)) + connection = new IncomingNodeConnection(context, new DummyWebSocket(), "127.0.0.1", node); else connection = new IncomingClientConnection(context, pubKey, new DummyWebSocket(), "127.0.0.1"); @@ -40,9 +40,10 @@ protected IncomingConnectionBase GetIncomingConnection(ExecutionContext context, return connection; } - protected OutgoingConnection GetOutgoingConnection(ExecutionContext context, RawPubKey keyPair, ConnectionState? state = null) + protected OutgoingConnection GetOutgoingConnection(ExecutionContext context, RawPubKey pubKey, ConnectionState? state = null) { - var connection = new OutgoingConnection(context, keyPair, new OutgoingMessageStorage(), new DummyConnectionWrapper(new DummyWebSocket())); + context.NodesManager.TryGetNode(pubKey, out var node); + var connection = new OutgoingConnection(context, node, new OutgoingMessageStorage(), new DummyConnectionWrapper(new DummyWebSocket())); if (state != null) connection.ConnectionState = state.Value; return connection; diff --git a/Centaurus.Test.Domain/OrderbookTests.cs b/Centaurus.Test.Domain/OrderbookTests.cs index c59b9383..65b7c0c4 100644 --- a/Centaurus.Test.Domain/OrderbookTests.cs +++ b/Centaurus.Test.Domain/OrderbookTests.cs @@ -50,12 +50,12 @@ public void Setup() account2.CreateBalance(secondAsset); account2.GetBalance(secondAsset).UpdateBalance(10000000000, UpdateSign.Plus); - context.Setup(new Snapshot + context.Init(new Snapshot { Accounts = new List { account1, account2 }, Apex = 0, Orders = new List(), - Settings = new ConstellationSettings + ConstellationSettings = new ConstellationSettings { Providers = new List { new ProviderSettings { diff --git a/Centaurus.Test.Domain/PendingQuantaPersistentTest.cs b/Centaurus.Test.Domain/PendingQuantaPersistentTest.cs index 03920af5..92584a51 100644 --- a/Centaurus.Test.Domain/PendingQuantaPersistentTest.cs +++ b/Centaurus.Test.Domain/PendingQuantaPersistentTest.cs @@ -52,27 +52,28 @@ await Task.Factory.StartNew(() => Assert.NotNull(quantumModel); - var quantum = quantumModel.ToDomainModel(); + var item = quantumModel.ToCatchupQuantaBatchItem(); + var quantum = (Quantum)item.Quantum; - Assert.IsTrue(context.Settings.KeyPair.Verify(quantum.Quantum.GetPayloadHash(), quantum.Signatures.First().PayloadSignature.Data)); + Assert.IsTrue(context.Settings.KeyPair.Verify(quantum.GetPayloadHash(), item.Signatures.First().PayloadSignature.Data)); - Assert.AreEqual(quantum.Quantum.Apex, result.Apex); - Assert.IsInstanceOf(quantum.Quantum); + Assert.AreEqual(quantum.Apex, result.Apex); + Assert.IsInstanceOf(item.Quantum); context = new ExecutionContext(settings, storage, new MockPaymentProviderFactory(), new DummyConnectionWrapperFactory()); - Assert.AreEqual(State.Rising, context.StateManager.State); + Assert.AreEqual(State.Rising, context.NodesManager.CurrentNode.State); - var targetQuantum = context.SyncStorage.GetQuanta(quantum.Quantum.Apex - 1, 1).FirstOrDefault(); + var targetQuantum = context.SyncStorage.GetQuanta(quantum.Apex - 1, 1).FirstOrDefault(); Assert.NotNull(targetQuantum); var quantumFromStorage = (Quantum)targetQuantum.Quantum; - Assert.IsTrue(context.Settings.KeyPair.Verify(quantumFromStorage.GetPayloadHash(), quantum.Signatures.First().PayloadSignature.Data)); + Assert.IsTrue(context.Settings.KeyPair.Verify(quantumFromStorage.GetPayloadHash(), item.Signatures.First().PayloadSignature.Data)); Assert.AreEqual(quantumFromStorage.Apex, result.Apex); Assert.IsInstanceOf(targetQuantum.Quantum); - context.StateManager.Rised(); - Assert.AreEqual(State.Running, context.StateManager.State); + //context.NodesManager.CurrentNode.Rised(); + Assert.AreEqual(State.Running, context.NodesManager.CurrentNode.State); Assert.IsNull(context.PersistentStorage.LoadPendingQuanta()); } diff --git a/Centaurus.Test.Domain/QuantumHandlerPerformanceTest.cs b/Centaurus.Test.Domain/QuantumHandlerPerformanceTest.cs index 13f8f3e8..8e1b9c8a 100644 --- a/Centaurus.Test.Domain/QuantumHandlerPerformanceTest.cs +++ b/Centaurus.Test.Domain/QuantumHandlerPerformanceTest.cs @@ -10,7 +10,7 @@ namespace Centaurus.Test { - public class QuantumHandlerPerformanceTest : BaseMessageHandlerTests + internal class QuantumHandlerPerformanceTest : BaseMessageHandlerTests { private ExecutionContext context; diff --git a/Centaurus.Test.Domain/QuantumStorageTests.cs b/Centaurus.Test.Domain/QuantumStorageTests.cs index 34025429..da609f92 100644 --- a/Centaurus.Test.Domain/QuantumStorageTests.cs +++ b/Centaurus.Test.Domain/QuantumStorageTests.cs @@ -48,7 +48,7 @@ public void QuantumStorageTest() context.SyncStorage.AddQuantum(quantum.Apex, new SyncQuantaBatchItem { Quantum = quantum, - AlphaSignature = new AuditorSignatureInternal + AlphaSignature = new NodeSignatureInternal { AuditorId = 0, PayloadSignature = new TinySignature diff --git a/Centaurus.Test.Exchange.Analytics/UpdatesTest.cs b/Centaurus.Test.Exchange.Analytics/UpdatesTest.cs index 6a3c7725..b44c9942 100644 --- a/Centaurus.Test.Exchange.Analytics/UpdatesTest.cs +++ b/Centaurus.Test.Exchange.Analytics/UpdatesTest.cs @@ -65,12 +65,12 @@ public void Setup() PaymentSubmitDelay = 0 }; - context.Setup(new Snapshot + context.Init(new Snapshot { Accounts = new List { account1, account2 }, Apex = 0, Orders = new List(), - Settings = new ConstellationSettings + ConstellationSettings = new ConstellationSettings { Providers = new List { stellarPaymentProvider }, Assets = new List { new AssetSettings { Code = "XLM", IsQuoteAsset = true }, new AssetSettings { Code = "USD" } }, diff --git a/Centaurus.Test.Integration/IntegrationTestEnvironmentExtensions.cs b/Centaurus.Test.Integration/IntegrationTestEnvironmentExtensions.cs index 6c871de7..729d4bd3 100644 --- a/Centaurus.Test.Integration/IntegrationTestEnvironmentExtensions.cs +++ b/Centaurus.Test.Integration/IntegrationTestEnvironmentExtensions.cs @@ -14,7 +14,7 @@ namespace Centaurus.Test { - public static class IntegrationTestEnvironmentExtensions + internal static class IntegrationTestEnvironmentExtensions { public static async Task InitConstellation(this IntegrationTestEnvironment environment) { @@ -92,8 +92,8 @@ public static async Task AssertState(Startup startup, State targetState, TimeSpa { Func> func = () => { - TestContext.Out.WriteLine(startup.Context.StateManager.State); - return Task.FromResult(startup.Context.StateManager.State == targetState); + TestContext.Out.WriteLine(startup.Context.NodesManager.CurrentNode.State); + return Task.FromResult(startup.Context.NodesManager.CurrentNode.State == targetState); }; await AssertDuringPeriod( @@ -240,7 +240,7 @@ public static async Task ProcessQuantumIsolated(this I while (environment.AlphaWrapper.Context.DataProvider.GetLastApex() != environment.AlphaWrapper.Context.QuantumHandler.CurrentApex) Thread.Sleep(100); - context.Setup(environment.AlphaWrapper.Context.DataProvider.GetSnapshot(environment.AlphaWrapper.Context.QuantumHandler.CurrentApex)); + context.Init(environment.AlphaWrapper.Context.DataProvider.GetSnapshot(environment.AlphaWrapper.Context.QuantumHandler.CurrentApex)); var messageType = quantum.GetType().Name; var account = default(Account); diff --git a/Centaurus.Test.Integration/IntegrationTests.cs b/Centaurus.Test.Integration/IntegrationTests.cs index 3508d662..728f607b 100644 --- a/Centaurus.Test.Integration/IntegrationTests.cs +++ b/Centaurus.Test.Integration/IntegrationTests.cs @@ -67,7 +67,7 @@ public async Task AuditorRestartTest() var auditorStartup = environment.AuditorWrappers.Values.Skip(1).First(); //first is Alpha auditorStartup.Shutdown(); - Assert.AreEqual(1, environment.AlphaWrapper.Context.StateManager.ConnectedAuditorsCount, "Auditors count assertion."); + Assert.AreEqual(1, environment.AlphaWrapper.Context.NodesManager.GetRemoteNodes().Count(n => n.State != State.Undefined), "Auditors count assertion."); await environment.AssertConstellationState(TimeSpan.FromSeconds(5), State.Ready); var clientsCount = 100; @@ -238,7 +238,7 @@ public async Task ScamQuantaTest(bool useFakeClient, bool useFakeAlpha, bool inv syncStorage.AddQuantum(apex, new SyncQuantaBatchItem { Quantum = requestQuantum, - AlphaSignature = new AuditorSignatureInternal + AlphaSignature = new NodeSignatureInternal { AuditorId = environment.AlphaWrapper.Context.AlphaId, PayloadSignature = new TinySignature diff --git a/Centaurus.Test.Integration/MockConnectionWrapper.cs b/Centaurus.Test.Integration/MockConnectionWrapper.cs index 335d4ac5..bd05135d 100644 --- a/Centaurus.Test.Integration/MockConnectionWrapper.cs +++ b/Centaurus.Test.Integration/MockConnectionWrapper.cs @@ -32,8 +32,8 @@ public override Task Connect(Uri uri, CancellationToken cancellationToken) if (!startupWrappers.TryGetValue(host, out var startup)) throw new Exception("Server is unavailable."); - if (!AlphaHostBuilder.ValidStates.Contains(startup.Context?.StateManager?.State ?? Models.State.Undefined)) - throw new InvalidOperationException("Alpha is not ready"); + //if (!AlphaHostBuilder.ValidStates.Contains(startup.Context?.NodesManager.CurrentNode.State ?? Models.State.Undefined)) + // throw new InvalidOperationException("Alpha is not ready"); var currentSideSocket = (MockWebSocket)WebSocket; currentSideSocket.KeyPair = new KeyPair(pubkey); diff --git a/Centaurus.Test.Integration/SDKHelper.cs b/Centaurus.Test.Integration/SDKHelper.cs index 33e5b9e6..22f711f2 100644 --- a/Centaurus.Test.Integration/SDKHelper.cs +++ b/Centaurus.Test.Integration/SDKHelper.cs @@ -8,6 +8,7 @@ using SdkState = Centaurus.NetSDK.ConstellationInfo.StateModel; using SdkProviderSettings = Centaurus.NetSDK.ConstellationInfo.ProviderSettings; using SdkProviderAsset = Centaurus.NetSDK.ConstellationInfo.ProviderSettings.ProviderAsset; +using Centaurus.Domain.Models; namespace Centaurus { diff --git a/Centaurus.Test.Utils/GlobalInitHelper.cs b/Centaurus.Test.Utils/GlobalInitHelper.cs index 987dad8f..470e4885 100644 --- a/Centaurus.Test.Utils/GlobalInitHelper.cs +++ b/Centaurus.Test.Utils/GlobalInitHelper.cs @@ -1,12 +1,11 @@ using Centaurus.Domain; +using Centaurus.Domain.StateManagers; using Centaurus.Models; using Centaurus.PaymentProvider; using Centaurus.PaymentProvider.Models; using Centaurus.PersistentStorage.Abstraction; -using NUnit.Framework; using System; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using System.Threading.Tasks; @@ -181,8 +180,8 @@ await Task.Factory.StartNew(() => public static void SetState(this ExecutionContext context, State state) { - var method = typeof(StateManager).GetMethod("SetState", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); - method.Invoke(context.StateManager, new object[] { state, null }); + var method = typeof(CurrentNode).GetMethod("SetState", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + method.Invoke(context.NodesManager.CurrentNode, new object[] { state, null }); } } -} +} \ No newline at end of file diff --git a/Centaurus.Test.XdrSerialization/MessageSerializationTests.cs b/Centaurus.Test.XdrSerialization/MessageSerializationTests.cs index 1617a6dd..3426f92f 100644 --- a/Centaurus.Test.XdrSerialization/MessageSerializationTests.cs +++ b/Centaurus.Test.XdrSerialization/MessageSerializationTests.cs @@ -56,9 +56,9 @@ public void OrderSerializationTest() [Test] public void NullValueSerializationTest() { - var signature = new AuditorSignatureInternal { AuditorId = 1, PayloadSignature = new TinySignature { Data = BinaryExtensions.RandomBytes(64) } }; + var signature = new NodeSignatureInternal { AuditorId = 1, PayloadSignature = new TinySignature { Data = BinaryExtensions.RandomBytes(64) } }; var rawData = XdrConverter.Serialize(signature); - signature = XdrConverter.Deserialize(rawData); + signature = XdrConverter.Deserialize(rawData); Assert.AreEqual(null, signature.TxSignature); } diff --git a/Centaurus/Alpha/AlphaHostBuilder.cs b/Centaurus/Alpha/AlphaHostBuilder.cs index 3f20bca2..5a5b2e9d 100644 --- a/Centaurus/Alpha/AlphaHostBuilder.cs +++ b/Centaurus/Alpha/AlphaHostBuilder.cs @@ -73,8 +73,6 @@ public IHost CreateHost() }).Build(); } - public static State[] ValidStates = new State[] { State.Rising, State.Running, State.Ready }; - private void SetupCertificate(Settings alphaSettings) { if (!alphaSettings.UseSecureConnection) @@ -169,9 +167,7 @@ public void ConfigureServices(IServiceCollection services) static async Task CentaurusWebSocketHandler(HttpContext context, Func next) { var centaurusContext = context.RequestServices.GetService(); - if (centaurusContext.StateManager != null - && ValidStates.Contains(centaurusContext.StateManager.State) - && TryGetPubKey(context, out var pubKey)) + if (TryGetPubKey(context, out var pubKey)) { using (var webSocket = await context.WebSockets.AcceptWebSocketAsync()) await centaurusContext.IncomingConnectionManager.OnNewConnection(webSocket, diff --git a/Centaurus/Alpha/Controllers/ConstellationController.cs b/Centaurus/Alpha/Controllers/ConstellationController.cs index b784adc8..7ccf32d2 100644 --- a/Centaurus/Alpha/Controllers/ConstellationController.cs +++ b/Centaurus/Alpha/Controllers/ConstellationController.cs @@ -1,4 +1,5 @@ using Centaurus.Domain; +using Centaurus.Domain.Models; using Centaurus.Models; using Microsoft.AspNetCore.Mvc; using System; @@ -21,33 +22,7 @@ public ConstellationController(ExecutionContext centaurusContext) [HttpGet("[action]")] public ConstellationInfo Info() { - ConstellationInfo info; - - var state = -1; - if (Context.StateManager != null) - state = (int)Context.StateManager.State; - if (state < (int)State.Running) - info = new ConstellationInfo - { - State = (State)state - }; - else - { - info = new ConstellationInfo - { - State = Context.StateManager.State, - Providers = Context.Constellation.Providers.ToArray(), - Auditors = Context.Constellation.Auditors - .Select(a => new ConstellationInfo.Auditor { PubKey = a.PubKey.GetAccountId(), Address = a.Address }) - .ToArray(), - MinAccountBalance = Context.Constellation.MinAccountBalance, - MinAllowedLotSize = Context.Constellation.MinAllowedLotSize, - Assets = Context.Constellation.Assets.ToArray(), - RequestRateLimits = Context.Constellation.RequestRateLimits - }; - } - - return info; + return Context.GetInfo(); } [HttpPost("[action]")] @@ -64,13 +39,7 @@ public async Task Update(ConstellationMessageEnvelope constellati if (constellationInitEnvelope == null) return StatusCode(415); - var constellationQuantum = new ConstellationQuantum { RequestEnvelope = constellationInitEnvelope }; - - constellationQuantum.Validate(Context); - - var quantumProcessingItem = Context.QuantumHandler.HandleAsync(constellationQuantum, Task.FromResult(true)); - - await quantumProcessingItem.OnProcessed; + await Context.HandleConstellationQuantum(constellationInitEnvelope); return new JsonResult(new InitResult { IsSuccess = true }); } diff --git a/Centaurus/Alpha/Models/RequestRateLimitsModel.cs b/Centaurus/Alpha/Models/RequestRateLimitsModel.cs deleted file mode 100644 index 88599af1..00000000 --- a/Centaurus/Alpha/Models/RequestRateLimitsModel.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Centaurus -{ - public class RequestRateLimitsModel - { - public uint MinuteLimit { get; set; } - - public uint HourLimit { get; set; } - } -} diff --git a/Centaurus/Program.cs b/Centaurus/Program.cs index ad62a1cd..a87caeea 100644 --- a/Centaurus/Program.cs +++ b/Centaurus/Program.cs @@ -46,7 +46,7 @@ static void ConfigureAndRun(T settings) logger = LogManager.GetCurrentClassLogger(); isLoggerInited = true; - var context = new Domain.ExecutionContext(settings, new PersistentStorageAbstraction(), PaymentProvidersFactoryBase.Default, OutgoingConnectionFactoryBase.Default); + var context = new Domain.ExecutionContext(settings); var startup = new Startup(context, AlphaHostFactoryBase.Default); var resetEvent = new ManualResetEvent(false); diff --git a/Centaurus/Startup.cs b/Centaurus/Startup.cs index bbe60b59..f8bf5c44 100644 --- a/Centaurus/Startup.cs +++ b/Centaurus/Startup.cs @@ -11,6 +11,8 @@ public class Startup : ContextualBase { private ManualResetEvent resetEvent; + + public Startup(Domain.ExecutionContext context, AlphaHostFactoryBase alphaHostFactory) : base(context) { @@ -30,17 +32,14 @@ public void Run(ManualResetEvent resetEvent) if (AlphaStartup != null) AlphaStartup.Run(); - Context.StateManager.StateChanged += Current_StateChanged; + Context.OnComplete += Context_OnComplete; } - private void Current_StateChanged(StateChangedEventArgs eventArgs) + private void Context_OnComplete() { - if (eventArgs.State == State.Failed) - { - var isSet = resetEvent.WaitOne(0); - if (!isSet) - resetEvent.Set(); - } + var isSet = resetEvent.WaitOne(0); + if (!isSet) + resetEvent.Set(); } public void Shutdown() From c788ba13b3f9ac7fa29701f7a191ef977db68d57 Mon Sep 17 00:00:00 2001 From: hawthorne-abendsen <49230725+hawthorne-abendsen@users.noreply.github.com> Date: Tue, 14 Dec 2021 22:39:59 +0200 Subject: [PATCH 05/13] Sync refactoring; SingleNodeSignatures sync; Nodes connection refactoring; --- Centaurus.Common/Settings/Settings.cs | 7 +- .../PublicInfo/ConstellationInfo.cs | 2 + Centaurus.Domain/Catchups/Catchup.cs | 70 +++--- .../Contexts/ConstellationSettingsManager.cs | 79 +++++++ Centaurus.Domain/Contexts/ExecutionContext.cs | 142 +++++------- Centaurus.Domain/Contexts/RoleManager.cs | 55 ----- Centaurus.Domain/DataProvider/DataProvider.cs | 7 +- ...PendingQuantumPersistentModelExtensions.cs | 2 +- .../QuantumPersistentModelExtensions.cs | 9 +- .../SignatureModelExtensions.cs | 4 +- .../Helpers/ConstellationQuantumExtesnions.cs | 4 +- .../ConstellationSettingsExtensions.cs | 21 +- Centaurus.Domain/Helpers/ContextHelper.cs | 4 +- .../Helpers/ExecutionContextExtensions.cs | 11 +- .../Helpers/RequestQuantumHelper.cs | 4 +- .../Helpers/SyncCursorUpdateExtensions.cs | 6 +- .../CatchupQuantaBatchHandler.cs | 2 +- .../CatchupQuantaBatchRequestHandler.cs | 15 +- .../EffectsRequestMessageHandler.cs | 2 +- .../HandshakeRequestHandler.cs | 16 +- .../HandshakeResponseHandler.cs | 4 +- .../MessageHandlers/MessageHandlerBase.cs | 16 +- .../Quanta/QuantumHandlerBase.cs | 4 +- .../QuantumMajoritySignaturesBatchHandler.cs | 8 +- .../RequestQuantaBatchHandler.cs | 8 +- ...cs => SingleNodeSignaturesBatchHandler.cs} | 14 +- .../MessageHandlers/StateMessageHandler.cs | 2 +- .../MessageHandlers/SyncCursorResetHandler.cs | 14 +- .../MessageHandlers/SyncQuantaBatchHandler.cs | 22 +- Centaurus.Domain/Nodes/Common/NodeApexes.cs | 2 +- Centaurus.Domain/Nodes/Common/NodeBase.cs | 18 +- .../Nodes/Common/NodeExtensions.cs | 10 +- .../Nodes/Common/NodeQuantaQueueLengths.cs | 2 +- Centaurus.Domain/Nodes/Common/NodeSettings.cs | 46 ++++ .../Nodes/Common/SyncCursorCollection.cs | 2 +- .../Nodes/Common/ValueAggregation.cs | 2 +- .../Nodes/CurrentNode/CurrentNode.cs | 12 +- .../Nodes/Managers/NodesManager.cs | 214 +++++++++++------- .../Nodes/Managers/StateNotifierWorker.cs | 3 +- .../Nodes/Managers/SyncSourceManager.cs | 142 ++++++++++++ .../Nodes/RemoteNode/RemoteNode.cs | 142 ------------ .../Nodes/RemoteNode/RemoteNodesCollection.cs | 75 ------ .../Nodes/RemoteNodes/RemoteNode.cs | 141 ++++++++++++ .../RemoteNodeConnection.cs | 36 +-- .../RemoteNodeConnectionManager.cs | 147 ++++++++++++ .../RemoteNodeCursor.cs | 0 .../RemoteNodes/RemoteNodesCollection.cs | 63 ++++++ .../Quanta/Handlers/QuantumHandler.cs | 61 +++-- .../Quanta/Processors/AlphaUpdateProcessor.cs | 23 +- .../ConstellationUpdateProcessor.cs | 7 +- .../Quanta/Processors/DepositProcessor.cs | 6 +- .../Processors/OrderCancellationProcessor.cs | 4 +- .../Processors/OrderRequestProcessor.cs | 14 +- .../Payments/PaymentRequestProcessor.cs | 16 +- .../Payments/WithdrawalRequestProcessor.cs | 6 +- .../Quanta/Processors/QuantumProcessorBase.cs | 8 +- .../QuantumProcessingItem.cs | 5 +- .../Quanta/QuantumSignatureValidator.cs | 4 +- .../Quanta/Sync/ApexItemsBatchPortion.cs | 17 +- Centaurus.Domain/Quanta/Sync/CursorGroup.cs | 1 - Centaurus.Domain/Quanta/Sync/ProxyWorker.cs | 8 +- .../Quanta/Sync/SyncCursorUpdate.cs | 6 +- .../Quanta/Sync/SyncQuantaDataWorker.cs | 25 +- Centaurus.Domain/Quanta/Sync/SyncStorage.cs | 76 +++++-- .../ResultManagers}/MajorityResult.cs | 6 +- .../ResultManagers/ResultManager.cs | 57 ++--- .../UpdatesManager/UpdatesManager.cs | 7 +- .../WebSockets/Centaurus/ConnectionBase.cs | 58 ++--- .../WebSockets/Centaurus/INodeConnection.cs | 3 +- .../Incoming/IncomingConnectionBase.cs | 7 +- .../Incoming/IncomingConnectionManager.cs | 91 +++----- .../Incoming/IncomingNodeConnection.cs | 1 - .../WebSockets/Centaurus/Notifier.cs | 2 +- .../Centaurus/Outgoing/OutgoingConnection.cs | 56 +---- .../Outgoing/OutgoingMessageStorage.cs | 120 ---------- .../GetPriceHistoryCommandHandler.cs | 4 +- Centaurus.Domain/exchange/Exchange.cs | 2 +- Centaurus.Domain/exchange/OrderMatcher.cs | 2 +- Centaurus.Domain/helpers/MajorityHelper.cs | 5 +- .../MessageEnvelopeExtenstions.cs | 6 +- .../Common/AuditorSignatureBase.cs | 2 +- .../InternalMessages/ConstellationQuantum.cs | 5 +- .../ConstellationRequestMessage.cs | 5 +- ...resBatch.cs => MajoritySignaturesBatch.cs} | 9 +- .../SingleNodeSignaturesBatch.cs} | 11 +- .../InternalMessages/StateMessage.cs | 2 +- .../InternalMessages/SyncCursor.cs | 4 +- .../InternalMessages/SyncQuantaBatch.cs | 3 - Centaurus.Models/Messages/Message.cs | 6 +- Centaurus.Models/Messages/MessageTypes.cs | 4 +- Centaurus.Models/Messages/RequestQuantum.cs | 16 ++ .../Requests/AccountDataRequestQuantum.cs | 2 +- .../Requests/ClientRequestQuantumBase.cs | 11 + Centaurus.Models/Requests/RequestQuantum.cs | 2 +- .../Requests/RequestQuantumBase.cs | 17 -- .../Requests/WithdrawalRequestQuantum.cs | 2 +- .../Results/AuditorSignaturesBatch.cs | 13 -- .../Query/StateQuery.cs | 3 +- .../Storage/StorageIterator.cs | 10 + .../AlphaMessageHandlersTests.cs | 54 ++--- .../AuditorMessageHandlersTests.cs | 8 +- .../BaseMessageHandlerTests.cs | 16 +- .../BaseQuantumHandlerTests.cs | 22 +- Centaurus.Test.Domain/OrderbookTests.cs | 12 +- .../PendingQuantaPersistentTest.cs | 8 +- .../QuantumHandlerPerformanceTest.cs | 4 +- Centaurus.Test.Domain/QuantumStorageTests.cs | 13 +- .../UpdatesTest.cs | 8 +- .../IntegrationTestEnvironment.cs | 2 +- .../IntegrationTestEnvironmentExtensions.cs | 2 +- .../IntegrationTests.cs | 23 +- Centaurus.Test.Utils/GlobalInitHelper.cs | 31 ++- Centaurus.Test.Utils/MockStorage.cs | 2 +- .../MessageSerializationTests.cs | 2 +- .../Controllers/ConstellationController.cs | 6 +- Centaurus/Startup.cs | 2 +- 116 files changed, 1432 insertions(+), 1235 deletions(-) create mode 100644 Centaurus.Domain/Contexts/ConstellationSettingsManager.cs delete mode 100644 Centaurus.Domain/Contexts/RoleManager.cs rename Centaurus.Domain/MessageHandlers/{ => Handshake}/HandshakeRequestHandler.cs (51%) rename Centaurus.Domain/MessageHandlers/{HandshakeResponseHandlers => Handshake}/HandshakeResponseHandler.cs (93%) rename Centaurus.Domain/MessageHandlers/{ResultBatchHandler.cs => SingleNodeSignaturesBatchHandler.cs} (55%) create mode 100644 Centaurus.Domain/Nodes/Common/NodeSettings.cs create mode 100644 Centaurus.Domain/Nodes/Managers/SyncSourceManager.cs delete mode 100644 Centaurus.Domain/Nodes/RemoteNode/RemoteNode.cs delete mode 100644 Centaurus.Domain/Nodes/RemoteNode/RemoteNodesCollection.cs create mode 100644 Centaurus.Domain/Nodes/RemoteNodes/RemoteNode.cs rename Centaurus.Domain/Nodes/{RemoteNode => RemoteNodes}/RemoteNodeConnection.cs (75%) create mode 100644 Centaurus.Domain/Nodes/RemoteNodes/RemoteNodeConnectionManager.cs rename Centaurus.Domain/Nodes/{RemoteNode => RemoteNodes}/RemoteNodeCursor.cs (100%) create mode 100644 Centaurus.Domain/Nodes/RemoteNodes/RemoteNodesCollection.cs rename {Centaurus.Models/Results => Centaurus.Domain/ResultManagers}/MajorityResult.cs (53%) delete mode 100644 Centaurus.Domain/WebSockets/Centaurus/Outgoing/OutgoingMessageStorage.cs rename Centaurus.Models/InternalMessages/{QuantumMajoritySignaturesBatch.cs => MajoritySignaturesBatch.cs} (57%) rename Centaurus.Models/{Results/AuditorResult.cs => InternalMessages/SingleNodeSignaturesBatch.cs} (58%) create mode 100644 Centaurus.Models/Messages/RequestQuantum.cs create mode 100644 Centaurus.Models/Requests/ClientRequestQuantumBase.cs delete mode 100644 Centaurus.Models/Requests/RequestQuantumBase.cs delete mode 100644 Centaurus.Models/Results/AuditorSignaturesBatch.cs diff --git a/Centaurus.Common/Settings/Settings.cs b/Centaurus.Common/Settings/Settings.cs index 47ffb9aa..90506ea0 100644 --- a/Centaurus.Common/Settings/Settings.cs +++ b/Centaurus.Common/Settings/Settings.cs @@ -42,8 +42,11 @@ public class Settings [Option("sync_batch_size", Default = 500, HelpText = "Max quanta sync batch size.")] public int SyncBatchSize { get; set; } - [Option("participation_level", Required = true, HelpText = "Centaurus node participation level. '1' = Prime and '2' = Auditor")] - public int ParticipationLevel { get; set; } + [Option("catchup_timeout", Default = 15, HelpText = "Catchup timeout in seconds.")] + public int CatchupTimeout { get; set; } + + [Option('p', "prime_node", Default = false, HelpText = "Centaurus node participation level. If true the node is Prime, otherwise the node is Auditor")] + public bool IsPrimeNode { get; set; } [Option("payment_config", Required = true, HelpText = "Payment providers config path.")] public string PaymentConfigPath { get; set; } diff --git a/Centaurus.Domain.Models/PublicInfo/ConstellationInfo.cs b/Centaurus.Domain.Models/PublicInfo/ConstellationInfo.cs index a5aacf9a..5136d5c7 100644 --- a/Centaurus.Domain.Models/PublicInfo/ConstellationInfo.cs +++ b/Centaurus.Domain.Models/PublicInfo/ConstellationInfo.cs @@ -4,6 +4,8 @@ namespace Centaurus.Domain.Models { public class ConstellationInfo { + public string Alpha { get; set; } + public ulong Apex { get; set; } public string PubKey { get; set; } diff --git a/Centaurus.Domain/Catchups/Catchup.cs b/Centaurus.Domain/Catchups/Catchup.cs index ba4c0dcf..b1421b0f 100644 --- a/Centaurus.Domain/Catchups/Catchup.cs +++ b/Centaurus.Domain/Catchups/Catchup.cs @@ -28,7 +28,7 @@ public Catchup(ExecutionContext context) private Dictionary validAuditorStates = new Dictionary(); private System.Timers.Timer applyDataTimer; - public async Task AddAuditorState(RawPubKey pubKey, CatchupQuantaBatch auditorState) + public async Task AddNodeBatch(RawPubKey pubKey, CatchupQuantaBatch nodeBatch) { await semaphoreSlim.WaitAsync(); try @@ -40,24 +40,13 @@ public async Task AddAuditorState(RawPubKey pubKey, CatchupQuantaBatch auditorSt return; } - if (!(applyDataTimer.Enabled || pubKey.Equals(Context.Settings.KeyPair))) //start timer + if (!(applyDataTimer.Enabled)) //start timer applyDataTimer.Start(); - if (!allAuditorStates.TryGetValue(pubKey, out var pendingAuditorBatch)) - { - pendingAuditorBatch = auditorState; - allAuditorStates.Add(pubKey, auditorState); - if (applyDataTimer.Enabled) - applyDataTimer.Reset(); - logger.Trace($"Auditor state from {pubKey.GetAccountId()} added."); - } - else if (!AddQuanta(pubKey, pendingAuditorBatch, auditorState)) //check if auditor sent all quanta already - { - logger.Warn($"Unable to add auditor {pubKey.GetAccountId()} state."); + if (!TryGetNodeBatch(pubKey, nodeBatch, out var aggregatedNodeBatch)) return; - } - if (pendingAuditorBatch.HasMore) //wait while auditor will send all quanta it has + if (aggregatedNodeBatch.HasMore) //wait while auditor will send all quanta it has { logger.Trace($"Auditor {pubKey.GetAccountId()} has more quanta. Timer is reset."); applyDataTimer.Reset(); //if timer is running reset it. We need to try to wait all possible auditors data @@ -66,7 +55,7 @@ public async Task AddAuditorState(RawPubKey pubKey, CatchupQuantaBatch auditorSt logger.Trace($"Auditor {pubKey.GetAccountId()} state is validated."); - validAuditorStates.Add(pubKey, pendingAuditorBatch); + validAuditorStates.Add(pubKey, aggregatedNodeBatch); int majority = Context.GetMajorityCount(), totalAuditorsCount = Context.GetTotalAuditorsCount(); @@ -84,6 +73,24 @@ public async Task AddAuditorState(RawPubKey pubKey, CatchupQuantaBatch auditorSt } } + private bool TryGetNodeBatch(RawPubKey pubKey, CatchupQuantaBatch batch, out CatchupQuantaBatch aggregatedNodeBatch) + { + if (!allAuditorStates.TryGetValue(pubKey, out aggregatedNodeBatch)) + { + aggregatedNodeBatch = batch; + allAuditorStates.Add(pubKey, batch); + applyDataTimer.Reset(); + logger.Trace($"Auditor state from {pubKey.GetAccountId()} added."); + } + else if (!AddQuanta(pubKey, aggregatedNodeBatch, batch)) //check if auditor sent all quanta already + { + logger.Warn($"Unable to add auditor {pubKey.GetAccountId()} state."); + return false; + } + + return true; + } + private bool AddQuanta(RawPubKey pubKey, CatchupQuantaBatch currentBatch, CatchupQuantaBatch newBatch) { if (!currentBatch.HasMore @@ -118,7 +125,7 @@ private async Task TryApplyAuditorsData() int majority = Context.GetMajorityCount(), totalAuditorsCount = Context.GetTotalAuditorsCount(); - if (Context.HasMajority(validAuditorStates.Count)) + if (Context.HasMajority(validAuditorStates.Count) || !Context.NodesManager.IsAlpha) { await ApplyAuditorsData(); allAuditorStates.Clear(); @@ -146,7 +153,7 @@ private async Task TryApplyAuditorsData() private void InitTimer() { applyDataTimer = new System.Timers.Timer(); - applyDataTimer.Interval = 15000; //15 sec + applyDataTimer.Interval = TimeSpan.FromSeconds(Context.Settings.CatchupTimeout).TotalMilliseconds; applyDataTimer.AutoReset = false; applyDataTimer.Elapsed += ApplyDataTimer_Elapsed; } @@ -162,7 +169,6 @@ private void ApplyDataTimer_Elapsed(object sender, System.Timers.ElapsedEventArg { semaphoreSlim.Release(); } - } private async Task ApplyAuditorsData() @@ -176,7 +182,7 @@ private async Task ApplyQuanta(List<(Quantum quantum, List auditors) GetAuditorsSettings(ConstellationSettings constellation) { - var auditors = constellation.Auditors - .Select(a => new RawPubKey(a.PubKey)) - .ToList(); - var alphaId = constellation.GetAuditorId(constellation.Alpha); - if (alphaId < 0) - throw new ArgumentNullException("Unable to find Alpha id."); - - return (alphaId, auditors); + if (Context.NodesManager.AlphaNode == null) + throw new ArgumentNullException("Alpha node is not connected."); + + return (Context.NodesManager.AlphaNode.Id, Context.NodesManager.AllNodes.Select(n => n.PubKey).ToList()); } private (Quantum quantum, List signatures) GetQuantumData(ulong apex, List allQuanta, int alphaId, List auditors) @@ -262,18 +264,18 @@ private async Task ApplyQuanta(List<(Quantum quantum, List s.AuditorId == alphaId)) + if (!currentItem.Signatures.Any(s => s.NodeId == alphaId)) continue; //validate each signature foreach (var signature in currentItem.Signatures) { //skip if current auditor is already presented in signatures, but force alpha signature to be checked - if (signature.AuditorId != alphaId && signatures.ContainsKey(signature.AuditorId)) + if (signature.NodeId != alphaId && signatures.ContainsKey(signature.NodeId)) continue; //try get auditor - var signer = auditors.ElementAtOrDefault(signature.AuditorId); + var signer = auditors.ElementAtOrDefault(signature.NodeId); //if auditor is not found or it's signature already added, move to the next signature if (signer == null || !signature.PayloadSignature.IsValid(signer, currentPayloadHash)) continue; @@ -281,7 +283,7 @@ private async Task ApplyQuanta(List<(Quantum quantum, List + /// This class holds recent constellation settings, to be able to obtain relevant node ids + /// + public class ConstellationSettingsManager : ContextualBase + { + public ConstellationSettingsManager(ExecutionContext context) + : base(context) + { + } + + public ConstellationSettings Current { get; private set; } + + public void Add(ConstellationSettings newSettings) + { + if (newSettings == null) + throw new ArgumentNullException(nameof(newSettings)); + + if (Current != null && Current.Apex >= newSettings.Apex) + throw new ArgumentException("New constellation settings apex is lower than the current one.", nameof(newSettings)); + + lock (syncRoot) + { + settings.Add(newSettings.Apex, newSettings); + Current = newSettings; + Cleanup(); + } + } + + public bool TryGetForApex(ulong apex, out ConstellationSettings apexSettings) + { + apexSettings = null; + if (Current == null) //if current is null, than there is no constellation settings yet + return false; + + if (apex >= Current.Apex) + { + apexSettings = Current; + return true; + } + + lock (syncRoot) + { + //looking for the first settings where apex is lower or equal to the request apex + foreach (var settingApex in settings.Keys) + { + //the setting is newer than apex + if (settingApex > apex) + continue; + + apexSettings = settings[apex]; + return true; + } + } + return false; + } + + private object syncRoot = new { }; + + private SortedDictionary settings = new SortedDictionary(); + + private void Cleanup() + { + //remove old data + if (settings.Count > 1000) + { + var firstApex = settings.Keys.First(); + settings.Remove(firstApex); + } + } + } +} diff --git a/Centaurus.Domain/Contexts/ExecutionContext.cs b/Centaurus.Domain/Contexts/ExecutionContext.cs index 590180f6..300f3618 100644 --- a/Centaurus.Domain/Contexts/ExecutionContext.cs +++ b/Centaurus.Domain/Contexts/ExecutionContext.cs @@ -31,9 +31,9 @@ public ExecutionContext(Settings settings) /// Application config /// Persistent storage object internal ExecutionContext( - Settings settings, - IPersistentStorage storage, - PaymentProvidersFactoryBase paymentProviderFactory, + Settings settings, + IPersistentStorage storage, + PaymentProvidersFactoryBase paymentProviderFactory, OutgoingConnectionFactoryBase connectionFactory) { DynamicSerializersInitializer.Init(); @@ -48,16 +48,12 @@ internal ExecutionContext( PersistentStorage.Connect(GetAbsolutePath(Settings.ConnectionString)); - RoleManager = new RoleManager((CentaurusNodeParticipationLevel)Settings.ParticipationLevel); - ExtensionsManager = new ExtensionsManager(GetAbsolutePath(Settings.ExtensionsConfigFilePath)); DataProvider = new DataProvider(this); QuantumProcessor = new QuantumProcessorsStorage(this); - PendingUpdatesManager = new UpdatesManager(this); - MessageHandlers = new MessageHandlers(this); InfoCommandsHandlers = new InfoCommandsHandlers(this); @@ -68,48 +64,58 @@ internal ExecutionContext( InfoConnectionManager = new InfoConnectionManager(this); - SyncQuantaDataWorker = new SyncQuantaDataWorker(this); - ProxyWorker = new ProxyWorker(this); Catchup = new Catchup(this); - var persistentData = DataProvider.GetPersistentData(); + ConstellationSettingsManager = new ConstellationSettingsManager(this); - StateNotifier = new StateNotifierWorker(this); + var persistentData = DataProvider.GetPersistentData(); var currentState = persistentData == default ? State.WaitingForInit : State.Rising; NodesManager = new NodesManager(this, currentState); + NodesManager.CurrentNode.StateChanged += CurrentNode_StateChanged; var lastApex = persistentData.snapshot?.Apex ?? 0; var lastHash = persistentData.snapshot?.LastHash ?? new byte[32]; - SyncStorage = new SyncStorage(this, lastApex); + StateNotifier = new StateNotifierWorker(this); + + PendingUpdatesManager = new UpdatesManager(this); QuantumHandler = new QuantumHandler(this, lastApex, lastHash); ResultManager = new ResultManager(this); - //apply data, if presented in db - if (persistentData != default) - { - //apply snapshot if not null - if (persistentData.snapshot != null) - Init(persistentData.snapshot); + PerformanceStatisticsManager = new PerformanceStatisticsManager(this); - if (persistentData.pendingQuanta != null) - HandlePendingQuanta(persistentData.pendingQuanta); + //apply snapshot if not null + if (persistentData.snapshot != null) + Init(persistentData.snapshot); + else + SetNodes().Wait(); - if (!IsAlpha) - NodesManager.CurrentNode.Rised(); - } + if (persistentData.pendingQuanta != null) + HandlePendingQuanta(persistentData.pendingQuanta); - if (Constellation == null) - SetNodes(); + SyncStorage = new SyncStorage(this, lastApex); + + SyncQuantaDataWorker = new SyncQuantaDataWorker(this); + } + + private void CurrentNode_StateChanged(StateChangedEventArgs stateChangedEventArgs) + { + var state = stateChangedEventArgs.State; + var prevState = stateChangedEventArgs.PrevState; + if (state != State.Ready && prevState == State.Ready) //close all connections (except auditors) + IncomingConnectionManager.CloseAllConnections(false).Wait(); + if (prevState == State.Rising && (state == State.Running || state == State.Ready)) + //after node successfully started, the pending quanta can be deleted + PersistentStorage.DeletePendingQuanta(); } private void HandlePendingQuanta(List pendingQuanta) { - _ = Catchup.AddAuditorState(Settings.KeyPair, new CatchupQuantaBatch { Quanta = pendingQuanta, HasMore = false }); + _ = Catchup.AddNodeBatch(Settings.KeyPair, new CatchupQuantaBatch { Quanta = pendingQuanta, HasMore = false }); } private string GetAbsolutePath(string path) @@ -123,18 +129,18 @@ private string GetAbsolutePath(string path) public void Init(Snapshot snapshot) { - UpdateConstellationSettings(snapshot.ConstellationSettings); + UpdateConstellationSettings(snapshot.ConstellationSettings).Wait(); AccountStorage = new AccountStorage(snapshot.Accounts); - Exchange = Exchange.RestoreExchange(Constellation.Assets, snapshot.Orders, RoleManager.ParticipationLevel == CentaurusNodeParticipationLevel.Prime); + Exchange = Exchange.RestoreExchange(ConstellationSettingsManager.Current.Assets, snapshot.Orders, Settings.IsPrimeNode); SetupPaymentProviders(snapshot.Cursors); AnalyticsManager = new AnalyticsManager( PersistentStorage, DepthsSubscription.Precisions.ToList(), - Constellation.Assets.Where(a => !a.IsQuoteAsset).Select(a => a.Code).ToList(), //all but quote asset + ConstellationSettingsManager.Current.Assets.Where(a => !a.IsQuoteAsset).Select(a => a.Code).ToList(), //all but quote asset snapshot.Orders.Select(o => o.Order.ToOrderInfo()).ToList() ); @@ -146,20 +152,12 @@ public void Init(Snapshot snapshot) Exchange.OnUpdates += Exchange_OnUpdates; } - public void UpdateConstellationSettings(ConstellationSettings constellationSettings) + public async Task UpdateConstellationSettings(ConstellationSettings constellationSettings) { - Constellation = constellationSettings; - - AuditorIds = Constellation.Auditors.ToDictionary(a => Constellation.GetAuditorId(a.PubKey), a => a.PubKey); - AuditorPubKeys = AuditorIds.ToDictionary(a => a.Value, a => a.Key); - AlphaId = AuditorPubKeys[Constellation.Alpha]; + ConstellationSettingsManager.Add(constellationSettings); //update current auditors - SetNodes(); - - SetRole(); - - IncomingConnectionManager.CleanupAuditorConnections(); + await SetNodes(); } public void Complete() @@ -187,39 +185,28 @@ private void PersistPendingQuanta() PendingUpdatesManager.UpdateBatch(true); //save all completed quanta PendingUpdatesManager.ApplyUpdates(); + + //if init quantum wasn't save, don't save pending quanta + if (PendingUpdatesManager.LastPersistedApex == 0) + return; + //persist all pending quanta PendingUpdatesManager.PersistPendingQuanta(); } - private void SetNodes() + private async Task SetNodes() { - var auditors = Constellation != null - ? Constellation.Auditors.ToList() + var auditors = ConstellationSettingsManager.Current != null + ? ConstellationSettingsManager.Current.Auditors.ToList() : Settings.GenesisAuditors.Select(a => new Auditor { PubKey = a.PubKey, Address = a.Address }).ToList(); - NodesManager.SetAuditors(auditors); - } - - private void SetRole() - { - if (RoleManager.ParticipationLevel == CentaurusNodeParticipationLevel.Auditor) - { - if (Constellation.Alpha.Equals((RawPubKey)Settings.KeyPair)) - throw new InvalidOperationException("Server with Auditor level cannot be set as Alpha."); - return; - } - if (Constellation.Alpha.Equals((RawPubKey)Settings.KeyPair)) - { - RoleManager.SetRole(CentaurusNodeRole.Alpha); - } - else - RoleManager.SetRole(CentaurusNodeRole.Beta); + await NodesManager.SetAuditors(auditors); } private void SetupPaymentProviders(Dictionary cursors) { PaymentProvidersManager?.Dispose(); - var settings = Constellation.Providers.Select(p => + var settings = ConstellationSettingsManager.Current.Providers.Select(p => { var providerId = PaymentProviderBase.GetProviderId(p.Provider, p.Name); cursors.TryGetValue(providerId, out var cursor); @@ -238,8 +225,8 @@ private void SetupPaymentProviders(Dictionary cursors) private void PaymentProvider_OnPaymentCommit(PaymentProviderBase paymentProvider, PaymentProvider.Models.DepositNotificationModel notification) { - if (!IsAlpha || NodesManager.CurrentNode.State != State.Ready) - throw new OperationCanceledException($"Current server is not ready to process deposits. Is Alpha: {IsAlpha}, State: {NodesManager.CurrentNode.State}"); + if (!(NodesManager.IsAlpha && NodesManager.CurrentNode.State == State.Ready)) + throw new OperationCanceledException($"Current server is not ready to process deposits. Is Alpha: {NodesManager.IsAlpha}, State: {NodesManager.CurrentNode.State}"); QuantumHandler.HandleAsync(new DepositQuantum { Source = notification.ToDomainModel() }, Task.FromResult(true)); } @@ -259,16 +246,7 @@ public void Dispose() PerformanceStatisticsManager.Dispose(); } - private async void AppState_StateChanged(StateChangedEventArgs stateChangedEventArgs) - { - var state = stateChangedEventArgs.State; - var prevState = stateChangedEventArgs.PrevState; - if (state != State.Ready && prevState == State.Ready) //close all connections (except auditors) - await IncomingConnectionManager.CloseAllConnections(false); - if (prevState == State.Rising && state == State.Running || state == State.Ready) - //after node successfully started, the pending quanta can be deleted - PersistentStorage.DeletePendingQuanta(); - } + public event Action OnComplete; public DataProvider DataProvider { get; } @@ -276,16 +254,12 @@ private async void AppState_StateChanged(StateChangedEventArgs stateChangedEvent public UpdatesManager PendingUpdatesManager { get; } - public bool IsAlpha => RoleManager.Role == CentaurusNodeRole.Alpha; - public ExtensionsManager ExtensionsManager { get; } public QuantumProcessorsStorage QuantumProcessor { get; } public SyncStorage SyncStorage { get; } - public RoleManager RoleManager { get; } - public IPersistentStorage PersistentStorage { get; } public Settings Settings { get; } @@ -296,6 +270,8 @@ private async void AppState_StateChanged(StateChangedEventArgs stateChangedEvent internal StateNotifierWorker StateNotifier { get; } + internal ConstellationSettingsManager ConstellationSettingsManager { get; } + public QuantumHandler QuantumHandler { get; } public IncomingConnectionManager IncomingConnectionManager { get; } @@ -322,20 +298,8 @@ private async void AppState_StateChanged(StateChangedEventArgs stateChangedEvent private PerformanceStatisticsManager PerformanceStatisticsManager { get; } - public HashSet AssetIds { get; private set; } - public AnalyticsManager AnalyticsManager { get; private set; } - public ConstellationSettings Constellation { get; private set; } - - public Dictionary AuditorIds { get; private set; } - - public Dictionary AuditorPubKeys { get; private set; } - public OutgoingConnectionFactoryBase OutgoingConnectionFactory { get; } - - public byte AlphaId { get; private set; } - - public event Action OnComplete; } } \ No newline at end of file diff --git a/Centaurus.Domain/Contexts/RoleManager.cs b/Centaurus.Domain/Contexts/RoleManager.cs deleted file mode 100644 index db8eadbc..00000000 --- a/Centaurus.Domain/Contexts/RoleManager.cs +++ /dev/null @@ -1,55 +0,0 @@ -using System; - -namespace Centaurus.Domain -{ - //TODO: add server types (FullNode, Auditor) - public class RoleManager - { - public RoleManager(CentaurusNodeParticipationLevel participationLevel) - { - ParticipationLevel = participationLevel; - if (participationLevel == CentaurusNodeParticipationLevel.Undefined) - throw new InvalidOperationException($"Invalid participation level."); - if (participationLevel == CentaurusNodeParticipationLevel.Auditor) - SetRole(CentaurusNodeRole.Auditor); - else - SetRole(CentaurusNodeRole.Beta); - } - - private object syncRoot = new { }; - - private CentaurusNodeRole role; - public CentaurusNodeRole Role => role; - - public CentaurusNodeParticipationLevel ParticipationLevel { get; } - - public void SetRole(CentaurusNodeRole role) - { - if (this.role != role) - { - if ((role == CentaurusNodeRole.Alpha || role == CentaurusNodeRole.Beta) && ParticipationLevel != CentaurusNodeParticipationLevel.Prime) - throw new Exception($"Only nodes with Prime participation level can be switched to {role}."); - lock (syncRoot) - { - if (this.role != role) - this.role = role; - } - } - } - } - - public enum CentaurusNodeRole - { - Undefined = 0, - Alpha = 1, //point of quantum sync - Beta = 2, //full node, can accept requests, can - Auditor = 3 - } - - public enum CentaurusNodeParticipationLevel - { - Undefined = 0, - Prime = 1, - Auditor = 2 - } -} \ No newline at end of file diff --git a/Centaurus.Domain/DataProvider/DataProvider.cs b/Centaurus.Domain/DataProvider/DataProvider.cs index d9405823..c6e39b11 100644 --- a/Centaurus.Domain/DataProvider/DataProvider.cs +++ b/Centaurus.Domain/DataProvider/DataProvider.cs @@ -119,7 +119,8 @@ public Snapshot GetSnapshot(ulong apex) if (minRevertApex == 0 && apex != lastApex || apex < minRevertApex) throw new InvalidOperationException($"Lack of data to revert to {apex} apex."); - var settings = GetConstellationSettings(); + //get current settings + var settings = GetConstellationSettings(ulong.MaxValue); if (settings == null) return null; @@ -272,9 +273,9 @@ public List LoadPendingQuanta() /// /// /// - private ConstellationSettings GetConstellationSettings() + public ConstellationSettings GetConstellationSettings(ulong apex) { - var settingsModel = Context.PersistentStorage.LoadSettings(ulong.MaxValue); + var settingsModel = Context.PersistentStorage.LoadSettings(apex); if (settingsModel == null) return null; diff --git a/Centaurus.Domain/DataProvider/PersistentModelExtensions/PendingQuantumPersistentModelExtensions.cs b/Centaurus.Domain/DataProvider/PersistentModelExtensions/PendingQuantumPersistentModelExtensions.cs index 21970432..da38e5f1 100644 --- a/Centaurus.Domain/DataProvider/PersistentModelExtensions/PendingQuantumPersistentModelExtensions.cs +++ b/Centaurus.Domain/DataProvider/PersistentModelExtensions/PendingQuantumPersistentModelExtensions.cs @@ -30,7 +30,7 @@ public static NodeSignatureInternal ToNodeSignature(this SignatureModel signatur { return new NodeSignatureInternal { - AuditorId = signature.AuditorId, + NodeId = signature.AuditorId, PayloadSignature = new TinySignature { Data = signature.PayloadSignature }, TxSignature = signature.TxSignature, TxSigner = signature.TxSigner diff --git a/Centaurus.Domain/DataProvider/PersistentModelExtensions/QuantumPersistentModelExtensions.cs b/Centaurus.Domain/DataProvider/PersistentModelExtensions/QuantumPersistentModelExtensions.cs index 908d132a..255b499f 100644 --- a/Centaurus.Domain/DataProvider/PersistentModelExtensions/QuantumPersistentModelExtensions.cs +++ b/Centaurus.Domain/DataProvider/PersistentModelExtensions/QuantumPersistentModelExtensions.cs @@ -12,17 +12,16 @@ public static SyncQuantaBatchItem ToBatchItemQuantum(this QuantumPersistentModel { return new SyncQuantaBatchItem { - Quantum = (Quantum)XdrConverter.Deserialize(quantumPersistentModel.RawQuantum), - AlphaSignature = quantumPersistentModel.Signatures.First().ToDomainModel() + Quantum = (Quantum)XdrConverter.Deserialize(quantumPersistentModel.RawQuantum) }; } - public static QuantumSignatures ToQuantumSignatures(this QuantumPersistentModel quantumPersistentModel) + public static MajoritySignaturesBatchItem ToMajoritySignatures(this QuantumPersistentModel quantumPersistentModel) { - return new QuantumSignatures + return new MajoritySignaturesBatchItem { Apex = quantumPersistentModel.Apex, - Signatures = quantumPersistentModel.Signatures.Skip(1).Select(s => s.ToDomainModel()).ToList() + Signatures = quantumPersistentModel.Signatures.Select(s => s.ToDomainModel()).ToList() }; } } diff --git a/Centaurus.Domain/DataProvider/PersistentModelExtensions/SignatureModelExtensions.cs b/Centaurus.Domain/DataProvider/PersistentModelExtensions/SignatureModelExtensions.cs index 871df88d..b69ab59f 100644 --- a/Centaurus.Domain/DataProvider/PersistentModelExtensions/SignatureModelExtensions.cs +++ b/Centaurus.Domain/DataProvider/PersistentModelExtensions/SignatureModelExtensions.cs @@ -15,7 +15,7 @@ public static SignatureModel ToPersistenModel(this NodeSignatureInternal auditor return new SignatureModel { - AuditorId = (byte)auditorSignature.AuditorId, + AuditorId = (byte)auditorSignature.NodeId, PayloadSignature = auditorSignature.PayloadSignature.Data, TxSignature = auditorSignature.TxSignature, TxSigner = auditorSignature.TxSigner @@ -29,7 +29,7 @@ public static NodeSignatureInternal ToDomainModel(this SignatureModel auditorSig return new NodeSignatureInternal { - AuditorId = auditorSignature.AuditorId, + NodeId = auditorSignature.AuditorId, PayloadSignature = new TinySignature { Data = auditorSignature.PayloadSignature }, TxSignature = auditorSignature.TxSignature, TxSigner = auditorSignature.TxSigner diff --git a/Centaurus.Domain/Helpers/ConstellationQuantumExtesnions.cs b/Centaurus.Domain/Helpers/ConstellationQuantumExtesnions.cs index 2b3bf6f0..7f69d574 100644 --- a/Centaurus.Domain/Helpers/ConstellationQuantumExtesnions.cs +++ b/Centaurus.Domain/Helpers/ConstellationQuantumExtesnions.cs @@ -19,7 +19,9 @@ public static void Validate(this ConstellationQuantum constellationQuantum, Exec if (constellationQuantum.RequestMessage == null || !(constellationQuantum.RequestMessage is ConstellationRequestMessage)) throw new Exception("Invalid message type."); - var envelope = constellationQuantum.RequestEnvelope; + var envelope = constellationQuantum.RequestEnvelope as ConstellationMessageEnvelope; + if (envelope == null) + throw new InvalidOperationException($"Constellation quantum must be wrapped with {typeof(ConstellationMessageEnvelope).Name}."); var signatures = envelope.Signatures; //validate that signatures are unique diff --git a/Centaurus.Domain/Helpers/ConstellationSettingsExtensions.cs b/Centaurus.Domain/Helpers/ConstellationSettingsExtensions.cs index cf55f761..c7c3c7c6 100644 --- a/Centaurus.Domain/Helpers/ConstellationSettingsExtensions.cs +++ b/Centaurus.Domain/Helpers/ConstellationSettingsExtensions.cs @@ -9,11 +9,28 @@ namespace Centaurus.Domain { public static class ConstellationSettingsExtensions { - public static byte GetAuditorId(this ConstellationSettings constellation, RawPubKey rawPubKey) + public static byte GetNodeId(this ConstellationSettings constellation, RawPubKey rawPubKey) { if (constellation == null) throw new ArgumentNullException(nameof(constellation)); - return (byte)constellation.Auditors.FindIndex(a => a.PubKey.Equals(rawPubKey)); + + var auditorIndex = constellation.Auditors.FindIndex(a => a.PubKey.Equals(rawPubKey)); + if (auditorIndex == -1) + throw new ArgumentNullException($"{rawPubKey.GetAccountId()} is not an auditor."); + //all auditor must have id that is greater than zero + return (byte)(auditorIndex + 1); + } + + public static bool TryGetNodePubKey(this ConstellationSettings constellation, byte nodeId, out RawPubKey pubKey) + { + pubKey = null; + if (constellation == null) + return false; + if (constellation.Auditors.Count < nodeId) + return false; + var nodeIndex = nodeId - 1; + pubKey = constellation.Auditors[nodeIndex].PubKey; + return true; } public static Snapshot ToSnapshot(this ConstellationSettings settings, ulong apex, List accounts, List orders, Dictionary cursors, byte[] quantumHash) diff --git a/Centaurus.Domain/Helpers/ContextHelper.cs b/Centaurus.Domain/Helpers/ContextHelper.cs index f99c092e..3cac70ca 100644 --- a/Centaurus.Domain/Helpers/ContextHelper.cs +++ b/Centaurus.Domain/Helpers/ContextHelper.cs @@ -14,9 +14,9 @@ public static class ContextHelper /// public static List GetRelevantAuditors(this ExecutionContext context) { - return (context.Constellation == null + return (context.ConstellationSettingsManager.Current == null ? context.Settings.GenesisAuditors.Select(a => a.PubKey) - : context.Constellation.Auditors.Select(a => (KeyPair)a.PubKey)) + : context.ConstellationSettingsManager.Current.Auditors.Select(a => (KeyPair)a.PubKey)) .ToList(); } } diff --git a/Centaurus.Domain/Helpers/ExecutionContextExtensions.cs b/Centaurus.Domain/Helpers/ExecutionContextExtensions.cs index 6ecddcdd..371cab8e 100644 --- a/Centaurus.Domain/Helpers/ExecutionContextExtensions.cs +++ b/Centaurus.Domain/Helpers/ExecutionContextExtensions.cs @@ -15,9 +15,9 @@ public static List GetAuditors(this ExecutionContext context) if (context == null) throw new ArgumentNullException(nameof(context)); - return (context.Constellation == null + return (context.ConstellationSettingsManager.Current == null ? context.Settings.GenesisAuditors.Select(a => (RawPubKey)a.PubKey) - : context.Constellation.Auditors.Select(a => a.PubKey)) + : context.ConstellationSettingsManager.Current.Auditors.Select(a => a.PubKey)) .ToList(); } @@ -34,13 +34,14 @@ public static ConstellationInfo GetInfo(this ExecutionContext context) PubKey = currentNode.AccountId }; - var constellationSettings = context.Constellation; + var constellationSettings = context.ConstellationSettingsManager.Current; if (constellationSettings != null) { info.Providers = constellationSettings.Providers.ToArray(); info.Auditors = constellationSettings.Auditors .Select(a => new ConstellationInfo.Auditor { PubKey = a.PubKey.GetAccountId(), Address = a.Address }) .ToArray(); + info.Alpha = context.NodesManager.AlphaNode?.AccountId ?? ""; info.MinAccountBalance = constellationSettings.MinAccountBalance; info.MinAllowedLotSize = constellationSettings.MinAllowedLotSize; info.Assets = constellationSettings.Assets.ToArray(); @@ -49,7 +50,7 @@ public static ConstellationInfo GetInfo(this ExecutionContext context) return info; } - public static async Task HandleConstellationQuantum(this ExecutionContext context, ConstellationMessageEnvelope constellationInitEnvelope) + public static async Task HandleConstellationQuantum(this ExecutionContext context, ConstellationMessageEnvelope constellationInitEnvelope) { if (context == null) throw new ArgumentNullException(nameof(context)); @@ -61,6 +62,8 @@ public static async Task HandleConstellationQuantum(this ExecutionContext contex var quantumProcessingItem = context.QuantumHandler.HandleAsync(constellationQuantum, Task.FromResult(true)); await quantumProcessingItem.OnProcessed; + + return quantumProcessingItem.Apex; } } } \ No newline at end of file diff --git a/Centaurus.Domain/Helpers/RequestQuantumHelper.cs b/Centaurus.Domain/Helpers/RequestQuantumHelper.cs index ccdb59f8..c57a1f01 100644 --- a/Centaurus.Domain/Helpers/RequestQuantumHelper.cs +++ b/Centaurus.Domain/Helpers/RequestQuantumHelper.cs @@ -22,7 +22,9 @@ public static RequestQuantumBase GetQuantum(MessageEnvelopeBase envelope) return new WithdrawalRequestQuantum { RequestEnvelope = envelope }; default: if (envelope.Message is SequentialRequestMessage) - return new RequestQuantum { RequestEnvelope = envelope }; + return new ClientRequestQuantum { RequestEnvelope = envelope }; + else if (envelope.Message is ConstellationRequestMessage) + return new ConstellationQuantum { RequestEnvelope = envelope }; else throw new Exception($"{envelope.Message.GetMessageType()} is not request quantum message."); } diff --git a/Centaurus.Domain/Helpers/SyncCursorUpdateExtensions.cs b/Centaurus.Domain/Helpers/SyncCursorUpdateExtensions.cs index 6f89c23f..199cfef6 100644 --- a/Centaurus.Domain/Helpers/SyncCursorUpdateExtensions.cs +++ b/Centaurus.Domain/Helpers/SyncCursorUpdateExtensions.cs @@ -14,8 +14,10 @@ public static SyncCursorType ToDomainCursorType(this XdrSyncCursorType syncCurso { case XdrSyncCursorType.Quanta: return SyncCursorType.Quanta; - case XdrSyncCursorType.Signatures: - return SyncCursorType.Signatures; + case XdrSyncCursorType.MajoritySignatures: + return SyncCursorType.MajoritySignatures; + case XdrSyncCursorType.SingleNodeSignatures: + return SyncCursorType.SingleNodeSignatures; default: throw new ArgumentNullException($"{syncCursorType} cursor type is not supported."); } diff --git a/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchHandler.cs b/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchHandler.cs index eeb465a0..a964b4eb 100644 --- a/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchHandler.cs +++ b/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchHandler.cs @@ -17,7 +17,7 @@ public CatchupQuantaBatchHandler(ExecutionContext context) public override Task HandleMessage(IncomingNodeConnection connection, IncomingMessage message) { - _ = Context.Catchup.AddAuditorState(connection.PubKey, (CatchupQuantaBatch)message.Envelope.Message); + _ = Context.Catchup.AddNodeBatch(connection.PubKey, (CatchupQuantaBatch)message.Envelope.Message); return Task.CompletedTask; } } diff --git a/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchRequestHandler.cs b/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchRequestHandler.cs index a80e3008..2aacf621 100644 --- a/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchRequestHandler.cs +++ b/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchRequestHandler.cs @@ -33,7 +33,9 @@ private async Task SendQuanta(OutgoingConnection connection, CatchupQuantaBatchR if (currentBatch.Count < 1 && aboveApex < Context.QuantumHandler.CurrentApex) throw new Exception("No quanta from database."); - var signaturesBatch = Context.SyncStorage.GetSignatures(aboveApex, batchSize).ToDictionary(s => s.Apex, s => s.Signatures); + var majoritySignaturesBatch = Context.SyncStorage + .GetMajoritySignatures(aboveApex, batchSize) + .ToDictionary(s => s.Apex, s => s.Signatures); var batch = new CatchupQuantaBatch { @@ -49,19 +51,12 @@ private async Task SendQuanta(OutgoingConnection connection, CatchupQuantaBatchR //if quantum was persisted, than it contains majority signatures already. Otherwise we need to obtain it from the result manager if (apex > Context.PendingUpdatesManager.LastPersistedApex) Context.ResultManager.TryGetSignatures(apex, out quantumSignatures); - else - { - var alphaSignature = quantum.AlphaSignature; - if (signaturesBatch.TryGetValue(apex, out quantumSignatures)) - quantumSignatures.Insert(0, quantum.AlphaSignature); - else - quantumSignatures = new List { quantum.AlphaSignature }; - } - if (quantumSignatures.Count < 1) + else if (!majoritySignaturesBatch.TryGetValue(apex, out quantumSignatures)) { Context.NodesManager.CurrentNode.Failed(new Exception($"Unable to find signatures for quantum {apex}")); return; } + batch.Quanta.Add(new CatchupQuantaBatchItem { Quantum = quantum.Quantum, diff --git a/Centaurus.Domain/MessageHandlers/EffectsRequestMessageHandler.cs b/Centaurus.Domain/MessageHandlers/EffectsRequestMessageHandler.cs index 7318af1a..a6d8c954 100644 --- a/Centaurus.Domain/MessageHandlers/EffectsRequestMessageHandler.cs +++ b/Centaurus.Domain/MessageHandlers/EffectsRequestMessageHandler.cs @@ -17,7 +17,7 @@ public EffectsRequestMessageHandler(ExecutionContext context) public override string SupportedMessageType { get; } = typeof(QuantumInfoRequest).Name; - public override ConnectionState[] ValidConnectionStates => new ConnectionState[] { ConnectionState.Ready }; + public override bool IsAuthenticatedOnly => true; public override async Task HandleMessage(IncomingClientConnection connection, IncomingMessage message) { diff --git a/Centaurus.Domain/MessageHandlers/HandshakeRequestHandler.cs b/Centaurus.Domain/MessageHandlers/Handshake/HandshakeRequestHandler.cs similarity index 51% rename from Centaurus.Domain/MessageHandlers/HandshakeRequestHandler.cs rename to Centaurus.Domain/MessageHandlers/Handshake/HandshakeRequestHandler.cs index fdc9e811..85e7a69f 100644 --- a/Centaurus.Domain/MessageHandlers/HandshakeRequestHandler.cs +++ b/Centaurus.Domain/MessageHandlers/Handshake/HandshakeRequestHandler.cs @@ -16,28 +16,18 @@ public HandshakeRequestHandler(ExecutionContext context) public override string SupportedMessageType { get; } = typeof(HandshakeRequest).Name; - public override ConnectionState[] ValidConnectionStates { get; } = new ConnectionState[] { ConnectionState.Ready }; + public override bool IsAuthenticatedOnly => false; public override async Task HandleMessage(OutgoingConnection connection, IncomingMessage message) { var handshakeRequest = (HandshakeRequest)message.Envelope.Message; - - var quantaCursor = new SyncCursor { Type = XdrSyncCursorType.Quanta, Cursor = Context.QuantumHandler.LastAddedQuantumApex }; - - //if Prime than the node will receive results from auditors - var signaturesCursor = new SyncCursor - { - Type = XdrSyncCursorType.Signatures, - Cursor = Context.PendingUpdatesManager.LastPersistedApex, - DisableSync = Context.RoleManager.ParticipationLevel == CentaurusNodeParticipationLevel.Prime - }; await connection.SendMessage(new HandshakeResponse { HandshakeData = handshakeRequest.HandshakeData }); - //after sending auditor handshake the connection becomes ready - connection.ConnectionState = ConnectionState.Ready; + //we know that we connect to auditor so we can mark connection as authenticated + connection.HandshakeDataSend(); } } } diff --git a/Centaurus.Domain/MessageHandlers/HandshakeResponseHandlers/HandshakeResponseHandler.cs b/Centaurus.Domain/MessageHandlers/Handshake/HandshakeResponseHandler.cs similarity index 93% rename from Centaurus.Domain/MessageHandlers/HandshakeResponseHandlers/HandshakeResponseHandler.cs rename to Centaurus.Domain/MessageHandlers/Handshake/HandshakeResponseHandler.cs index a93f597b..5e2cf371 100644 --- a/Centaurus.Domain/MessageHandlers/HandshakeResponseHandlers/HandshakeResponseHandler.cs +++ b/Centaurus.Domain/MessageHandlers/Handshake/HandshakeResponseHandler.cs @@ -15,7 +15,7 @@ public HandshakeResponseHandler(ExecutionContext context) { } - public override ConnectionState[] ValidConnectionStates { get; } = new ConnectionState[] { ConnectionState.Connected }; + public override bool IsAuthenticatedOnly => false; public override string SupportedMessageType { get; } = typeof(HandshakeResponse).Name; @@ -42,7 +42,7 @@ private async Task HandleClientHandshake(IncomingClientConnection clientConnecti { //if Alpha is not ready, close connection if (clientConnection.Context.NodesManager.CurrentNode.State != State.Ready) - throw new ConnectionCloseException(WebSocketCloseStatus.ProtocolError, "Alpha is not in Ready state."); + throw new ConnectionCloseException(WebSocketCloseStatus.ProtocolError, "Server is not ready."); //if account not presented, throw UnauthorizedException if (clientConnection.Account == null) diff --git a/Centaurus.Domain/MessageHandlers/MessageHandlerBase.cs b/Centaurus.Domain/MessageHandlers/MessageHandlerBase.cs index 9c93ba3b..b5e8c013 100644 --- a/Centaurus.Domain/MessageHandlers/MessageHandlerBase.cs +++ b/Centaurus.Domain/MessageHandlers/MessageHandlerBase.cs @@ -20,9 +20,9 @@ protected MessageHandlerBase(ExecutionContext context) public abstract string SupportedMessageType { get; } /// - /// Only specified states are valid for the handler. Should be null or empty if any state is valid. + /// Only messages from the authenticated connections are allowed. /// - public virtual ConnectionState[] ValidConnectionStates { get; } + public virtual bool IsAuthenticatedOnly { get; } /// /// If set to true, than messages will be handled only if the other side is an auditor. @@ -36,14 +36,10 @@ protected MessageHandlerBase(ExecutionContext context) public virtual Task Validate(ConnectionBase connection, IncomingMessage message) { - if ((ValidConnectionStates?.Length ?? 0) > 0 - && Array.IndexOf(ValidConnectionStates, connection.ConnectionState) < 0) - throw new InvalidStateException( - SupportedMessageType.ToString(), - connection.ConnectionState.ToString(), - ValidConnectionStates.Select(s => s.ToString()).ToArray()); + if (IsAuthenticatedOnly && !connection.IsAuthenticated) + throw new UnauthorizedException(); - if (!message.Envelope.IsSignatureValid(connection.PubKey, message.MessageHash, connection.IsAuditor && connection.ConnectionState == ConnectionState.Ready)) + if (!message.Envelope.IsSignatureValid(connection.PubKey, message.MessageHash, connection.IsAuditor && connection.IsAuthenticated)) { throw new UnauthorizedException(); } @@ -63,7 +59,7 @@ private void ValidateAuditor(ConnectionBase connection) private void ValidateClient(ConnectionBase connection) { if (connection is IncomingClientConnection clientConnection //check requests count only for a client connection - && clientConnection.ConnectionState == ConnectionState.Ready //increment requests count only for validated connection + && clientConnection.IsAuthenticated //increment requests count only for validated connection && !clientConnection.Account.RequestCounter.IncRequestCount(DateTime.UtcNow.Ticks, out var error)) throw new TooManyRequestsException(error); } diff --git a/Centaurus.Domain/MessageHandlers/Quanta/QuantumHandlerBase.cs b/Centaurus.Domain/MessageHandlers/Quanta/QuantumHandlerBase.cs index 59910915..13f5ed4d 100644 --- a/Centaurus.Domain/MessageHandlers/Quanta/QuantumHandlerBase.cs +++ b/Centaurus.Domain/MessageHandlers/Quanta/QuantumHandlerBase.cs @@ -17,11 +17,11 @@ protected QuantumHandlerBase(ExecutionContext context) { } - public override ConnectionState[] ValidConnectionStates { get; } = new ConnectionState[] { ConnectionState.Ready }; + public override bool IsAuthenticatedOnly => true; public override Task HandleMessage(ConnectionBase connection, IncomingMessage message) { - if (Context.IsAlpha) + if (Context.NodesManager.IsAlpha) { Context.QuantumHandler.HandleAsync(RequestQuantumHelper.GetQuantum(message.Envelope), Task.FromResult(true)); } diff --git a/Centaurus.Domain/MessageHandlers/QuantumMajoritySignaturesBatchHandler.cs b/Centaurus.Domain/MessageHandlers/QuantumMajoritySignaturesBatchHandler.cs index f0a1d315..e208272d 100644 --- a/Centaurus.Domain/MessageHandlers/QuantumMajoritySignaturesBatchHandler.cs +++ b/Centaurus.Domain/MessageHandlers/QuantumMajoritySignaturesBatchHandler.cs @@ -18,14 +18,14 @@ public QuantumMajoritySignaturesBatchHandler(ExecutionContext context) public override bool IsAuditorOnly => true; - public override string SupportedMessageType { get; } = typeof(QuantumMajoritySignaturesBatch).Name; + public override string SupportedMessageType { get; } = typeof(MajoritySignaturesBatch).Name; - public override ConnectionState[] ValidConnectionStates => new ConnectionState[] { ConnectionState.Ready }; + public override bool IsAuthenticatedOnly => true; public override Task HandleMessage(OutgoingConnection connection, IncomingMessage message) { - var signaturesBatch = (QuantumMajoritySignaturesBatch)message.Envelope.Message; - foreach (var signatures in signaturesBatch.Signatures) + var signaturesBatch = (MajoritySignaturesBatch)message.Envelope.Message; + foreach (var signatures in signaturesBatch.Items) Context.ResultManager.Add(signatures); return Task.CompletedTask; } diff --git a/Centaurus.Domain/MessageHandlers/RequestQuantaBatchHandler.cs b/Centaurus.Domain/MessageHandlers/RequestQuantaBatchHandler.cs index 1c7c6ab6..7e30be21 100644 --- a/Centaurus.Domain/MessageHandlers/RequestQuantaBatchHandler.cs +++ b/Centaurus.Domain/MessageHandlers/RequestQuantaBatchHandler.cs @@ -6,7 +6,7 @@ namespace Centaurus.Domain { - internal class RequestQuantaBatchHandler : MessageHandlerBase + internal class RequestQuantaBatchHandler : MessageHandlerBase { public RequestQuantaBatchHandler(ExecutionContext context) : base(context) @@ -14,12 +14,14 @@ public RequestQuantaBatchHandler(ExecutionContext context) } + public override bool IsAuditorOnly => true; + public override string SupportedMessageType { get; } = typeof(RequestQuantaBatch).Name; - public override Task HandleMessage(IncomingNodeConnection connection, IncomingMessage message) + public override Task HandleMessage(ConnectionBase connection, IncomingMessage message) { var requests = (RequestQuantaBatch)message.Envelope.Message; - if (Context.IsAlpha) + if (Context.NodesManager.IsAlpha) { foreach (var request in requests.Requests) { diff --git a/Centaurus.Domain/MessageHandlers/ResultBatchHandler.cs b/Centaurus.Domain/MessageHandlers/SingleNodeSignaturesBatchHandler.cs similarity index 55% rename from Centaurus.Domain/MessageHandlers/ResultBatchHandler.cs rename to Centaurus.Domain/MessageHandlers/SingleNodeSignaturesBatchHandler.cs index f8254364..f3410679 100644 --- a/Centaurus.Domain/MessageHandlers/ResultBatchHandler.cs +++ b/Centaurus.Domain/MessageHandlers/SingleNodeSignaturesBatchHandler.cs @@ -4,24 +4,24 @@ namespace Centaurus.Domain { - internal class ResultBatchHandler : MessageHandlerBase + internal class SingleNodeSignaturesBatchHandler : MessageHandlerBase { - public ResultBatchHandler(ExecutionContext context) + public SingleNodeSignaturesBatchHandler(ExecutionContext context) : base(context) { } - public override string SupportedMessageType { get; } = typeof(AuditorSignaturesBatch).Name; + public override string SupportedMessageType { get; } = typeof(SingleNodeSignaturesBatch).Name; - public override ConnectionState[] ValidConnectionStates { get; } = new ConnectionState[] { ConnectionState.Ready }; + public override bool IsAuthenticatedOnly => true; public override bool IsAuditorOnly { get; } = true; public override Task HandleMessage(ConnectionBase connection, IncomingMessage message) { - var resultsBatch = (AuditorSignaturesBatch)message.Envelope.Message; - foreach (var result in resultsBatch.AuditorResultMessages) - Context.ResultManager.Add(new QuantumSignatures + var resultsBatch = (SingleNodeSignaturesBatch)message.Envelope.Message; + foreach (var result in resultsBatch.Items) + Context.ResultManager.Add(new MajoritySignaturesBatchItem { Apex = result.Apex, Signatures = new List { result.Signature } diff --git a/Centaurus.Domain/MessageHandlers/StateMessageHandler.cs b/Centaurus.Domain/MessageHandlers/StateMessageHandler.cs index 58b3f95e..4db468d6 100644 --- a/Centaurus.Domain/MessageHandlers/StateMessageHandler.cs +++ b/Centaurus.Domain/MessageHandlers/StateMessageHandler.cs @@ -15,7 +15,7 @@ public StateMessageHandler(ExecutionContext executionContext) public override bool IsAuditorOnly => true; - public override ConnectionState[] ValidConnectionStates => new[] { ConnectionState.Ready }; + public override bool IsAuthenticatedOnly => true; public override Task HandleMessage(ConnectionBase connection, IncomingMessage message) { diff --git a/Centaurus.Domain/MessageHandlers/SyncCursorResetHandler.cs b/Centaurus.Domain/MessageHandlers/SyncCursorResetHandler.cs index bf996264..878c0393 100644 --- a/Centaurus.Domain/MessageHandlers/SyncCursorResetHandler.cs +++ b/Centaurus.Domain/MessageHandlers/SyncCursorResetHandler.cs @@ -9,7 +9,7 @@ namespace Centaurus.Domain.Handlers.AlphaHandlers { - internal class SyncCursorResetHandler : MessageHandlerBase + internal class SyncCursorResetHandler : MessageHandlerBase { public SyncCursorResetHandler(ExecutionContext context) : base(context) @@ -20,22 +20,20 @@ public SyncCursorResetHandler(ExecutionContext context) public override bool IsAuditorOnly { get; } = true; - public override ConnectionState[] ValidConnectionStates { get; } = - new ConnectionState[] { - ConnectionState.Ready - }; + public override bool IsAuthenticatedOnly => true; - public override Task HandleMessage(IncomingNodeConnection connection, IncomingMessage message) + public override Task HandleMessage(ConnectionBase connection, IncomingMessage message) { var cursorResetRequest = (SyncCursorReset)message.Envelope.Message; + var auditorConnection = (INodeConnection)connection; foreach (var cursor in cursorResetRequest.Cursors) { var cursorType = cursor.Type.ToDomainCursorType(); if (cursor.DisableSync) - connection.Node.DisableSync(cursorType); + auditorConnection.Node.DisableSync(cursorType); else - connection.Node.SetCursor(cursorType, default, cursor.Cursor, true); + auditorConnection.Node.SetCursor(cursorType, default, cursor.Cursor, true); } return Task.CompletedTask; diff --git a/Centaurus.Domain/MessageHandlers/SyncQuantaBatchHandler.cs b/Centaurus.Domain/MessageHandlers/SyncQuantaBatchHandler.cs index 0f3a5579..f09f2809 100644 --- a/Centaurus.Domain/MessageHandlers/SyncQuantaBatchHandler.cs +++ b/Centaurus.Domain/MessageHandlers/SyncQuantaBatchHandler.cs @@ -21,7 +21,7 @@ public SyncQuantaBatchHandler(ExecutionContext context) public override string SupportedMessageType { get; } = typeof(SyncQuantaBatch).Name; - public override ConnectionState[] ValidConnectionStates => new ConnectionState[] { ConnectionState.Ready }; + public override bool IsAuthenticatedOnly => true; public override async Task HandleMessage(OutgoingConnection connection, IncomingMessage message) { @@ -30,21 +30,14 @@ public override async Task HandleMessage(OutgoingConnection connection, Incoming private async Task AddQuantaToQueue(OutgoingConnection connection, IncomingMessage message) { - var quantumHandler = Context.QuantumHandler; var quantaBatch = (SyncQuantaBatch)message.Envelope.Message; - - //get last known apex - var lastKnownApex = quantumHandler.LastAddedQuantumApex; - var quanta = quantaBatch.Quanta; + var lastKnownApex = Context.QuantumHandler.CurrentApex; foreach (var processedQuantum in quanta) { var quantum = (Quantum)processedQuantum.Quantum; - if (quantum.Apex <= lastKnownApex) - { - logger.Trace($"Received {quantum.Apex}, expected {lastKnownApex + 1}"); + if (quantum.Apex <= lastKnownApex) //delayed quantum continue; - } if (quantum.Apex != lastKnownApex + 1) { @@ -53,20 +46,15 @@ await connection.SendMessage(new SyncCursorReset Cursors = new List { new SyncCursor { Type = XdrSyncCursorType.Quanta, - Cursor = quantumHandler.LastAddedQuantumApex + Cursor = Context.QuantumHandler.CurrentApex } } }.CreateEnvelope()); - logger.Warn($"Batch has invalid quantum apexes (current: {quantumHandler.LastAddedQuantumApex}, received: {quantum.Apex}). New apex cursor request sent."); + logger.Warn($"Batch has invalid quantum apexes (current: {Context.QuantumHandler.CurrentApex}, received: {quantum.Apex}). New apex cursor request sent."); return; } Context.QuantumHandler.HandleAsync(quantum, QuantumSignatureValidator.Validate(quantum)); - Context.ResultManager.Add(new QuantumSignatures - { - Apex = quantum.Apex, - Signatures = new List { processedQuantum.AlphaSignature } - }); lastKnownApex = quantum.Apex; } } diff --git a/Centaurus.Domain/Nodes/Common/NodeApexes.cs b/Centaurus.Domain/Nodes/Common/NodeApexes.cs index c93fcb3e..d3b8fe75 100644 --- a/Centaurus.Domain/Nodes/Common/NodeApexes.cs +++ b/Centaurus.Domain/Nodes/Common/NodeApexes.cs @@ -1,4 +1,4 @@ -namespace Centaurus.Domain.StateManagers +namespace Centaurus.Domain.Nodes.Common { internal class NodeApexes : ValueAggregation { diff --git a/Centaurus.Domain/Nodes/Common/NodeBase.cs b/Centaurus.Domain/Nodes/Common/NodeBase.cs index 04ad8573..434b7e1a 100644 --- a/Centaurus.Domain/Nodes/Common/NodeBase.cs +++ b/Centaurus.Domain/Nodes/Common/NodeBase.cs @@ -1,9 +1,6 @@ -using Centaurus.Domain.Quanta.Sync; -using Centaurus.Domain.StateManagers; +using Centaurus.Domain.Nodes.Common; using Centaurus.Models; using System; -using System.Collections.Generic; -using System.Linq; namespace Centaurus.Domain { @@ -16,6 +13,19 @@ public NodeBase(ExecutionContext context, RawPubKey rawPubKey) AccountId = rawPubKey.GetAccountId(); } + public bool IsAlpha => Context.NodesManager.AlphaNode == this; + + public abstract bool IsPrimeNode { get; } + + public void UpdateSettings(NodeSettings nodeSettings) + { + Settings = nodeSettings; + } + + public NodeSettings Settings { get; protected set; } + + public byte Id => (byte)(Settings?.Id ?? 0); + public RawPubKey PubKey { get; } public string AccountId { get; } diff --git a/Centaurus.Domain/Nodes/Common/NodeExtensions.cs b/Centaurus.Domain/Nodes/Common/NodeExtensions.cs index 6aa7bc4e..4a50bab7 100644 --- a/Centaurus.Domain/Nodes/Common/NodeExtensions.cs +++ b/Centaurus.Domain/Nodes/Common/NodeExtensions.cs @@ -1,5 +1,4 @@ -using Centaurus.Domain.StateManagers; -using Centaurus.Models; +using Centaurus.Models; using System; namespace Centaurus.Domain @@ -38,12 +37,5 @@ public static bool IsReadyToHandleQuanta(this NodeBase node) return node.IsRunning() || node.IsWaitingForInit(); } - - public static ConnectionBase GetConnection(this RemoteNode node) - { - if (node == null) - throw new ArgumentNullException(nameof(node)); - return (ConnectionBase)node.IncomingConnection ?? node.OutgoingConnection; - } } } diff --git a/Centaurus.Domain/Nodes/Common/NodeQuantaQueueLengths.cs b/Centaurus.Domain/Nodes/Common/NodeQuantaQueueLengths.cs index ee9cb444..9ae6fe55 100644 --- a/Centaurus.Domain/Nodes/Common/NodeQuantaQueueLengths.cs +++ b/Centaurus.Domain/Nodes/Common/NodeQuantaQueueLengths.cs @@ -3,7 +3,7 @@ using System.Linq; using System.Text; -namespace Centaurus.Domain.StateManagers +namespace Centaurus.Domain.Nodes.Common { internal class NodeQuantaQueueLengths : ValueAggregation { diff --git a/Centaurus.Domain/Nodes/Common/NodeSettings.cs b/Centaurus.Domain/Nodes/Common/NodeSettings.cs new file mode 100644 index 00000000..d3b04994 --- /dev/null +++ b/Centaurus.Domain/Nodes/Common/NodeSettings.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Centaurus.Domain.Nodes.Common +{ + public class NodeSettings + { + public NodeSettings(int id, Uri address) + { + Id = id; + Address = address; + } + + public int Id { get; } + + public Uri Address { get; } + + public override bool Equals(object obj) + { + return obj is NodeSettings settings && + Id == settings.Id && + EqualityComparer.Default.Equals(Address, settings.Address); + } + + public override int GetHashCode() + { + return HashCode.Combine(Id, Address); + } + + public static bool operator ==(NodeSettings lhs, NodeSettings rhs) + { + if (lhs is null) + { + if (rhs is null) + { + return true; + } + return false; + } + return lhs.Equals(rhs); + } + + public static bool operator !=(NodeSettings lhs, NodeSettings rhs) => !(lhs == rhs); + } +} diff --git a/Centaurus.Domain/Nodes/Common/SyncCursorCollection.cs b/Centaurus.Domain/Nodes/Common/SyncCursorCollection.cs index f585f7a0..c9e449af 100644 --- a/Centaurus.Domain/Nodes/Common/SyncCursorCollection.cs +++ b/Centaurus.Domain/Nodes/Common/SyncCursorCollection.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Linq; -namespace Centaurus.Domain.StateManagers +namespace Centaurus.Domain.Nodes.Common { internal class SyncCursorCollection { diff --git a/Centaurus.Domain/Nodes/Common/ValueAggregation.cs b/Centaurus.Domain/Nodes/Common/ValueAggregation.cs index 80dad63b..bc144564 100644 --- a/Centaurus.Domain/Nodes/Common/ValueAggregation.cs +++ b/Centaurus.Domain/Nodes/Common/ValueAggregation.cs @@ -3,7 +3,7 @@ using System.Linq; using System.Text; -namespace Centaurus.Domain.StateManagers +namespace Centaurus.Domain.Nodes.Common { internal abstract class ValueAggregation { diff --git a/Centaurus.Domain/Nodes/CurrentNode/CurrentNode.cs b/Centaurus.Domain/Nodes/CurrentNode/CurrentNode.cs index b4d7064a..2775efd0 100644 --- a/Centaurus.Domain/Nodes/CurrentNode/CurrentNode.cs +++ b/Centaurus.Domain/Nodes/CurrentNode/CurrentNode.cs @@ -2,16 +2,18 @@ using NLog; using System; -namespace Centaurus.Domain.StateManagers +namespace Centaurus.Domain { internal class CurrentNode : NodeBase { - public CurrentNode(ExecutionContext context, RawPubKey rawPubKey, State initState) - :base(context, rawPubKey) + public CurrentNode(ExecutionContext context, State initState) + :base(context, context?.Settings.KeyPair) { State = initState; } + public override bool IsPrimeNode => Context.Settings.IsPrimeNode; + public event Action StateChanged; public void Init(State state) @@ -68,13 +70,13 @@ public void UpdateData(ulong currentApex, ulong lastPersistedApex, int quantaQue /// private void UpdateDelay() { - if (Context.IsAlpha) + if (Context.NodesManager.IsAlpha) return; lock (syncRoot) { var isDelayed = State.Chasing == State; var currentApex = Context.QuantumHandler.CurrentApex; - var syncApex = Context.NodesManager.SyncSource.LastApex; + var syncApex = Context.NodesManager.SyncSourceManager.Source?.LastApex; if (isDelayed) { if (syncApex <= currentApex || syncApex - currentApex < RunningDelayTreshold) diff --git a/Centaurus.Domain/Nodes/Managers/NodesManager.cs b/Centaurus.Domain/Nodes/Managers/NodesManager.cs index f001a0de..78276e9f 100644 --- a/Centaurus.Domain/Nodes/Managers/NodesManager.cs +++ b/Centaurus.Domain/Nodes/Managers/NodesManager.cs @@ -1,13 +1,17 @@ -using Centaurus.Domain.StateManagers; +using Centaurus.Domain.Nodes; +using Centaurus.Domain.Nodes.Common; +using Centaurus.Domain.RemoteNodes; using Centaurus.Models; using NLog; using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; -using System.Text; +using System.Threading.Tasks; namespace Centaurus.Domain { + //TODO: move SyncSource logic to separate class internal class NodesManager : ContextualBase { static Logger logger = LogManager.GetCurrentClassLogger(); @@ -17,87 +21,171 @@ public NodesManager(ExecutionContext context, State currentNodeInitState) nodes = new RemoteNodesCollection(); nodes.OnAdd += Nodes_OnAdd; nodes.OnRemove += Nodes_OnRemove; - CurrentNode = new CurrentNode(Context, Context.Settings.KeyPair, currentNodeInitState); + CurrentNode = new CurrentNode(Context, currentNodeInitState); + SyncSourceManager = new SyncSourceManager(Context); } - public RemoteNode SyncSource { get; private set; } - public CurrentNode CurrentNode { get; } + public SyncSourceManager SyncSourceManager { get; } + public bool IsMajorityReady { get; private set; } + public List AllNodes { get; private set; } + + public NodeBase AlphaNode { get; private set; } + + public Dictionary NodeIds { get; private set; } + + public Dictionary NodePubKeys { get; private set; } + + /// + /// Is current node is Alpha + /// + public bool IsAlpha => CurrentNode.IsAlpha; + + public bool TryGetNode(RawPubKey rawPubKey, out RemoteNode node) + { + return nodes.TryGetNode(rawPubKey, out node); + } + public List GetRemoteNodes() { return nodes.GetAllNodes(); } - //TODO: handle address switch - public void SetAuditors(List auditors) + public async Task SetAuditors(List auditors) + { + var invalidatedNodes = await UpdateNodes(auditors); + RemoveNodes(invalidatedNodes); + SetAllNodes(); + SetNodeIds(); + SetAlphaNode(); + CalcMajorityReadiness(); + } + + public bool IsNode(RawPubKey pubKey) { + return NodePubKeys.ContainsKey(pubKey); + } + + private void SetAlphaNode() + { + if (Context.ConstellationSettingsManager.Current == null) + return; + var alphaNode = AllNodes.FirstOrDefault(n => n.PubKey.Equals(Context.ConstellationSettingsManager.Current.Alpha)); + if (alphaNode == null) + throw new Exception("Alpha node is not found"); + AlphaNode = alphaNode; + } + + private void SetNodeIds() + { + var nodeIds = new Dictionary(); + var nodePubkeys = new Dictionary(); + foreach (var node in AllNodes) + { + if (node.Id == 0) + continue; + nodeIds.Add(node.Id, node.PubKey); + nodePubkeys.Add(node.PubKey, node.Id); + } + NodeIds = nodeIds; + NodePubKeys = nodePubkeys; + } + + private void RemoveNodes(List invalidatedNodes) + { + foreach (var node in invalidatedNodes) + nodes.RemoveNode(node); + } + + /// + /// + /// + /// + /// Invalidated nodes + private async Task> UpdateNodes(List auditors) + { + var currentNodes = GetRemoteNodes().ToDictionary(n => n.PubKey, n => n); var currentAuditorKey = (RawPubKey)Context.Settings.KeyPair; - var ueHttps = Context.Settings.UseSecureConnection; - var newNodes = new Dictionary>(); foreach (var auditor in auditors) { - var pubKey = auditor.PubKey; - if (pubKey.Equals(currentAuditorKey)) + if (auditor.PubKey.Equals(currentAuditorKey)) + { + UpdateCurrentNode(auditor.PubKey); continue; + } + await SetRemoteNode(auditor); + currentNodes.Remove(auditor.PubKey); + } + //all nodes that weren't removed are no more relevant + return currentNodes.Values.ToList(); + } - UriHelper.TryCreateWsConnection(auditor.Address, ueHttps, out var uri); - newNodes.Add(pubKey, new Lazy(() => new RemoteNode(Context, pubKey, uri))); + private async Task SetRemoteNode(Auditor auditor) + { + var pubKey = auditor.PubKey; + UriHelper.TryCreateWsConnection(auditor.Address, Context.Settings.UseSecureConnection, out var uri); + var settings = new NodeSettings(GetNodeId(pubKey), uri); + if (nodes.TryGetNode(pubKey, out var node)) + { + if (node.Settings == settings) + return; + var shouldReconnect = settings.Address != node.Address; + node.UpdateSettings(settings); + if (shouldReconnect) + { + await node.CloseOutConnection(); + node.ConnectTo(); + } + return; } - nodes.SetNodes(newNodes); + node = new RemoteNode(Context, pubKey); + node.UpdateSettings(settings); + nodes.AddNode(node); } - public void EnqueueResult(AuditorResult result) + private void UpdateCurrentNode(RawPubKey pubKey) { - foreach (var node in GetRemoteNodes()) - node.EnqueueResult(result); + var nodeId = GetNodeId(pubKey); + if (nodeId != CurrentNode.Id) + CurrentNode.UpdateSettings(new NodeSettings(nodeId, null)); } - public void EnqueueMessage(MessageEnvelopeBase message) + private int GetNodeId(RawPubKey pubKey) { - foreach (var node in GetRemoteNodes()) - { - node.EnqueueMessage(message); - } + if (Context.ConstellationSettingsManager.Current == null) + return 0; + return Context.ConstellationSettingsManager.Current.GetNodeId(pubKey); } - public bool TryGetNode(RawPubKey rawPubKey, out RemoteNode node) + private void SetAllNodes() { - return nodes.TryGetNode(rawPubKey, out node); + var allNodes = GetRemoteNodes().Cast().ToList(); + allNodes.Insert(0, CurrentNode); + AllNodes = allNodes; } private void Nodes_OnAdd(RemoteNode node) { node.OnStateUpdated += Node_OnStateUpdated; node.OnLastApexUpdated += Node_OnLastApexUpdated; - node.Connect(); + node.ConnectTo(); CalcMajorityReadiness(); } - private void Nodes_OnRemove(RemoteNode node) + private async void Nodes_OnRemove(RemoteNode node) { node.OnStateUpdated -= Node_OnStateUpdated; node.OnLastApexUpdated -= Node_OnLastApexUpdated; - node.CloseOugoingConnection(); + await node.CloseConnections(); CalcMajorityReadiness(); } private void Node_OnLastApexUpdated(RemoteNode node) { - //update current sync node - if (!Context.IsAlpha && node.LastApex > 0) - { - var syncSourceLastApex = SyncSource.LastApex; - if (SyncSource == null || - syncSourceLastApex < node.LastApex //if the current node ahead - && node.LastApex - syncSourceLastApex > 1000 //and the apexes difference greater than 1000 - && (DateTime.UtcNow - SyncSourceUpdateDate) > TimeSpan.FromSeconds(1)) //and the last sync source update was later than second ago - { - SetSyncSource(node); - } - } + } private void Node_OnStateUpdated(RemoteNode node) @@ -105,53 +193,13 @@ private void Node_OnStateUpdated(RemoteNode node) CalcMajorityReadiness(); } - private DateTime SyncSourceUpdateDate; - private object syncSourceSyncRoot = new { }; - private readonly RemoteNodesCollection nodes; - private void SetSyncSource(RemoteNode auditorState) - { - lock (syncSourceSyncRoot) - { - ClearCurrentSyncCursor(); - - SyncSource = auditorState; - SyncSourceUpdateDate = DateTime.UtcNow; - - SetCurrentSyncCursor(); - } - } - - private void ClearCurrentSyncCursor() - { - if (SyncSource != null) - Context.Notify(SyncSource.PubKey, new SyncCursorReset - { - Cursors = new List { - new SyncCursor { Type = XdrSyncCursorType.Quanta, DisableSync = true }, - new SyncCursor { Type = XdrSyncCursorType.Signatures, DisableSync = true }, - } - }.CreateEnvelope()); - } - - private void SetCurrentSyncCursor() - { - if (SyncSource != null) - Context.Notify(SyncSource.PubKey, new SyncCursorReset - { - Cursors = new List { - new SyncCursor { Type = XdrSyncCursorType.Quanta, Cursor = Context.QuantumHandler.LastAddedQuantumApex }, - new SyncCursor { Type = XdrSyncCursorType.Signatures, Cursor = Context.PendingUpdatesManager.LastPersistedApex }, - } - }.CreateEnvelope()); - } - private void CalcMajorityReadiness() { var majorityReadiness = false; //if Prime node than it must be connected with other nodes - if (Context.RoleManager.ParticipationLevel == CentaurusNodeParticipationLevel.Prime) + if (Context.NodesManager.CurrentNode.IsPrimeNode) { majorityReadiness = GetReadinessForPrimeNode(); } @@ -174,13 +222,13 @@ private void TrySetReadiness(bool majorityReadiness) private bool GetReadinessForPrimeNode() { //if current server is Alpha, than ignore Alpha validation - var isAlphaReady = Context.IsAlpha; + var isAlphaReady = Context.NodesManager.IsAlpha; var connectedCount = 0; foreach (var node in nodes.GetAllNodes()) { if (!(node.State == State.Ready || node.State == State.Running)) continue; - if (Context.Constellation.Alpha.Equals(node.PubKey)) + if (node == AlphaNode) isAlphaReady = true; connectedCount++; } @@ -196,7 +244,7 @@ private bool GetReadinessForAuditorNode() { foreach (var node in nodes.GetAllNodes()) { - if (!node.PubKey.Equals(Context.Constellation.Alpha)) + if (node != AlphaNode) continue; //if auditor doesn't have connections with another auditors, we only need to verify alpha's state return node.State == State.Ready || node.State == State.Running; diff --git a/Centaurus.Domain/Nodes/Managers/StateNotifierWorker.cs b/Centaurus.Domain/Nodes/Managers/StateNotifierWorker.cs index 97be30c2..b556a190 100644 --- a/Centaurus.Domain/Nodes/Managers/StateNotifierWorker.cs +++ b/Centaurus.Domain/Nodes/Managers/StateNotifierWorker.cs @@ -36,13 +36,14 @@ private void BroadcastTimer_Elapsed(object sender, ElapsedEventArgs e) var lastPersistedApex = Context.PendingUpdatesManager.LastPersistedApex; var quantaQueueLenght = Context.QuantumHandler.QuantaQueueLenght; var updateDate = DateTime.UtcNow; + var updateDateInTicks = updateDate.Ticks; var updateMessage = new StateMessage { State = currentNode.State, CurrentApex = currentApex, LastPersistedApex = lastPersistedApex, QuantaQueueLength = quantaQueueLenght, - UpdateDate = updateDate + UpdateDate = updateDateInTicks }; currentNode.UpdateData(currentApex, lastPersistedApex, quantaQueueLenght, updateDate); Context.NotifyAuditors(updateMessage.CreateEnvelope()); diff --git a/Centaurus.Domain/Nodes/Managers/SyncSourceManager.cs b/Centaurus.Domain/Nodes/Managers/SyncSourceManager.cs new file mode 100644 index 00000000..c0795152 --- /dev/null +++ b/Centaurus.Domain/Nodes/Managers/SyncSourceManager.cs @@ -0,0 +1,142 @@ +using Centaurus.Models; +using NLog; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Timers; + +namespace Centaurus.Domain.Nodes +{ + internal class SyncSourceManager : ContextualBase + { + static Logger logger = LogManager.GetCurrentClassLogger(); + public SyncSourceManager(ExecutionContext context) + : base(context) + { + InitTimer(); + } + + Timer syncSourceSwitchTimer = new Timer(); + private void InitTimer() + { + syncSourceSwitchTimer.AutoReset = false; + syncSourceSwitchTimer.Interval = TimeSpan.FromSeconds(2).TotalMilliseconds; + syncSourceSwitchTimer.Elapsed += SyncSourceSwitchTimer_Elapsed; + syncSourceSwitchTimer.Start(); + } + + private void SyncSourceSwitchTimer_Elapsed(object sender, ElapsedEventArgs e) + { + try + { + ChooseNewSyncNode(); + } + catch (Exception exc) + { + logger.Error(exc); + } + finally + { + syncSourceSwitchTimer.Start(); + } + } + + public RemoteNode Source { get; private set; } + + private object syncRoot = new { }; + + private void ChooseNewSyncNode() + { + lock (syncRoot) + { + if (Context.NodesManager.IsAlpha) + { + ClearCurrentSyncCursor(); + return; + } + + //first try to connect to prime nodes + var candidateNode = Context.NodesManager.GetRemoteNodes() + .Where(n => n.IsConnected) //only connected + .OrderByDescending(n => n.IsAlpha) //first of all try to set alpha + .ThenByDescending(n => n.IsPrimeNode) //then try to connect to prime node + .ThenByDescending(n => n.LastApex) + .FirstOrDefault(); + + if (candidateNode == null) + return; + SetSyncSource(candidateNode); + } + } + + private bool ShouldUpdateSyncSource(RemoteNode node) + { + if (Source == null) + return true; + + return node.LastApex > Source.LastApex //if the candidate node is ahead + && node.LastApex - Source.LastApex > 1000; //and the apexes difference greater than 1000 + } + + private void SetSyncSource(RemoteNode node) + { + ClearCurrentSyncCursor(); + SetCurrentSyncCursor(node); + } + + private void ClearCurrentSyncCursor() + { + if (Source != null) + { + var connection = Source.GetConnection(); + Source = null; + if (connection == null) + return; + + _ = connection.SendMessage(new SyncCursorReset + { + Cursors = new List { + new SyncCursor { Type = XdrSyncCursorType.Quanta, DisableSync = true }, + new SyncCursor { Type = XdrSyncCursorType.MajoritySignatures, DisableSync = true }, + } + }.CreateEnvelope()); + } + } + + private void SetCurrentSyncCursor(RemoteNode node) + { + Source = node; + var connection = Source.GetConnection(); + if (connection == null) + throw new Exception("Unable to find sync source connection"); + + var cursors = new List + { + new SyncCursor { + Type = XdrSyncCursorType.Quanta, + Cursor = Context.QuantumHandler.CurrentApex + } + }; + + if (IsMajoritySignaturesCursorRequired()) + cursors.Add(new SyncCursor + { + Type = XdrSyncCursorType.MajoritySignatures, + Cursor = Context.PendingUpdatesManager.LastPersistedApex + }); + + _ = connection.SendMessage(new SyncCursorReset + { + Cursors = cursors + }.CreateEnvelope()); + } + + private bool IsMajoritySignaturesCursorRequired() + { + //if the current node is not a prime node, we need to wait for a majority signatures + //or if the constellation is not ready + return !(Context.NodesManager.CurrentNode.IsPrimeNode && Context.NodesManager.IsMajorityReady); + } + } +} \ No newline at end of file diff --git a/Centaurus.Domain/Nodes/RemoteNode/RemoteNode.cs b/Centaurus.Domain/Nodes/RemoteNode/RemoteNode.cs deleted file mode 100644 index cc6867ed..00000000 --- a/Centaurus.Domain/Nodes/RemoteNode/RemoteNode.cs +++ /dev/null @@ -1,142 +0,0 @@ -using Centaurus.Domain.Quanta.Sync; -using Centaurus.Models; -using System; -using System.Collections.Generic; - -namespace Centaurus.Domain.StateManagers -{ - internal partial class RemoteNode : NodeBase - { - public RemoteNode(ExecutionContext context, RawPubKey rawPubKey, Uri address) - : base(context, rawPubKey) - { - Address = address; - } - - private Uri address; - public Uri Address - { - get => address; - private set - { - if (value == address) - return; - - address = value; - - if (atomicConnection != null) - { - atomicConnection.Shutdown(); - atomicConnection = null; - } - if (value != null) - atomicConnection = new RemoteNodeConnection(this); - } - } - - private RemoteNodeConnection atomicConnection; - - public event Action OnStateUpdated; - - public event Action OnLastApexUpdated; - - public void RegisterIncomingConnection(IncomingNodeConnection connection) - { - if (connection == null) - throw new ArgumentNullException(nameof(connection)); - - IncomingConnection = connection; - } - - public void RemoveIncomingConnection() - { - IncomingConnection = null; - if (this.GetConnection() == null) - { - State = State.Undefined; - cursors.Clear(); - } - } - - public void Connect() - { - atomicConnection?.Run(); - } - - public void CloseOugoingConnection() - { - if (atomicConnection != null) - { - atomicConnection.Shutdown(); - atomicConnection = null; - } - } - - public void EnqueueResult(AuditorResult result) - { - atomicConnection?.OutgoingResultsStorage.EnqueueResult(result); - } - - public void EnqueueMessage(MessageEnvelopeBase message) - { - atomicConnection?.OutgoingMessageStorage.EnqueueMessage(message); - } - - public IncomingNodeConnection IncomingConnection { get; private set; } - - public OutgoingConnection OutgoingConnection => atomicConnection?.Connection; - - public List GetActiveCursors() - { - return cursors.GetActiveCursors(); - } - - public void SetCursor(SyncCursorType cursorType, DateTime timeToken, ulong currentCursor, bool force = false) - { - var cursor = cursors.Get(cursorType, true); - cursor.SetCursor(currentCursor, timeToken, force); - } - - public void DisableSync(SyncCursorType cursorType) - { - var cursor = cursors.Get(cursorType); - cursor?.DisableSync(); - } - - public void Update(StateMessage stateMessage) - { - lock (syncRoot) - { - //skip old data - if (UpdateDate > stateMessage.UpdateDate) - return; - SetState(stateMessage); - SetApex(stateMessage.UpdateDate, stateMessage.CurrentApex); - SetQuantaQueueLength(stateMessage.UpdateDate, stateMessage.QuantaQueueLength); - LastPersistedApex = stateMessage.LastPersistedApex; - UpdateDate = stateMessage.UpdateDate; - } - } - - protected override void SetApex(DateTime updateDate, ulong apex) - { - var lastApex = LastApex; - base.SetApex(updateDate, apex); - if (lastApex != LastApex) - OnLastApexUpdated?.Invoke(this); - } - - private void SetState(StateMessage stateMessage) - { - if (State != stateMessage.State) - { - State = stateMessage.State; - OnStateUpdated?.Invoke(this); - } - } - - private object syncRoot = new { }; - - private SyncCursorCollection cursors = new SyncCursorCollection(); - } -} \ No newline at end of file diff --git a/Centaurus.Domain/Nodes/RemoteNode/RemoteNodesCollection.cs b/Centaurus.Domain/Nodes/RemoteNode/RemoteNodesCollection.cs deleted file mode 100644 index 78431ded..00000000 --- a/Centaurus.Domain/Nodes/RemoteNode/RemoteNodesCollection.cs +++ /dev/null @@ -1,75 +0,0 @@ -using Centaurus.Models; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -namespace Centaurus.Domain.StateManagers -{ - internal class RemoteNodesCollection - { - private object syncRoot = new { }; - private Dictionary nodes = new Dictionary(); - - public void SetNodes(Dictionary> nodeItems) - { - lock (syncRoot) - { - //copy current auditors - var oldNodes = nodes.ToDictionary(a => a.Key, a => a.Value); - nodes.Clear(); - foreach (var nodeItem in nodeItems) - { - var pubKey = nodeItem.Key; - var node = nodeItem.Value; - //if the auditor node already presented, re-add it. Otherwise create and add the new instance. - if (oldNodes.Remove(pubKey, out var auditorState)) - { - nodes.Add(pubKey, auditorState); - continue; - } - nodes.Add(pubKey, node.Value); - OnAdd?.Invoke(node.Value); - } - //all nodes that are not presented in new auditors list, should be removed - NodesRemoved(oldNodes.Values.ToList()); - } - } - - public event Action OnAdd; - - public event Action OnRemove; - - public List GetAllNodes() - { - lock (syncRoot) - { - return nodes.Values.ToList(); - } - } - - public bool TryGetNode(RawPubKey pubKey, out RemoteNode node) - { - lock (syncRoot) - return nodes.TryGetValue(pubKey, out node); - } - - internal void Clear() - { - lock (syncRoot) - { - var removedNodes = nodes.Values.ToList(); - nodes.Clear(); - NodesRemoved(removedNodes); - } - } - - private void NodesRemoved(List removedNodes) - { - foreach (var removedNode in removedNodes) - { - OnRemove?.Invoke(removedNode); - } - } - } -} \ No newline at end of file diff --git a/Centaurus.Domain/Nodes/RemoteNodes/RemoteNode.cs b/Centaurus.Domain/Nodes/RemoteNodes/RemoteNode.cs new file mode 100644 index 00000000..35f57d9e --- /dev/null +++ b/Centaurus.Domain/Nodes/RemoteNodes/RemoteNode.cs @@ -0,0 +1,141 @@ +using Centaurus.Domain.Nodes.Common; +using Centaurus.Domain.Quanta.Sync; +using Centaurus.Domain.RemoteNodes; +using Centaurus.Models; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading.Tasks; + +namespace Centaurus.Domain +{ + internal partial class RemoteNode : NodeBase + { + public RemoteNode(ExecutionContext context, RawPubKey rawPubKey) + : base(context, rawPubKey) + { + connectionManager = new RemoteNodeConnectionManager(this); + connectionManager.OnConnected += ConnectionManager_OnConnected; + connectionManager.OnConnectionClosed += ConnectionManager_OnConnectionClosed; + } + + private void ConnectionManager_OnConnectionClosed() + { + State = State.Undefined; + cursors.Clear(); + } + + private void ConnectionManager_OnConnected() + { + //subscribe for the current remote node signatures + if (Context.NodesManager.CurrentNode.IsPrimeNode) + { + var connection = connectionManager.GetConnection(); + _ = connection.SendMessage(new SyncCursorReset + { + Cursors = new List { + new SyncCursor { + Type = XdrSyncCursorType.SingleNodeSignatures, + Cursor = Context.PendingUpdatesManager.LastPersistedApex } + } + }); + } + } + + //TODO: find a way to verify that node is Prime + public override bool IsPrimeNode => Address != null; + + public Uri Address => Settings.Address; + + public bool IsConnected => connectionManager.IsConnected; + + public event Action OnStateUpdated; + + public event Action OnLastApexUpdated; + + public List GetActiveCursors() + { + return cursors.GetActiveCursors(); + } + + public void SetCursor(SyncCursorType cursorType, DateTime timeToken, ulong currentCursor, bool force = false) + { + var cursor = cursors.Get(cursorType, true); + cursor.SetCursor(currentCursor, timeToken, force); + } + + public void DisableSync(SyncCursorType cursorType) + { + var cursor = cursors.Get(cursorType); + cursor?.DisableSync(); + } + + public void Update(StateMessage stateMessage) + { + lock (syncRoot) + { + //skip old data + var updateDate = new DateTime(stateMessage.UpdateDate); + if (UpdateDate > updateDate) + return; + SetState(stateMessage); + SetApex(updateDate, stateMessage.CurrentApex); + SetQuantaQueueLength(updateDate, stateMessage.QuantaQueueLength); + LastPersistedApex = stateMessage.LastPersistedApex; + UpdateDate = updateDate; + } + } + + public ConnectionBase GetConnection() + { + return connectionManager.GetConnection(); + } + + public void RegisterIncomingConnection(IncomingNodeConnection auditorConnection) + { + connectionManager.RegisterIncomingConnection(auditorConnection); + } + + public void RemoveIncomingConnection() + { + connectionManager.RemoveIncomingConnection(); + } + + public async Task CloseOutConnection() + { + await connectionManager.CloseOutConnection(); + } + + public void ConnectTo() + { + connectionManager.ConnectTo(); + } + + public async Task CloseConnections() + { + await connectionManager.CloseConnections(); + } + + private void SetState(StateMessage stateMessage) + { + if (State != stateMessage.State) + { + State = stateMessage.State; + OnStateUpdated?.Invoke(this); + } + } + + protected override void SetApex(DateTime updateDate, ulong apex) + { + var lastApex = LastApex; + base.SetApex(updateDate, apex); + if (lastApex != LastApex) + OnLastApexUpdated?.Invoke(this); + } + + private object syncRoot = new { }; + + private SyncCursorCollection cursors = new SyncCursorCollection(); + private RemoteNodeConnectionManager connectionManager; + } +} \ No newline at end of file diff --git a/Centaurus.Domain/Nodes/RemoteNode/RemoteNodeConnection.cs b/Centaurus.Domain/Nodes/RemoteNodes/RemoteNodeConnection.cs similarity index 75% rename from Centaurus.Domain/Nodes/RemoteNode/RemoteNodeConnection.cs rename to Centaurus.Domain/Nodes/RemoteNodes/RemoteNodeConnection.cs index dc72e327..1536f6fc 100644 --- a/Centaurus.Domain/Nodes/RemoteNode/RemoteNodeConnection.cs +++ b/Centaurus.Domain/Nodes/RemoteNodes/RemoteNodeConnection.cs @@ -7,7 +7,7 @@ using System.Threading.Tasks; using System.Web; -namespace Centaurus.Domain.StateManagers +namespace Centaurus.Domain.RemoteNodes { internal class RemoteNodeConnection : ContextualBase { @@ -16,16 +16,10 @@ public RemoteNodeConnection(RemoteNode node) { Node = node ?? throw new ArgumentNullException(nameof(node)); - OutgoingMessageStorage = new OutgoingMessageStorage(); - OutgoingResultsStorage = new OutgoingResultsStorage(OutgoingMessageStorage); } public RemoteNode Node { get; } - public OutgoingMessageStorage OutgoingMessageStorage { get; } - - public OutgoingResultsStorage OutgoingResultsStorage { get; } - public void Run() { try @@ -39,27 +33,32 @@ public void Run() } } - public void Shutdown() + public async Task Shutdown() { + var closeTask = Task.CompletedTask; lock (syncRoot) { isAborted = true; - Connection?.CloseConnection(); - OutgoingResultsStorage.Dispose(); + if (Connection != null) + closeTask = Connection.CloseConnection(); } + await closeTask; } public RawPubKey PubKey => Node.PubKey; public Uri Address => Node.Address; + public event Action OnConnected; + + public event Action OnConnectionClosed; + private void ConnectToAuditor() { Task.Factory.StartNew(async () => { Uri connectionUri = GetConnectionUri(); var connectionAttempts = 0; - var listenTask = default(Task); while (!isAborted) { //TODO: remove this condition after refactoring result message broadcasting @@ -71,8 +70,10 @@ private void ConnectToAuditor() } lock (syncRoot) { - if (!isAborted) - Connection = new OutgoingConnection(Context, Node, OutgoingMessageStorage, Context.OutgoingConnectionFactory.GetConnection()); + if (isAborted) + return; + Connection = new OutgoingConnection(Context, Node, Context.OutgoingConnectionFactory.GetConnection()); + Connection.OnAuthenticated += Connection_OnAuthenticated; } try { @@ -89,16 +90,23 @@ private void ConnectToAuditor() } finally { + Connection.OnAuthenticated -= Connection_OnAuthenticated; Connection.Dispose(); Connection = null; + OnConnectionClosed?.Invoke(); } } }); } + private void Connection_OnAuthenticated(ConnectionBase obj) + { + OnConnected?.Invoke(); + } + private Uri GetConnectionUri() { - var uriBuilder = new UriBuilder(Address); + var uriBuilder = new UriBuilder(new Uri(Address, WebSocketConstants.CentaurusWebSocketEndPoint)); var query = HttpUtility.ParseQueryString(uriBuilder.Query); query[WebSocketConstants.PubkeyParamName] = Context.Settings.KeyPair.AccountId; uriBuilder.Query = query.ToString(); diff --git a/Centaurus.Domain/Nodes/RemoteNodes/RemoteNodeConnectionManager.cs b/Centaurus.Domain/Nodes/RemoteNodes/RemoteNodeConnectionManager.cs new file mode 100644 index 00000000..6be97c14 --- /dev/null +++ b/Centaurus.Domain/Nodes/RemoteNodes/RemoteNodeConnectionManager.cs @@ -0,0 +1,147 @@ +using System; +using System.Threading.Tasks; + +namespace Centaurus.Domain.RemoteNodes +{ + internal class RemoteNodeConnectionManager + { + public RemoteNodeConnectionManager(RemoteNode remoteNode) + { + node = remoteNode; + } + + public event Action OnConnectionClosed; + public event Action OnConnected; + + public void ConnectTo() + { + if (atomicConnection != null) + { + throw new InvalidOperationException("Already connected."); + } + if (node.Address != null) + { + atomicConnection = new RemoteNodeConnection(node); + atomicConnection.OnConnected += AtomicConnection_OnConnected; + atomicConnection.OnConnectionClosed += AtomicConnection_OnConnectionClosed; + atomicConnection.Run(); + } + } + + public ConnectionBase GetConnection() + { + if (IncomingConnection?.IsAuthenticated ?? false) + return IncomingConnection; + return OutgoingConnection?.IsAuthenticated ?? false ? OutgoingConnection : null; + } + + private void AtomicConnection_OnConnected() + { + OutConnected(); + } + + private void AtomicConnection_OnConnectionClosed() + { + OutConnectionClosed(); + } + + public async Task CloseInConnection() + { + if (IncomingConnection != null) + await IncomingConnection.CloseConnection(); + } + + public async Task CloseConnections() + { + await Task.WhenAll( + CloseOutConnection(), + CloseInConnection() + ); + } + + public async Task CloseOutConnection() + { + if (atomicConnection != null) + { + await atomicConnection.Shutdown(); + atomicConnection.OnConnected -= AtomicConnection_OnConnected; + atomicConnection.OnConnectionClosed -= AtomicConnection_OnConnectionClosed; + atomicConnection = null; + } + } + + private void OutConnected() + { + lock (syncRoot) + { + IsOutgoningConnectionEstablished = true; + if (!IsConnected) + IsConnected = true; + } + } + + private void OutConnectionClosed() + { + lock (syncRoot) + { + if (!IsOutgoningConnectionEstablished) + return; + IsOutgoningConnectionEstablished = false; + if (IsConnected && !IsIncomingConnectionEstablished) + IsConnected = false; + } + } + + public void RegisterIncomingConnection(IncomingNodeConnection connection) + { + if (connection == null) + throw new ArgumentNullException(nameof(connection)); + + lock (syncRoot) + { + IncomingConnection = connection; + if (!IsConnected) + IsConnected = true; + } + } + + public void RemoveIncomingConnection() + { + lock (syncRoot) + { + IncomingConnection = null; + if (IsConnected && !IsOutgoningConnectionEstablished) + IsConnected = false; + } + } + + public IncomingNodeConnection IncomingConnection { get; private set; } + + public OutgoingConnection OutgoingConnection => atomicConnection?.Connection; + + private bool isConnected; + public bool IsConnected + { + get => isConnected; + private set + { + if (isConnected == value) + return; + + isConnected = value; + if (isConnected) + OnConnected?.Invoke(); + else + OnConnectionClosed?.Invoke(); + } + } + + public bool IsOutgoningConnectionEstablished { get; private set; } + + public bool IsIncomingConnectionEstablished { get; private set; } + + private RemoteNodeConnection atomicConnection; + private RemoteNode node; + private object syncRoot = new { }; + } +} diff --git a/Centaurus.Domain/Nodes/RemoteNode/RemoteNodeCursor.cs b/Centaurus.Domain/Nodes/RemoteNodes/RemoteNodeCursor.cs similarity index 100% rename from Centaurus.Domain/Nodes/RemoteNode/RemoteNodeCursor.cs rename to Centaurus.Domain/Nodes/RemoteNodes/RemoteNodeCursor.cs diff --git a/Centaurus.Domain/Nodes/RemoteNodes/RemoteNodesCollection.cs b/Centaurus.Domain/Nodes/RemoteNodes/RemoteNodesCollection.cs new file mode 100644 index 00000000..f26add7f --- /dev/null +++ b/Centaurus.Domain/Nodes/RemoteNodes/RemoteNodesCollection.cs @@ -0,0 +1,63 @@ +using Centaurus.Models; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Centaurus.Domain.RemoteNodes +{ + internal class RemoteNodesCollection + { + private object syncRoot = new { }; + private Dictionary nodes = new Dictionary(); + + public void AddNode(RemoteNode remoteNode) + { + lock (syncRoot) + { + nodes.Add(remoteNode.PubKey, remoteNode); + OnAdd?.Invoke(remoteNode); + } + } + + public void RemoveNode(RemoteNode remoteNode) + { + lock (syncRoot) + { + if (nodes.Remove(remoteNode.PubKey)) + OnRemove?.Invoke(remoteNode); + } + } + + public event Action OnAdd; + + public event Action OnRemove; + + public List GetAllNodes() + { + lock (syncRoot) + { + return nodes.Values.ToList(); + } + } + + public bool TryGetNode(RawPubKey pubKey, out RemoteNode node) + { + lock (syncRoot) + return nodes.TryGetValue(pubKey, out node); + } + + internal void Clear() + { + lock (syncRoot) + { + var removedNodes = nodes.Values.ToList(); + nodes.Clear(); + foreach (var node in removedNodes) + { + OnRemove?.Invoke(node); + } + } + } + } +} \ No newline at end of file diff --git a/Centaurus.Domain/Quanta/Handlers/QuantumHandler.cs b/Centaurus.Domain/Quanta/Handlers/QuantumHandler.cs index 768b8f3f..b4594b0e 100644 --- a/Centaurus.Domain/Quanta/Handlers/QuantumHandler.cs +++ b/Centaurus.Domain/Quanta/Handlers/QuantumHandler.cs @@ -16,7 +16,7 @@ public class QuantumHandler : ContextualBase public QuantumHandler(ExecutionContext context, ulong lastApex, byte[] lastQuantumHash) : base(context) { - CurrentApex = LastAddedQuantumApex = lastApex; + CurrentApex = lastApex; LastQuantumHash = lastQuantumHash ?? throw new ArgumentNullException(nameof(lastQuantumHash)); Task.Factory.StartNew(RunQuantumWorker).Unwrap(); } @@ -32,47 +32,49 @@ public QuantumHandler(ExecutionContext context, ulong lastApex, byte[] lastQuant /// Quantum to handles public QuantumProcessingItem HandleAsync(Quantum quantum, Task signatureValidation) { - if (Context.IsAlpha //if current node is not alpha, than we need to keep process quanta + if (Context.NodesManager.IsAlpha //if current node is not alpha, than we need to keep process quanta && QuantaThrottlingManager.Current.IsThrottlingEnabled && QuantaThrottlingManager.Current.MaxItemsPerSecond <= awaitedQuanta.Count) throw new TooManyRequestsException("Server is too busy. Try again later."); var processingItem = new QuantumProcessingItem(quantum, signatureValidation); - if (!Context.IsAlpha) - LastAddedQuantumApex = processingItem.Quantum.Apex; lock (awaitedQuantaSyncRoot) awaitedQuanta.Enqueue(processingItem); return processingItem; } - public ulong LastAddedQuantumApex { get; private set; } - public ulong CurrentApex { get; private set; } public byte[] LastQuantumHash { get; private set; } public int QuantaQueueLenght => awaitedQuanta.Count; + private bool TryGetHandlingItem(out QuantumProcessingItem handlingItem) + { + lock (awaitedQuantaSyncRoot) + return awaitedQuanta.TryDequeue(out handlingItem); + } + private async Task RunQuantumWorker() { while (true) { try { - var handlingItem = default(QuantumProcessingItem); - lock (awaitedQuantaSyncRoot) - awaitedQuanta.TryDequeue(out handlingItem); - if (handlingItem != null) + if (TryGetHandlingItem(out var handlingItem)) { //after Alpha switch, quanta queue can contain requests that should be send to new Alpha server - if (!Context.IsAlpha && handlingItem.Quantum.Apex == 0) + if (!IsReadyToHandle(handlingItem)) { //if the quantum is not client request, than new Alpha will generate it, so we can skip it if (handlingItem.Quantum is RequestQuantumBase request) Context.ProxyWorker.AddRequestsToQueue(request.RequestEnvelope); continue; } + //skip delayed quanta + if (handlingItem.Apex != 0 && handlingItem.Apex <= CurrentApex) + continue; await HandleItem(handlingItem); - if (Context.IsAlpha && QuantaThrottlingManager.Current.IsThrottlingEnabled) + if (Context.NodesManager.IsAlpha && QuantaThrottlingManager.Current.IsThrottlingEnabled) Thread.Sleep(QuantaThrottlingManager.Current.SleepTime); } else @@ -93,6 +95,21 @@ private async Task RunQuantumWorker() } } + private bool IsReadyToHandle(QuantumProcessingItem handlingItem) + { + return Context.NodesManager.IsAlpha + || handlingItem.Quantum.Apex != 0 + || Context.NodesManager.CurrentNode.State == State.WaitingForInit + || ShouldProcessConstellationQuantum(handlingItem); + } + + private bool ShouldProcessConstellationQuantum(QuantumProcessingItem handlingItem) + { + var isAlphaConnectionTimedOut = (Context.NodesManager.AlphaNode.UpdateDate - DateTime.UtcNow).TotalMilliseconds > 2000; + var isConstellationUpdateQuantum = handlingItem.Quantum is ConstellationQuantum; + return isAlphaConnectionTimedOut && isConstellationUpdateQuantum; + } + async Task HandleItem(QuantumProcessingItem processingItem) { try @@ -114,7 +131,7 @@ async Task HandleItem(QuantumProcessingItem processingItem) var originalException = exc.InnerException; processingItem.SetException(originalException); NotifyOnException(processingItem, originalException); - if (!Context.IsAlpha) //if current node is auditor, than all quanta received from Alpha must pass the validation + if (!Context.NodesManager.IsAlpha) //if current node is auditor, than all quanta received from Alpha must pass the validation throw originalException; } catch (Exception exc) @@ -128,7 +145,7 @@ async Task HandleItem(QuantumProcessingItem processingItem) void NotifyOnException(QuantumProcessingItem processingItem, Exception exc) { if (processingItem.Initiator != null && !EnvironmentHelper.IsTest) - Context.Notify(processingItem.Initiator.Pubkey, ((RequestQuantumBase)processingItem.Quantum).RequestEnvelope.CreateResult(exc).CreateEnvelope()); + Context.Notify(processingItem.Initiator.Pubkey, ((ClientRequestQuantumBase)processingItem.Quantum).RequestEnvelope.CreateResult(exc).CreateEnvelope()); } void ValidateQuantum(QuantumProcessingItem processingItem) @@ -143,18 +160,20 @@ void ValidateQuantum(QuantumProcessingItem processingItem) else { if (quantum.Apex != CurrentApex + 1) + { throw new Exception($"Current quantum apex is {quantum.Apex} but {CurrentApex + 1} was expected."); + } if (!quantum.PrevHash.AsSpan().SequenceEqual(LastQuantumHash)) throw new Exception($"Quantum previous hash doesn't equal to last quantum hash."); } - if (!Context.IsAlpha || Context.NodesManager.CurrentNode.State == State.Rising) + if (!Context.NodesManager.IsAlpha || Context.NodesManager.CurrentNode.State == State.Rising) ValidateRequestQuantum(processingItem); } Account GetAccountWrapper(Quantum quantum) { - if (quantum is RequestQuantumBase requestQuantum) + if (quantum is ClientRequestQuantumBase requestQuantum) { var account = Context.AccountStorage.GetAccount(requestQuantum.RequestMessage.Account); if (account == null) @@ -190,10 +209,10 @@ async Task ProcessQuantum(QuantumProcessingItem processingItem) CurrentApex = quantum.Apex; LastQuantumHash = processingItem.QuantumHash; - ProcessResult(processingItem); + //add quantum to quantum sync storage + Context.SyncStorage.AddQuantum(new SyncQuantaBatchItem { Quantum = quantum }); - if (Context.IsAlpha) - LastAddedQuantumApex = processingItem.Quantum.Apex; + ProcessResult(processingItem); processingItem.Processed(); @@ -208,7 +227,7 @@ void ProcessResult(QuantumProcessingItem processingItem) void ValidateRequestQuantum(QuantumProcessingItem processingItem) { - var request = processingItem.Quantum as RequestQuantumBase; + var request = processingItem.Quantum as ClientRequestQuantumBase; if (request == null) return; if (!processingItem.Initiator.RequestCounter.IncRequestCount(request.Timestamp, out string error)) @@ -229,7 +248,7 @@ QuantumProcessorBase GetProcessor(Quantum quantum) string GetMessageType(Quantum quantum) { - if (quantum is RequestQuantumBase requestQuantum) + if (quantum is ClientRequestQuantumBase requestQuantum) return requestQuantum.RequestMessage.GetMessageType(); else if (quantum is ConstellationQuantum constellationQuantum) return constellationQuantum.RequestMessage.GetMessageType(); diff --git a/Centaurus.Domain/Quanta/Processors/AlphaUpdateProcessor.cs b/Centaurus.Domain/Quanta/Processors/AlphaUpdateProcessor.cs index 5fb0d25f..f280d571 100644 --- a/Centaurus.Domain/Quanta/Processors/AlphaUpdateProcessor.cs +++ b/Centaurus.Domain/Quanta/Processors/AlphaUpdateProcessor.cs @@ -19,38 +19,41 @@ public AlphaUpdateProcessor(ExecutionContext context) public override string SupportedMessageType { get; } = typeof(AlphaUpdate).Name; - public override Task Process(QuantumProcessingItem processingItem) + public override async Task Process(QuantumProcessingItem processingItem) { var alphaUpdate = (AlphaUpdate)((ConstellationQuantum)processingItem.Quantum).RequestMessage; //make copy of current settings - var newConstellationSettings = XdrConverter.Deserialize(XdrConverter.Serialize(Context.Constellation)); + var newConstellationSettings = XdrConverter.Deserialize(XdrConverter.Serialize(Context.ConstellationSettingsManager.Current)); newConstellationSettings.Apex = processingItem.Apex; newConstellationSettings.Alpha = alphaUpdate.Alpha; - processingItem.AddConstellationUpdate(Context.Constellation, Context.Constellation); + processingItem.AddConstellationUpdate(Context.ConstellationSettingsManager.Current, Context.ConstellationSettingsManager.Current); - Context.UpdateConstellationSettings(newConstellationSettings); + await Context.UpdateConstellationSettings(newConstellationSettings); - return Task.FromResult((QuantumResultMessageBase)processingItem.Quantum.CreateEnvelope().CreateResult(ResultStatusCode.Success)); + return (QuantumResultMessageBase)processingItem.Quantum.CreateEnvelope().CreateResult(ResultStatusCode.Success); } public override Task Validate(QuantumProcessingItem processingItem) { var currentState = Context.NodesManager.CurrentNode.State; if (currentState == State.Undefined || currentState == State.WaitingForInit) - throw new InvalidOperationException($"Constellation is not initialized yet."); + throw new InvalidOperationException($"ConstellationSettingsManager.Current is not initialized yet."); ((ConstellationQuantum)processingItem.Quantum).Validate(Context); var alphaUpdate = (AlphaUpdate)((ConstellationQuantum)processingItem.Quantum).RequestMessage; - if (alphaUpdate.Alpha.Equals(Context.Constellation.Alpha)) - throw new InvalidOperationException($"{Context.Constellation.Alpha.GetAccountId()} is Alpha already."); + if (alphaUpdate.LastUpdateApex != Context.ConstellationSettingsManager.Current.Apex) + throw new InvalidOperationException($"Last update apex is invalid."); - if (!Context.Constellation.Auditors.Any(a => a.PubKey.Equals(Context.Constellation.Alpha))) - throw new InvalidOperationException($"{Context.Constellation.Alpha.GetAccountId()} is not an auditor."); + if (alphaUpdate.Alpha.Equals(Context.ConstellationSettingsManager.Current.Alpha)) + throw new InvalidOperationException($"{Context.ConstellationSettingsManager.Current.Alpha.GetAccountId()} is Alpha already."); + + if (!Context.ConstellationSettingsManager.Current.Auditors.Any(a => a.PubKey.Equals(Context.ConstellationSettingsManager.Current.Alpha))) + throw new InvalidOperationException($"{Context.ConstellationSettingsManager.Current.Alpha.GetAccountId()} is not an auditor."); return Task.CompletedTask; } diff --git a/Centaurus.Domain/Quanta/Processors/ConstellationUpdateProcessor.cs b/Centaurus.Domain/Quanta/Processors/ConstellationUpdateProcessor.cs index c35643ed..9cc6ae5e 100644 --- a/Centaurus.Domain/Quanta/Processors/ConstellationUpdateProcessor.cs +++ b/Centaurus.Domain/Quanta/Processors/ConstellationUpdateProcessor.cs @@ -24,7 +24,7 @@ public override Task Process(QuantumProcessingItem pro var settings = updateQuantum.ToConstellationSettings(processingItem.Apex); - processingItem.AddConstellationUpdate(settings, Context.Constellation); + processingItem.AddConstellationUpdate(settings, Context.ConstellationSettingsManager.Current); var updateSnapshot = settings.ToSnapshot( processingItem.Apex, @@ -73,6 +73,9 @@ public override Task Validate(QuantumProcessingItem processingItem) if (constellationUpdate.Auditors == null || constellationUpdate.Auditors.Count() < minAuditorsCount) throw new ArgumentException($"Min auditors count is {minAuditorsCount}"); + if (constellationUpdate.LastUpdateApex != (Context.ConstellationSettingsManager.Current?.Apex ?? 0ul)) + throw new InvalidOperationException($"Last update apex is invalid."); + if (!constellationUpdate.Auditors.All(a => a.Address == null || Uri.TryCreate($"http://{a.Address}", UriKind.Absolute, out _))) throw new InvalidOperationException("At least one auditor's address is invalid."); @@ -90,7 +93,7 @@ public override Task Validate(QuantumProcessingItem processingItem) throw new ArgumentException("All asset values must be unique"); if (constellationUpdate.Assets.Count(a => a.IsQuoteAsset) != 1) - throw new ArgumentException("Constellation must contain one quote asset."); + throw new ArgumentException("ConstellationSettingsManager.Current must contain one quote asset."); if (constellationUpdate.Assets.Any(a => a.Code.Length > 4)) throw new Exception("Asset code should not exceed 4 bytes"); diff --git a/Centaurus.Domain/Quanta/Processors/DepositProcessor.cs b/Centaurus.Domain/Quanta/Processors/DepositProcessor.cs index 0c4658f1..ab132fd0 100644 --- a/Centaurus.Domain/Quanta/Processors/DepositProcessor.cs +++ b/Centaurus.Domain/Quanta/Processors/DepositProcessor.cs @@ -68,11 +68,11 @@ private void ProcessDeposit(Deposit deposit, QuantumProcessingItem quantumProces var account = Context.AccountStorage.GetAccount(deposit.Destination); if (account == null) { - var baseAsset = Context.Constellation.QuoteAsset.Code; + var baseAsset = Context.ConstellationSettingsManager.Current.QuoteAsset.Code; //ignore registration with non-base asset or with amount that is less than MinAccountBalance - if (deposit.Asset != baseAsset || deposit.Amount < Context.Constellation.MinAccountBalance) + if (deposit.Asset != baseAsset || deposit.Amount < Context.ConstellationSettingsManager.Current.MinAccountBalance) return; - quantumProcessingItem.AddAccountCreate(Context.AccountStorage, deposit.Destination, Context.Constellation.RequestRateLimits); + quantumProcessingItem.AddAccountCreate(Context.AccountStorage, deposit.Destination, Context.ConstellationSettingsManager.Current.RequestRateLimits); account = Context.AccountStorage.GetAccount(deposit.Destination); } diff --git a/Centaurus.Domain/Quanta/Processors/OrderCancellationProcessor.cs b/Centaurus.Domain/Quanta/Processors/OrderCancellationProcessor.cs index 552b37a5..aa2b0a5e 100644 --- a/Centaurus.Domain/Quanta/Processors/OrderCancellationProcessor.cs +++ b/Centaurus.Domain/Quanta/Processors/OrderCancellationProcessor.cs @@ -19,7 +19,7 @@ public override Task Process(QuantumProcessingItem qua { UpdateNonce(quantumProcessingItem); - Context.Exchange.RemoveOrder(quantumProcessingItem, Context.Constellation.QuoteAsset.Code); + Context.Exchange.RemoveOrder(quantumProcessingItem, Context.ConstellationSettingsManager.Current.QuoteAsset.Code); var resultMessage = quantumProcessingItem.Quantum.CreateEnvelope().CreateResult(ResultStatusCode.Success); return Task.FromResult((QuantumResultMessageBase)resultMessage); @@ -29,7 +29,7 @@ public override Task Validate(QuantumProcessingItem quantumProcessingItem) { ValidateNonce(quantumProcessingItem); - var quantum = (RequestQuantum)quantumProcessingItem.Quantum; + var quantum = (ClientRequestQuantum)quantumProcessingItem.Quantum; var orderRequest = (OrderCancellationRequest)quantum.RequestMessage; var orderWrapper = Context.Exchange.OrderMap.GetOrder(orderRequest.OrderId); diff --git a/Centaurus.Domain/Quanta/Processors/OrderRequestProcessor.cs b/Centaurus.Domain/Quanta/Processors/OrderRequestProcessor.cs index 17a01ec5..34dc55ec 100644 --- a/Centaurus.Domain/Quanta/Processors/OrderRequestProcessor.cs +++ b/Centaurus.Domain/Quanta/Processors/OrderRequestProcessor.cs @@ -21,10 +21,10 @@ public override Task Process(QuantumProcessingItem pro { UpdateNonce(processingItem); - var quantum = (RequestQuantumBase)processingItem.Quantum; + var quantum = (ClientRequestQuantumBase)processingItem.Quantum; var orderRequest = (OrderRequest)quantum.RequestEnvelope.Message; - Context.Exchange.ExecuteOrder(orderRequest.Asset, Context.Constellation.QuoteAsset.Code, processingItem); + Context.Exchange.ExecuteOrder(orderRequest.Asset, Context.ConstellationSettingsManager.Current.QuoteAsset.Code, processingItem); return Task.FromResult((QuantumResultMessageBase)quantum.CreateEnvelope().CreateResult(ResultStatusCode.Success)); } @@ -35,14 +35,14 @@ public override Task Validate(QuantumProcessingItem processingItem) { ValidateNonce(processingItem); - var quantum = (RequestQuantumBase)processingItem.Quantum; + var quantum = (ClientRequestQuantumBase)processingItem.Quantum; var orderRequest = (OrderRequest)quantum.RequestEnvelope.Message; - var baseAsset = Context.Constellation.QuoteAsset; + var baseAsset = Context.ConstellationSettingsManager.Current.QuoteAsset; if (baseAsset.Code == orderRequest.Asset) throw new BadRequestException("Order asset must be different from quote asset."); - var orderAsset = Context.Constellation.Assets.FirstOrDefault(a => a.Code == orderRequest.Asset); + var orderAsset = Context.ConstellationSettingsManager.Current.Assets.FirstOrDefault(a => a.Code == orderRequest.Asset); if (orderAsset == null) throw new BadRequestException("Invalid asset identifier: " + orderRequest.Asset); @@ -53,7 +53,7 @@ public override Task Validate(QuantumProcessingItem processingItem) var quoteAmount = OrderMatcher.EstimateQuoteAmount(orderRequest.Amount, orderRequest.Price, orderRequest.Side); //check that lot size is greater than minimum allowed lot - if (quoteAmount < Context.Constellation.MinAllowedLotSize) + if (quoteAmount < Context.ConstellationSettingsManager.Current.MinAllowedLotSize) throw new BadRequestException("Lot size is smaller than the minimum allowed lot."); //check required balances @@ -66,7 +66,7 @@ public override Task Validate(QuantumProcessingItem processingItem) else { var balance = processingItem.Initiator.GetBalance(baseAsset.Code); - if (!balance.HasSufficientBalance(quoteAmount, Context.Constellation.MinAccountBalance)) + if (!balance.HasSufficientBalance(quoteAmount, Context.ConstellationSettingsManager.Current.MinAccountBalance)) throw new BadRequestException("Insufficient funds"); } diff --git a/Centaurus.Domain/Quanta/Processors/Payments/PaymentRequestProcessor.cs b/Centaurus.Domain/Quanta/Processors/Payments/PaymentRequestProcessor.cs index 51b41e1d..20d9e025 100644 --- a/Centaurus.Domain/Quanta/Processors/Payments/PaymentRequestProcessor.cs +++ b/Centaurus.Domain/Quanta/Processors/Payments/PaymentRequestProcessor.cs @@ -19,13 +19,13 @@ public override Task Process(QuantumProcessingItem qua { UpdateNonce(quantumProcessingItem); - var request = (RequestQuantumBase)quantumProcessingItem.Quantum; + var request = (ClientRequestQuantumBase)quantumProcessingItem.Quantum; var payment = (PaymentRequest)request.RequestMessage; var destinationAccount = Context.AccountStorage.GetAccount(payment.Destination); if (destinationAccount == null) { - quantumProcessingItem.AddAccountCreate(Context.AccountStorage, payment.Destination, Context.Constellation.RequestRateLimits); + quantumProcessingItem.AddAccountCreate(Context.AccountStorage, payment.Destination, Context.ConstellationSettingsManager.Current.RequestRateLimits); destinationAccount = Context.AccountStorage.GetAccount(payment.Destination); } @@ -44,21 +44,21 @@ public override Task Validate(QuantumProcessingItem quantumProcessingItem) { ValidateNonce(quantumProcessingItem); - var request = (RequestQuantumBase)quantumProcessingItem.Quantum; + var request = (ClientRequestQuantumBase)quantumProcessingItem.Quantum; var payment = (PaymentRequest)request.RequestMessage; if (payment.Destination == null || payment.Destination.IsZero()) throw new BadRequestException("Destination should be valid public key"); - var baseAsset = Context.Constellation.QuoteAsset.Code; + var baseAsset = Context.ConstellationSettingsManager.Current.QuoteAsset.Code; var destinationAccount = Context.AccountStorage.GetAccount(payment.Destination); //TODO: should we allow payment that is less than min account balance? if (destinationAccount == null) { if (payment.Asset != baseAsset) throw new BadRequestException("Account excepts only XLM asset."); - if (payment.Amount < Context.Constellation.MinAccountBalance) - throw new BadRequestException($"Min payment amount is {Context.Constellation.MinAccountBalance} for this account."); + if (payment.Amount < Context.ConstellationSettingsManager.Current.MinAccountBalance) + throw new BadRequestException($"Min payment amount is {Context.ConstellationSettingsManager.Current.MinAccountBalance} for this account."); } if (payment.Destination.Equals(quantumProcessingItem.Initiator.Pubkey)) @@ -67,10 +67,10 @@ public override Task Validate(QuantumProcessingItem quantumProcessingItem) if (payment.Amount <= 0) throw new BadRequestException("Amount should be greater than 0"); - if (!Context.Constellation.Assets.Any(a => a.Code == payment.Asset)) + if (!Context.ConstellationSettingsManager.Current.Assets.Any(a => a.Code == payment.Asset)) throw new BadRequestException($"Asset {payment.Asset} is not supported"); - var minBalance = payment.Asset == baseAsset ? Context.Constellation.MinAccountBalance : 0; + var minBalance = payment.Asset == baseAsset ? Context.ConstellationSettingsManager.Current.MinAccountBalance : 0; if (!(quantumProcessingItem.Initiator.GetBalance(payment.Asset)?.HasSufficientBalance(payment.Amount, minBalance) ?? false)) throw new BadRequestException("Insufficient funds"); diff --git a/Centaurus.Domain/Quanta/Processors/Payments/WithdrawalRequestProcessor.cs b/Centaurus.Domain/Quanta/Processors/Payments/WithdrawalRequestProcessor.cs index 0968cf8d..d50cbe78 100644 --- a/Centaurus.Domain/Quanta/Processors/Payments/WithdrawalRequestProcessor.cs +++ b/Centaurus.Domain/Quanta/Processors/Payments/WithdrawalRequestProcessor.cs @@ -36,7 +36,7 @@ public override Task Validate(QuantumProcessingItem quantumProcessingItem) var withdrawalQuantum = (WithdrawalRequestQuantum)quantumProcessingItem.Quantum; var withdrawalRequest = (WithdrawalRequest)withdrawalQuantum.RequestMessage; - var centaurusAsset = Context.Constellation.Assets.FirstOrDefault(a => a.Code == withdrawalRequest.Asset); + var centaurusAsset = Context.ConstellationSettingsManager.Current.Assets.FirstOrDefault(a => a.Code == withdrawalRequest.Asset); if (centaurusAsset == null || centaurusAsset.IsSuspended) throw new BadRequestException($"Constellation doesn't support asset '{withdrawalRequest.Asset}'."); @@ -47,9 +47,9 @@ public override Task Validate(QuantumProcessingItem quantumProcessingItem) if (providerAsset == null) throw new BadRequestException($"Current provider doesn't support withdrawal of asset {centaurusAsset.Code}."); - var baseAsset = Context.Constellation.QuoteAsset.Code; + var baseAsset = Context.ConstellationSettingsManager.Current.QuoteAsset.Code; - var minBalance = centaurusAsset.Code == baseAsset ? Context.Constellation.MinAccountBalance : 0; + var minBalance = centaurusAsset.Code == baseAsset ? Context.ConstellationSettingsManager.Current.MinAccountBalance : 0; if (!(sourceAccount.GetBalance(centaurusAsset.Code)?.HasSufficientBalance(withdrawalRequest.Amount + withdrawalRequest.Fee, minBalance) ?? false)) throw new BadRequestException($"Insufficient balance."); diff --git a/Centaurus.Domain/Quanta/Processors/QuantumProcessorBase.cs b/Centaurus.Domain/Quanta/Processors/QuantumProcessorBase.cs index 8d97fc17..d5e9bd4b 100644 --- a/Centaurus.Domain/Quanta/Processors/QuantumProcessorBase.cs +++ b/Centaurus.Domain/Quanta/Processors/QuantumProcessorBase.cs @@ -37,7 +37,7 @@ public QuantumProcessorBase(ExecutionContext context) public static void UpdateNonce(QuantumProcessingItem processingItem) { - var requestQuantum = (RequestQuantumBase)processingItem.Quantum; + var requestQuantum = (ClientRequestQuantumBase)processingItem.Quantum; var requestMessage = requestQuantum.RequestMessage; processingItem.AddNonceUpdate(processingItem.Initiator, requestMessage.Nonce, processingItem.Initiator.Nonce); @@ -45,13 +45,13 @@ public static void UpdateNonce(QuantumProcessingItem processingItem) public static void ValidateNonce(QuantumProcessingItem processingItem) { - var requestQuantum = processingItem.Quantum as RequestQuantumBase; + var requestQuantum = processingItem.Quantum as ClientRequestQuantumBase; if (requestQuantum == null) - throw new BadRequestException($"Invalid message type. Client quantum message should be of type {typeof(RequestQuantumBase).Name}."); + throw new BadRequestException($"Invalid message type. Client quantum message should be of type {typeof(ClientRequestQuantumBase).Name}."); var requestMessage = requestQuantum.RequestEnvelope.Message as SequentialRequestMessage; if (requestMessage == null) - throw new BadRequestException($"Invalid message type. {typeof(RequestQuantumBase).Name} should contain message of type {typeof(SequentialRequestMessage).Name}."); + throw new BadRequestException($"Invalid message type. {typeof(ClientRequestQuantumBase).Name} should contain message of type {typeof(SequentialRequestMessage).Name}."); if (requestMessage.Nonce < 1 || processingItem.Initiator.Nonce >= requestMessage.Nonce) throw new UnauthorizedException($"Specified nonce is invalid. Current nonce: {processingItem.Initiator.Nonce}; request nonce: {requestMessage.Nonce}."); diff --git a/Centaurus.Domain/Quanta/QuantaProcessingItems/QuantumProcessingItem.cs b/Centaurus.Domain/Quanta/QuantaProcessingItems/QuantumProcessingItem.cs index cc730cb5..97d10538 100644 --- a/Centaurus.Domain/Quanta/QuantaProcessingItems/QuantumProcessingItem.cs +++ b/Centaurus.Domain/Quanta/QuantaProcessingItems/QuantumProcessingItem.cs @@ -45,8 +45,6 @@ public QuantumProcessingItem(Quantum quantum, Task signatureValidation) public QuantumPersistentModel PersistentModel { get; private set; } - public byte AlphaId { get; private set; } - public byte CurrentAuditorId { get; private set; } public Task OnProcessed => onProcessedTaskSource.Task; @@ -167,7 +165,7 @@ public void Complete(ExecutionContext context, QuantumResultMessageBase quantumR //add quantum data to updates batch and assign persistent model PersistentModel = context.PendingUpdatesManager.AddQuantum(this); - AlphaId = context.AlphaId; + CurrentAuditorId = context.NodesManager.CurrentNode.Id; } List GetRawEffectsDataContainer() @@ -224,7 +222,6 @@ void BuildProcessingResult(ExecutionContext context, QuantumResultMessageBase qu PayloadHash = payloadHash; Effects = rawEffects; ResultMessage = quantumResultMessage; - CurrentAuditorId = context.AuditorPubKeys[context.Settings.KeyPair]; //assign the serialized quantum to the result ResultMessage.Request = new RequestInfo { Data = RawQuantum }; diff --git a/Centaurus.Domain/Quanta/QuantumSignatureValidator.cs b/Centaurus.Domain/Quanta/QuantumSignatureValidator.cs index b06eb747..6e7cc92e 100644 --- a/Centaurus.Domain/Quanta/QuantumSignatureValidator.cs +++ b/Centaurus.Domain/Quanta/QuantumSignatureValidator.cs @@ -16,7 +16,7 @@ static QuantumSignatureValidator() public static Task Validate(Quantum quantum) { - if (quantum is RequestQuantumBase request) + if (quantum is ClientRequestQuantumBase request) { var verificationTask = new TaskCompletionSource(); verificationTasks.Add(() => VerifyQuantumSignature(verificationTask, request)); @@ -33,7 +33,7 @@ private static void StartVerifications() Parallel.ForEach(partitioner, options, verify => verify()); } - private static void VerifyQuantumSignature(TaskCompletionSource verificationTask, RequestQuantumBase request) + private static void VerifyQuantumSignature(TaskCompletionSource verificationTask, ClientRequestQuantumBase request) { try { diff --git a/Centaurus.Domain/Quanta/Sync/ApexItemsBatchPortion.cs b/Centaurus.Domain/Quanta/Sync/ApexItemsBatchPortion.cs index 6f7102c5..dfa07335 100644 --- a/Centaurus.Domain/Quanta/Sync/ApexItemsBatchPortion.cs +++ b/Centaurus.Domain/Quanta/Sync/ApexItemsBatchPortion.cs @@ -39,12 +39,12 @@ protected SyncPortion GetBatchData() { switch (source) { - case ApexItemsBatch signaturesBatch: + case ApexItemsBatch majoritySignaturesBatch: { - var items = signaturesBatch.GetItems(Start, Size, true); - var batch = new QuantumMajoritySignaturesBatch + var items = majoritySignaturesBatch.GetItems(Start, Size, true); + var batch = new MajoritySignaturesBatch { - Signatures = items + Items = items }; return new SyncPortion(batch.CreateEnvelope().ToByteArray(), items.Last().Apex); } @@ -57,6 +57,15 @@ protected SyncPortion GetBatchData() }; return new SyncPortion(batch.CreateEnvelope().ToByteArray(), items.Last().Apex); } + case ApexItemsBatch signaturesBatch: + { + var items = signaturesBatch.GetItems(Start, Size, true); + var batch = new SingleNodeSignaturesBatch + { + Items = items + }; + return new SyncPortion(batch.CreateEnvelope().ToByteArray(), items.Last().Apex); + } default: throw new NotImplementedException($"{nameof(source)} is not supported."); } diff --git a/Centaurus.Domain/Quanta/Sync/CursorGroup.cs b/Centaurus.Domain/Quanta/Sync/CursorGroup.cs index 3841c173..6627e9ff 100644 --- a/Centaurus.Domain/Quanta/Sync/CursorGroup.cs +++ b/Centaurus.Domain/Quanta/Sync/CursorGroup.cs @@ -1,5 +1,4 @@ using Centaurus.Domain.Quanta.Sync; -using Centaurus.Domain.StateManagers; using System; using System.Collections.Generic; diff --git a/Centaurus.Domain/Quanta/Sync/ProxyWorker.cs b/Centaurus.Domain/Quanta/Sync/ProxyWorker.cs index df739e7a..017bcce3 100644 --- a/Centaurus.Domain/Quanta/Sync/ProxyWorker.cs +++ b/Centaurus.Domain/Quanta/Sync/ProxyWorker.cs @@ -46,7 +46,7 @@ private void Run() hasQuantaToSend = requestsToSend != null; if (hasQuantaToSend) { - if (!Context.NodesManager.TryGetNode(Context.Constellation.Alpha, out var node)) + if (!Context.NodesManager.TryGetNode(Context.ConstellationSettingsManager.Current.Alpha, out var node)) throw new Exception($"Unable to get Alpha node."); var connection = node.GetConnection(); if (connection == null) @@ -79,11 +79,11 @@ public void SetAuditorConnection(OutgoingConnection alpha) public OutgoingConnection AlphaConnection { get; private set; } - public void AddRequestsToQueue(params MessageEnvelopeBase[] clientRequests) + public void AddRequestsToQueue(params MessageEnvelopeBase[] requests) { - if (!Context.IsAlpha) + if (!Context.NodesManager.IsAlpha) lock (quantaCollectionSyncRoot) - requests.AddRange(clientRequests); + this.requests.AddRange(requests); //TODO: add to QuantumHandler } diff --git a/Centaurus.Domain/Quanta/Sync/SyncCursorUpdate.cs b/Centaurus.Domain/Quanta/Sync/SyncCursorUpdate.cs index 9b046b39..b8b1b33b 100644 --- a/Centaurus.Domain/Quanta/Sync/SyncCursorUpdate.cs +++ b/Centaurus.Domain/Quanta/Sync/SyncCursorUpdate.cs @@ -1,12 +1,12 @@ -using Centaurus.Domain.StateManagers; -using System; +using System; namespace Centaurus.Domain.Quanta.Sync { public enum SyncCursorType { Quanta = 0, - Signatures = 1 + MajoritySignatures = 1, + SingleNodeSignatures = 2 } internal class SyncCursorUpdate diff --git a/Centaurus.Domain/Quanta/Sync/SyncQuantaDataWorker.cs b/Centaurus.Domain/Quanta/Sync/SyncQuantaDataWorker.cs index 54101b60..c4d4b631 100644 --- a/Centaurus.Domain/Quanta/Sync/SyncQuantaDataWorker.cs +++ b/Centaurus.Domain/Quanta/Sync/SyncQuantaDataWorker.cs @@ -1,5 +1,4 @@ using Centaurus.Domain.Quanta.Sync; -using Centaurus.Domain.StateManagers; using Centaurus.Models; using NLog; using System; @@ -73,7 +72,7 @@ private List> SendQuantaData(List cursor foreach (var cursorGroup in cursorGroups) { var cursorSendingTasks = ProcessCursorGroup(cursorGroup); - if (cursorSendingTasks.Count > 0) + if (cursorSendingTasks != null) sendingQuantaTasks.AddRange(cursorSendingTasks); } return sendingQuantaTasks; @@ -87,8 +86,6 @@ private List> ProcessCursorGroup(CursorGroup cursorGr var currentCursor = cursorGroup.BatchId; var cursorType = cursorGroup.CursorType; - var lastBatchApex = batch.LastDataApex; - var sendingQuantaTasks = new List>(); foreach (var node in cursorGroup.Nodes) { @@ -102,12 +99,12 @@ private List> ProcessCursorGroup(CursorGroup cursorGr private static Task SendSingleBatch(SyncPortion batch, ulong currentCursor, SyncCursorType cursorType, NodeCursorData currentAuditor) { var connection = currentAuditor.Node.GetConnection(); - if (currentAuditor.Cursor < batch.LastDataApex || connection == null) + if (currentAuditor.Cursor >= batch.LastDataApex || connection == null) return null; var sendMessageTask = connection.SendMessage(batch.Data.AsMemory()); return sendMessageTask.ContinueWith(t => { - if (!t.IsFaulted) + if (t.IsFaulted) { HandleFaultedSendTask(t, currentCursor, batch, cursorType, currentAuditor); return null; @@ -127,8 +124,11 @@ private bool TryGetBatch(CursorGroup cursorGroup, out SyncPortion batch) case SyncCursorType.Quanta: batch = Context.SyncStorage.GetQuanta(cursorGroup.BatchId, force); break; - case SyncCursorType.Signatures: - batch = Context.SyncStorage.GetSignatures(cursorGroup.BatchId, force); + case SyncCursorType.MajoritySignatures: + batch = Context.SyncStorage.GetMajoritySignatures(cursorGroup.BatchId, force); + break; + case SyncCursorType.SingleNodeSignatures: + batch = Context.SyncStorage.GetCurrentNodeSignatures(cursorGroup.BatchId, force); break; default: throw new NotImplementedException($"{cursorGroup.CursorType} cursor type is not supported."); @@ -139,11 +139,12 @@ private bool TryGetBatch(CursorGroup cursorGroup, out SyncPortion batch) private bool ValidateCursorGroup(CursorGroup cursorGroup) { var currentCursor = cursorGroup.BatchId; - if (currentCursor == Context.QuantumHandler.CurrentApex) + var currentApex = Context.QuantumHandler.CurrentApex; + if (currentCursor == currentApex) return false; - if (currentCursor > Context.QuantumHandler.CurrentApex && GetIsCurrentNodeReady()) + if (currentCursor > currentApex && GetIsCurrentNodeReady()) { - var message = $"Auditors {string.Join(',', cursorGroup.Nodes.Select(a => a.Node.AccountId))} is above current constellation state."; + var message = $"Node(s) {string.Join(',', cursorGroup.Nodes.Select(a => a.Node.AccountId))} is above current constellation state, cursor {currentCursor}, current apex {currentApex} type {cursorGroup.CursorType}."; if (cursorGroup.Nodes.Count >= Context.GetMajorityCount() - 1) //-1 is current server logger.Error(message); else @@ -166,7 +167,7 @@ private async Task SyncQuantaData() if (!hasPendingQuanta) Thread.Sleep(50); - if (!Context.IsAlpha || !GetIsCurrentNodeReady()) + if (!GetIsCurrentNodeReady()) { hasPendingQuanta = false; continue; diff --git a/Centaurus.Domain/Quanta/Sync/SyncStorage.cs b/Centaurus.Domain/Quanta/Sync/SyncStorage.cs index 9c85378a..5fc67777 100644 --- a/Centaurus.Domain/Quanta/Sync/SyncStorage.cs +++ b/Centaurus.Domain/Quanta/Sync/SyncStorage.cs @@ -54,12 +54,26 @@ public SyncPortion GetQuanta(ulong from, bool force) /// Exclusive /// if force, than batch would be return even if it's not fulfilled yet /// - public SyncPortion GetSignatures(ulong from, bool force) + public SyncPortion GetMajoritySignatures(ulong from, bool force) { var quantaBatchStart = GetBatchApexStart(from + 1); var batch = GetBatch(quantaBatchStart); - return batch.Signatures.GetData(from, force); + return batch.MajoritySignatures.GetData(from, force); + } + + /// + /// + /// + /// Exclusive + /// if force, than batch would be return even if it's not fulfilled yet + /// + public SyncPortion GetCurrentNodeSignatures(ulong from, bool force) + { + var quantaBatchStart = GetBatchApexStart(from + 1); + var batch = GetBatch(quantaBatchStart); + + return batch.CurrentNodeSignatures.GetData(from, force); } /// @@ -82,26 +96,47 @@ public List GetQuanta(ulong from, int limit) /// Exclusive /// /// - public List GetSignatures(ulong from, int limit) + public List GetMajoritySignatures(ulong from, int limit) { var quantaBatchStart = GetBatchApexStart(from + 1); var batch = GetBatch(quantaBatchStart); - return batch.Signatures.GetItems(from, limit); + return batch.MajoritySignatures.GetItems(from, limit); } - public void AddQuantum(ulong apex, SyncQuantaBatchItem quantum) + /// + /// + /// + /// Exclusive + /// + /// + public List GetCurrentNodeSignatures(ulong from, int limit) + { + var quantaBatchStart = GetBatchApexStart(from + 1); + var batch = GetBatch(quantaBatchStart); + + return batch.CurrentNodeSignatures.GetItems(from, limit); + } + + public void AddQuantum(SyncQuantaBatchItem quantum) { - var batchStart = GetBatchApexStart(apex); + var batchStart = GetBatchApexStart(quantum.Apex); var batch = GetBatch(batchStart); - batch.Quanta.Add(apex, quantum); + batch.Quanta.Add(quantum.Apex, quantum); } - public void AddSignatures(ulong apex, QuantumSignatures signatures) + public void AddSignatures(MajoritySignaturesBatchItem signatures) { - var batchStart = GetBatchApexStart(apex); + var batchStart = GetBatchApexStart(signatures.Apex); var batch = GetBatch(batchStart); - batch.Signatures.Add(apex, signatures); + batch.MajoritySignatures.Add(signatures.Apex, signatures); + } + + public void AddCurrentNodeSignature(SingleNodeSignaturesBatchItem signature) + { + var batchStart = GetBatchApexStart(signature.Apex); + var batch = GetBatch(batchStart); + batch.CurrentNodeSignatures.Add(signature.Apex, signature); } private object persistedBatchIdsSyncRoot = new { }; @@ -218,38 +253,43 @@ private SyncStorageItem LoadBatch(ulong batchStartApex, int batchSize) logger.Info($"Quanta batch with apex start {batchStartApex} is loaded."); var quanta = new List(); - var signatures = new List(); + var majoritySignatures = new List(); + var currentNodeSignatures = new List(); foreach (var rawQuantum in rawQuanta) { quanta.Add(rawQuantum.ToBatchItemQuantum()); - signatures.Add(rawQuantum.ToQuantumSignatures()); + majoritySignatures.Add(rawQuantum.ToMajoritySignatures()); } if (batchStartApex == 0) //insert null at 0 position, otherwise index will not be relevant to apex { quanta.Insert(0, null); - signatures.Insert(0, null); + majoritySignatures.Insert(0, null); + currentNodeSignatures.Insert(0, null); } - return new SyncStorageItem(Context, batchStartApex, PortionSize, quanta, signatures); + return new SyncStorageItem(Context, batchStartApex, PortionSize, quanta, majoritySignatures, currentNodeSignatures); } class SyncStorageItem: ContextualBase { - public SyncStorageItem(ExecutionContext context, ulong batchId, int portionSize, List quanta, List signatures) + public SyncStorageItem(ExecutionContext context, ulong batchId, int portionSize, List quanta, List majoritySignatures, List signatures) :base(context) { BatchStart = batchId; BatchEnd = batchId + BatchSize - 1; //batch start is inclusive Quanta = new ApexItemsBatch(Context, batchId, BatchSize, portionSize, quanta); - Signatures = new ApexItemsBatch(Context, batchId, BatchSize, portionSize, signatures); + MajoritySignatures = new ApexItemsBatch(Context, batchId, BatchSize, portionSize, majoritySignatures); + CurrentNodeSignatures = new ApexItemsBatch(Context, batchId, BatchSize, portionSize, signatures); } - public ApexItemsBatch Signatures { get; } + public ApexItemsBatch MajoritySignatures { get; } public ApexItemsBatch Quanta { get; } - public bool IsFulfilled => Quanta.IsFulfilled && Signatures.IsFulfilled; + public ApexItemsBatch CurrentNodeSignatures { get; } + + public bool IsFulfilled => Quanta.IsFulfilled && MajoritySignatures.IsFulfilled; public ulong BatchStart { get; } diff --git a/Centaurus.Models/Results/MajorityResult.cs b/Centaurus.Domain/ResultManagers/MajorityResult.cs similarity index 53% rename from Centaurus.Models/Results/MajorityResult.cs rename to Centaurus.Domain/ResultManagers/MajorityResult.cs index 1d3d7948..dda6b742 100644 --- a/Centaurus.Models/Results/MajorityResult.cs +++ b/Centaurus.Domain/ResultManagers/MajorityResult.cs @@ -1,8 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace Centaurus.Models +namespace Centaurus.Domain { public enum MajorityResult { diff --git a/Centaurus.Domain/ResultManagers/ResultManager.cs b/Centaurus.Domain/ResultManagers/ResultManager.cs index 1acc4296..6abce776 100644 --- a/Centaurus.Domain/ResultManagers/ResultManager.cs +++ b/Centaurus.Domain/ResultManagers/ResultManager.cs @@ -37,7 +37,7 @@ public void Add(QuantumProcessingItem processingResult) results.Add(processingResult); } - public void Add(QuantumSignatures resultMessage) + public void Add(MajoritySignaturesBatchItem resultMessage) { auditorResults.Add(resultMessage); } @@ -73,7 +73,7 @@ private ulong GetBatchApexStart(ulong apex) private BlockingCollection results = new BlockingCollection(); - private BlockingCollection auditorResults = new BlockingCollection(); + private BlockingCollection auditorResults = new BlockingCollection(); private void ProcessResults() { @@ -112,7 +112,7 @@ private void ProcessAuditorResults() }); } - private void AddInternal(QuantumSignatures signaturesMessage) + private void AddInternal(MajoritySignaturesBatchItem signaturesMessage) { //auditor can send delayed results if (signaturesMessage.Apex <= Context.PendingUpdatesManager.LastPersistedApex) @@ -179,7 +179,7 @@ private void UpdateCache() { var currentBatchId = currentBatchIds.Last(); var nextBatchId = currentBatchId + batchSize; - if (nextBatchId - Context.QuantumHandler.LastAddedQuantumApex < advanceThreshold) + if (nextBatchId - Context.QuantumHandler.CurrentApex < advanceThreshold) CreateBatch(nextBatchId); //copy batch to be able to modify original var _currentBatchIds = currentBatchIds.ToList(); @@ -258,7 +258,7 @@ public void Add(NodeSignatureInternal signature) lock (syncRoot) { - if (processedAuditors.Contains(signature.AuditorId)) + if (processedAuditors.Contains(signature.NodeId)) return; //if current server is not Alpha than it can delay @@ -269,22 +269,23 @@ public void Add(NodeSignatureInternal signature) } //obtain auditor from constellation - if (!Manager.Context.AuditorIds.TryGetValue((byte)signature.AuditorId, out var auditor)) + if (!(Manager.Context.ConstellationSettingsManager.TryGetForApex(Apex, out var constellation) + && constellation.TryGetNodePubKey((byte)signature.NodeId, out var nodePubKey))) return; //check if signature is valid and tx signature is presented for TxResultMessage - if (signature.PayloadSignature.IsValid(auditor, ProcessingItem.PayloadHash)) + if (signature.PayloadSignature.IsValid(nodePubKey, ProcessingItem.PayloadHash)) AddValidSignature(signature); majorityResult = GetMajorityResult(); - if (IsFinalized || majorityResult == MajorityResult.Unknown || !IsSignedByAlpha()) + if (IsFinalized || majorityResult == MajorityResult.Unknown) return; else if (majorityResult == MajorityResult.Unreachable) { var votesCount = signatures.Count; var quantum = ProcessingItem.Quantum; - SequentialRequestMessage requestMessage = quantum is RequestQuantumBase requestQuantum + SequentialRequestMessage requestMessage = quantum is ClientRequestQuantumBase requestQuantum ? requestQuantum.RequestMessage : null; @@ -310,40 +311,20 @@ public void Add(QuantumProcessingItem quantumProcessingItem) //mark as acknowledged ProcessingItem.Acknowledged(); //send result to auditors - Manager.Context.NodesManager.EnqueueResult(new AuditorResult { Apex = Apex, Signature = signature }); + Manager.Context.SyncStorage.AddCurrentNodeSignature(new SingleNodeSignaturesBatchItem { Apex = Apex, Signature = signature }); ProcessOutrunSignatures(this); } } private void AddValidSignature(NodeSignatureInternal signature) { - processedAuditors.Add(signature.AuditorId); + processedAuditors.Add(signature.NodeId); //skip if processed or current auditor already sent the result if (IsFinalized) return; - //Alpha signature must always be first - if (signature.AuditorId == ProcessingItem.AlphaId) - { - signatures.Insert(0, signature); - ProcessingItem.ResultMessage.PayloadProof.Signatures.Insert(0, signature.PayloadSignature); - //add quantum to quantum sync storage - Manager.Context.SyncStorage.AddQuantum(Apex, - new SyncQuantaBatchItem - { - Quantum = ProcessingItem.Quantum, - AlphaSignature = signature - }); - } - else - { - signatures.Add(signature); - ProcessingItem.ResultMessage.PayloadProof.Signatures.Add(signature.PayloadSignature); - } - } - private bool IsSignedByAlpha() - { - return signatures.Any(s => s.AuditorId == ProcessingItem.AlphaId); + signatures.Add(signature); + ProcessingItem.ResultMessage.PayloadProof.Signatures.Add(signature.PayloadSignature); } private NodeSignatureInternal AddCurrentNodeSignature() @@ -359,7 +340,7 @@ private NodeSignatureInternal GetSignature() { var currentAuditorSignature = new NodeSignatureInternal { - AuditorId = ProcessingItem.CurrentAuditorId, + NodeId = ProcessingItem.CurrentAuditorId, PayloadSignature = ProcessingItem.PayloadHash.Sign(Manager.Context.Settings.KeyPair) }; @@ -425,10 +406,10 @@ private void PersistSignatures() //assign signatures to persistent model ProcessingItem.PersistentModel.Signatures = signatures.Select(s => s.ToPersistenModel()).ToList(); //add signatures to sync storage - Manager.Context.SyncStorage.AddSignatures(Apex, new QuantumSignatures - { - Apex = Apex, - Signatures = signatures.Skip(1).ToList() //skip first. The first signature will be synced with quantum + Manager.Context.SyncStorage.AddSignatures(new MajoritySignaturesBatchItem + { + Apex = Apex, + Signatures = signatures.ToList() }); } diff --git a/Centaurus.Domain/UpdatesManager/UpdatesManager.cs b/Centaurus.Domain/UpdatesManager/UpdatesManager.cs index 7e9e938b..f02ddd8d 100644 --- a/Centaurus.Domain/UpdatesManager/UpdatesManager.cs +++ b/Centaurus.Domain/UpdatesManager/UpdatesManager.cs @@ -39,6 +39,9 @@ public void UpdateBatch(bool force = false) SyncRoot.Wait(); try { + if (pendingUpdates.QuantaCount < 1) + return; + logger.Info($"About to updated batch. Id: {pendingUpdates.Id}, apex range: {pendingUpdates.FirstApex}-{pendingUpdates.LastApex}, quanta: {pendingUpdates.QuantaCount}, effects: {pendingUpdates.EffectsCount}, affected accounts: {pendingUpdates.AffectedAccountsCount}."); pendingUpdates.Complete(Context); @@ -98,11 +101,11 @@ public void ApplyUpdates() } catch (Exception exc) { - updates = null; if (Context.NodesManager.CurrentNode.State != State.Failed) { Context.NodesManager.CurrentNode.Failed(new Exception("Saving failed.", exc)); } + return; } } } @@ -164,7 +167,7 @@ public QuantumPersistentModel AddQuantum(QuantumProcessingItem quantumProcessing })); if (quantumProcessingItem.HasSettingsUpdate) - pendingUpdates.AddConstellation(Context.Constellation.ToPesrsistentModel()); + pendingUpdates.AddConstellation(Context.ConstellationSettingsManager.Current.ToPesrsistentModel()); pendingUpdates.AddAffectedAccounts(quantumProcessingItem.Effects.Select(e => e.Account).Where(a => a != null)); diff --git a/Centaurus.Domain/WebSockets/Centaurus/ConnectionBase.cs b/Centaurus.Domain/WebSockets/Centaurus/ConnectionBase.cs index 49a2f77a..1a9c8b1e 100644 --- a/Centaurus.Domain/WebSockets/Centaurus/ConnectionBase.cs +++ b/Centaurus.Domain/WebSockets/Centaurus/ConnectionBase.cs @@ -10,22 +10,6 @@ namespace Centaurus { - public enum ConnectionState - { - /// - /// Connection established. - /// - Connected = 0, - /// - /// Ready to receive and send messages. - /// - Ready = 1, - /// - /// Connection was closed. - /// - Closed = 3 - } - public abstract class ConnectionBase : ContextualBase, IDisposable { static Logger logger = LogManager.GetCurrentClassLogger(); @@ -53,40 +37,24 @@ public ConnectionBase(Domain.ExecutionContext context, KeyPair pubKey, WebSocket public string PubKeyAddress { get; } + public abstract bool IsAuditor { get; } + + public bool IsAuthenticated { get; private set; } + protected virtual int inBufferSize { get; } = 1024; protected virtual int outBufferSize { get; } = 64 * 1024; protected XdrBufferFactory.RentedBuffer incommingBuffer; protected XdrBufferFactory.RentedBuffer outgoingBuffer; - public abstract bool IsAuditor { get; } - - ConnectionState connectionState; - /// - /// Current connection state - /// - public ConnectionState ConnectionState + protected void Authenticated() { - get - { - return connectionState; - } - set - { - if (connectionState != value) - { - var prevValue = connectionState; - connectionState = value; - logger.Trace($"Connection {PubKeyAddress} is in {connectionState} state. Prev state is {prevValue}."); - OnConnectionStateChanged?.Invoke((this, prevValue, connectionState)); - - if (value == ConnectionState.Ready && prevValue == ConnectionState.Connected) //prevent multiple invocation - Context.ExtensionsManager.ConnectionReady(this); - } - } + IsAuthenticated = true; + OnAuthenticated?.Invoke(this); } - public event Action<(ConnectionBase connection, ConnectionState prev, ConnectionState current)> OnConnectionStateChanged; + public event Action OnAuthenticated; + public event Action OnClosed; protected readonly CancellationTokenSource cancellationTokenSource; protected readonly CancellationToken cancellationToken; @@ -216,7 +184,7 @@ public async Task Listen() while (webSocket.State != WebSocketState.Closed && webSocket.State != WebSocketState.Aborted && !cancellationToken.IsCancellationRequested) { //if connection isn't validated yet 256 bytes of max message is enough for handling handshake response - var maxLength = ConnectionState == ConnectionState.Connected ? WebSocketExtension.ChunkSize : 0; + var maxLength = IsAuthenticated ? 0 : WebSocketExtension.ChunkSize; var messageType = await webSocket.GetWebsocketBuffer(incommingBuffer, cancellationToken, maxLength); if (!cancellationToken.IsCancellationRequested) { @@ -261,8 +229,8 @@ public async Task Listen() //prevent recursive error sending if (!IsAuditor && !(envelope == null || envelope.Message is ResultMessage)) - _ = SendMessage(envelope.CreateResult(statusCode).CreateEnvelope()); - if (statusCode == ResultStatusCode.InternalError || !Context.IsAlpha) + await SendMessage(envelope.CreateResult(statusCode).CreateEnvelope()); + if (statusCode == ResultStatusCode.InternalError || !Context.NodesManager.IsAlpha) logger.Error(exc); } } @@ -291,7 +259,7 @@ public async Task Listen() } finally { - ConnectionState = ConnectionState.Closed; + OnClosed?.Invoke(this); } } diff --git a/Centaurus.Domain/WebSockets/Centaurus/INodeConnection.cs b/Centaurus.Domain/WebSockets/Centaurus/INodeConnection.cs index d2758b77..686cf11f 100644 --- a/Centaurus.Domain/WebSockets/Centaurus/INodeConnection.cs +++ b/Centaurus.Domain/WebSockets/Centaurus/INodeConnection.cs @@ -1,5 +1,4 @@ -using Centaurus.Domain.StateManagers; -using Centaurus.Models; +using Centaurus.Models; namespace Centaurus.Domain { diff --git a/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingConnectionBase.cs b/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingConnectionBase.cs index 6697e66b..b35982f9 100644 --- a/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingConnectionBase.cs +++ b/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingConnectionBase.cs @@ -23,7 +23,7 @@ protected void SendHandshake() { await SendMessage(new HandshakeRequest { HandshakeData = handshakeData }); await Task.Delay(5000); //wait for 5 sec to validate connection - if (ConnectionState == ConnectionState.Connected) + if (!IsAuthenticated) await CloseConnection(WebSocketCloseStatus.ProtocolError, "Handshake response wasn't send."); }); } @@ -32,11 +32,10 @@ protected void SendHandshake() public bool TryValidate(HandshakeData handshakeData) { - if (handshakeData == null - || !handshakeData.Equals(this.handshakeData)) + if (!this.handshakeData.Equals(handshakeData)) return false; - ConnectionState = ConnectionState.Ready; + Authenticated(); return true; } } diff --git a/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingConnectionManager.cs b/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingConnectionManager.cs index 4e07a05f..55ce783a 100644 --- a/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingConnectionManager.cs +++ b/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingConnectionManager.cs @@ -1,15 +1,11 @@ -using Centaurus.Domain; +using Centaurus.Domain.WebSockets; using Centaurus.Models; using NLog; using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Net.WebSockets; -using System.Threading.Tasks; using System.Linq; -using Centaurus.Domain.WebSockets; -using Centaurus.Domain.StateManagers; +using System.Net.WebSockets; using System.Threading; +using System.Threading.Tasks; namespace Centaurus.Domain { @@ -69,37 +65,10 @@ internal async Task CloseAllConnections(bool includingAuditors = true) try { //skip if auditor - if (!includingAuditors && Context.Constellation.Auditors.Any(a => a.PubKey.Equals(pk)) + if (!includingAuditors && Context.NodesManager.IsNode(pk) || !connections.TryRemove(pk, out var connection)) continue; await RemoveConnection(connection); - - } - catch (Exception e) - { - logger.Error(e, "Unable to close connection"); - } - } - - internal void CleanupAuditorConnections() - { - var irrelevantAuditors = Context.NodesManager.GetRemoteNodes() - .Where(ca => !Context.Constellation.Auditors.Any(a => a.PubKey.Equals(ca))) - .Select(ca => ca.PubKey) - .ToList(); - - foreach (var pk in irrelevantAuditors) - try - { - if (!connections.TryRemove(pk, out var connection)) - continue; - UnsubscribeAndClose(connection) - .ContinueWith(t => - { - //we need to drop it, to prevent sending private constellation data - if (t.IsFaulted) - Context.NodesManager.CurrentNode.Failed(new Exception("Unable to drop irrelevant auditor connection.")); - }); } catch (Exception e) { @@ -113,17 +82,28 @@ internal void CleanupAuditorConnections() private static State[] ValidStates = new State[] { State.Rising, State.Running, State.Ready }; - void Subscribe(IncomingConnectionBase connection) + private void Subscribe(IncomingConnectionBase connection) + { + connection.OnAuthenticated += Connection_OnAuthenticated; + connection.OnClosed += Connection_OnClosed; + } + + private void Unsubscribe(IncomingConnectionBase connection) + { + connection.OnAuthenticated -= Connection_OnAuthenticated; + connection.OnClosed -= Connection_OnClosed; + } + private void Connection_OnAuthenticated(ConnectionBase connection) { - connection.OnConnectionStateChanged += OnConnectionStateChanged; + AddConnection((IncomingConnectionBase)connection); } - void Unsubscribe(IncomingConnectionBase connection) + private void Connection_OnClosed(ConnectionBase connection) { - connection.OnConnectionStateChanged -= OnConnectionStateChanged; + _ = RemoveConnection((IncomingConnectionBase)connection); } - async Task UnsubscribeAndClose(IncomingConnectionBase connection) + private async Task UnsubscribeAndClose(IncomingConnectionBase connection) { Unsubscribe(connection); await connection.CloseConnection(); @@ -136,35 +116,18 @@ void AddConnection(IncomingConnectionBase connection) logger.Trace($"{connection.PubKey} is connected."); } - void OnConnectionStateChanged((ConnectionBase connection, ConnectionState prev, ConnectionState current) args) - { - var connection = (IncomingConnectionBase)args.connection; - switch (args.current) - { - case ConnectionState.Ready: - //avoid multiple validation event firing - if (args.prev == ConnectionState.Connected) - AddConnection(connection); - break; - case ConnectionState.Closed: - //if prev connection wasn't validated than the validated connection could be dropped - if (args.prev == ConnectionState.Ready) - _ = RemoveConnection(connection); - break; - default: - break; - } - } - async Task RemoveConnection(IncomingConnectionBase connection) { await UnsubscribeAndClose(connection); - connections.TryRemove(connection); - if (connection.IsAuditor) + if (connection.IsAuthenticated) { - var nodeConnection = (IncomingNodeConnection)connection; - nodeConnection.Node.RemoveIncomingConnection(); + connections.TryRemove(connection); + if (connection.IsAuditor) + { + var nodeConnection = (IncomingNodeConnection)connection; + nodeConnection.Node.RemoveIncomingConnection(); + } } } diff --git a/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingNodeConnection.cs b/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingNodeConnection.cs index 676a0e0c..ad778e73 100644 --- a/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingNodeConnection.cs +++ b/Centaurus.Domain/WebSockets/Centaurus/Incoming/IncomingNodeConnection.cs @@ -1,5 +1,4 @@ using Centaurus.Domain; -using Centaurus.Domain.StateManagers; using System; using System.Net.WebSockets; diff --git a/Centaurus.Domain/WebSockets/Centaurus/Notifier.cs b/Centaurus.Domain/WebSockets/Centaurus/Notifier.cs index a07fc480..7d896b3f 100644 --- a/Centaurus.Domain/WebSockets/Centaurus/Notifier.cs +++ b/Centaurus.Domain/WebSockets/Centaurus/Notifier.cs @@ -20,10 +20,10 @@ public static void Notify(this ExecutionContext context, RawPubKey account, Mess { if (context == null) throw new ArgumentNullException(nameof(context)); - context.ExtensionsManager.BeforeNotify(account, envelope); if (context.IncomingConnectionManager.TryGetConnection(account, out IncomingConnectionBase connection)) try { + context.ExtensionsManager.BeforeNotify(account, envelope); _ = connection.SendMessage(envelope); } catch { } diff --git a/Centaurus.Domain/WebSockets/Centaurus/Outgoing/OutgoingConnection.cs b/Centaurus.Domain/WebSockets/Centaurus/Outgoing/OutgoingConnection.cs index 7ee5dfe9..ba3eb952 100644 --- a/Centaurus.Domain/WebSockets/Centaurus/Outgoing/OutgoingConnection.cs +++ b/Centaurus.Domain/WebSockets/Centaurus/Outgoing/OutgoingConnection.cs @@ -1,12 +1,6 @@ -using Centaurus.Client; -using Centaurus.Domain.StateManagers; -using Centaurus.Models; using NLog; using System; -using System.Linq; -using System.Threading; using System.Threading.Tasks; -using static Centaurus.Domain.StateNotifierWorker; namespace Centaurus.Domain { @@ -14,16 +8,12 @@ internal class OutgoingConnection : ConnectionBase, INodeConnection { static Logger logger = LogManager.GetCurrentClassLogger(); private readonly OutgoingConnectionWrapperBase connection; - private readonly OutgoingMessageStorage outgoingMessageStorage; - public OutgoingConnection(ExecutionContext context, RemoteNode node, OutgoingMessageStorage outgoingMessageStorage, OutgoingConnectionWrapperBase connection) + public OutgoingConnection(ExecutionContext context, RemoteNode node, OutgoingConnectionWrapperBase connection) : base(context, node?.PubKey, connection.WebSocket) { this.connection = connection ?? throw new ArgumentNullException(nameof(connection)); - this.outgoingMessageStorage = outgoingMessageStorage ?? throw new ArgumentNullException(nameof(outgoingMessageStorage)); - Node = node; - //we know that we connect to auditor so we can set connection state to validated immediately - ConnectionState = ConnectionState.Ready; + Node = node ?? throw new ArgumentNullException(nameof(node)); } const int BufferSize = 50 * 1024 * 1024; @@ -33,52 +23,16 @@ public OutgoingConnection(ExecutionContext context, RemoteNode node, OutgoingMes public RemoteNode Node { get; } - public override bool IsAuditor => throw new NotImplementedException(); + public override bool IsAuditor => true; public async Task EstablishConnection(Uri uri) { await connection.Connect(uri, cancellationToken); - ProcessOutgoingMessageQueue(); } - - Task sendingMessageTask; - - private void ProcessOutgoingMessageQueue() + public void HandshakeDataSend() { - sendingMessageTask = Task.Factory.StartNew(async () => - { - while (!cancellationToken.IsCancellationRequested) - { - try - { - if (!cancellationToken.IsCancellationRequested - && Node.IsRunning() - && outgoingMessageStorage.TryPeek(out MessageEnvelopeBase message) - ) - { - try - { - await base.SendMessage(message); - if (!outgoingMessageStorage.TryDequeue(out message)) - logger.Error("Unable to dequeue"); - } - catch (Exception exc) - { - logger.Error(exc); - } - } - else - { - Thread.Sleep(50); - } - } - catch (Exception e) - { - logger.Error(e); - } - } - }, cancellationToken).Unwrap(); + Authenticated(); } public override void Dispose() diff --git a/Centaurus.Domain/WebSockets/Centaurus/Outgoing/OutgoingMessageStorage.cs b/Centaurus.Domain/WebSockets/Centaurus/Outgoing/OutgoingMessageStorage.cs deleted file mode 100644 index 8f20200a..00000000 --- a/Centaurus.Domain/WebSockets/Centaurus/Outgoing/OutgoingMessageStorage.cs +++ /dev/null @@ -1,120 +0,0 @@ -using Centaurus.Models; -using NLog; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; - -namespace Centaurus.Domain -{ - public class OutgoingMessageStorage - { - private readonly Queue outgoingMessages = new Queue(); - - public void EnqueueMessage(Message message) - { - if (message == null) - throw new ArgumentNullException(nameof(message)); - EnqueueMessage(message.CreateEnvelope()); - } - - public void EnqueueMessage(MessageEnvelopeBase message) - { - if (message == null) - throw new ArgumentNullException(nameof(message)); - lock (outgoingMessages) - outgoingMessages.Enqueue(message); - } - - public bool TryPeek(out MessageEnvelopeBase message) - { - lock (outgoingMessages) - return outgoingMessages.TryPeek(out message); - } - - public bool TryDequeue(out MessageEnvelopeBase message) - { - lock (outgoingMessages) - return outgoingMessages.TryDequeue(out message); - } - } - - public class OutgoingResultsStorage : IDisposable - { - const int MaxMessageBatchSize = 500; - - static Logger logger = LogManager.GetCurrentClassLogger(); - readonly OutgoingMessageStorage outgoingMessageStorage; - readonly List results = new List(); - object syncRoot = new { }; - - public OutgoingResultsStorage(OutgoingMessageStorage outgoingMessageStorage) - { - this.outgoingMessageStorage = outgoingMessageStorage ?? throw new ArgumentNullException(nameof(outgoingMessageStorage)); - Run(); - } - - private CancellationTokenSource cts = new CancellationTokenSource(); - - private void Run() - { - Task.Factory.StartNew(() => - { - while (!cts.Token.IsCancellationRequested) - { - try - { - var resultsBatch = default(List); - lock (syncRoot) - { - if (results.Count != 0) - { - resultsBatch = results.Take(MaxMessageBatchSize).ToList(); - var removeCount = Math.Min(MaxMessageBatchSize, results.Count); - results.RemoveRange(0, removeCount); - - logger.Trace($"Results batch sent. Batch size: {resultsBatch.Count}, apexes: {resultsBatch[0].Apex}-{resultsBatch[resultsBatch.Count - 1].Apex}."); - } - } - if (resultsBatch != default) - { - outgoingMessageStorage.EnqueueMessage( - new AuditorSignaturesBatch - { - AuditorResultMessages = resultsBatch - }.CreateEnvelope()); - } - else - { - Thread.Sleep(50); - } - } - catch (Exception exc) - { - logger.Error(exc); - } - } - }, cts.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default); - } - - /// - /// - /// - /// - /// Buffer to use for serialization - public void EnqueueResult(AuditorResult result) - { - if (result == null) - throw new ArgumentNullException(nameof(result)); - - lock (syncRoot) - results.Add(result); - } - - public void Dispose() - { - cts.Dispose(); - } - } -} \ No newline at end of file diff --git a/Centaurus.Domain/WebSockets/Info/CommandHandlers/GetPriceHistoryCommandHandler.cs b/Centaurus.Domain/WebSockets/Info/CommandHandlers/GetPriceHistoryCommandHandler.cs index fb164de8..e4e2de05 100644 --- a/Centaurus.Domain/WebSockets/Info/CommandHandlers/GetPriceHistoryCommandHandler.cs +++ b/Centaurus.Domain/WebSockets/Info/CommandHandlers/GetPriceHistoryCommandHandler.cs @@ -17,8 +17,8 @@ public GetPriceHistoryCommandHandler(ExecutionContext context) public override BaseResponse Handle(InfoWebSocketConnection infoWebSocket, GetPriceHistoryCommand command) { - var asset = Context.Constellation.Assets.FirstOrDefault(a => a.Code == command.Market); - if (asset == null || asset.Code == Context.Constellation.QuoteAsset.Code) + var asset = Context.ConstellationSettingsManager.Current.Assets.FirstOrDefault(a => a.Code == command.Market); + if (asset == null || asset.Code == Context.ConstellationSettingsManager.Current.QuoteAsset.Code) throw new BadRequestException("Invalid market."); var res = Context.AnalyticsManager.PriceHistoryManager.GetPriceHistory(command.Cursor, command.Market, command.Period); diff --git a/Centaurus.Domain/exchange/Exchange.cs b/Centaurus.Domain/exchange/Exchange.cs index 79fa3747..d593bb71 100644 --- a/Centaurus.Domain/exchange/Exchange.cs +++ b/Centaurus.Domain/exchange/Exchange.cs @@ -61,7 +61,7 @@ public void ExecuteOrder(string asset, string baseAsset, QuantumProcessingItem q public void RemoveOrder(QuantumProcessingItem quantumProcessingItem, string baseAsset) { - var request = (RequestQuantumBase)quantumProcessingItem.Quantum; + var request = (ClientRequestQuantumBase)quantumProcessingItem.Quantum; var orderCancellation = (OrderCancellationRequest)request.RequestMessage; var orderWrapper = OrderMap.GetOrder(orderCancellation.OrderId); var orderbook = GetOrderbook(orderWrapper.Order.Asset, orderWrapper.Order.Side); diff --git a/Centaurus.Domain/exchange/OrderMatcher.cs b/Centaurus.Domain/exchange/OrderMatcher.cs index 661d873c..b78a00fb 100644 --- a/Centaurus.Domain/exchange/OrderMatcher.cs +++ b/Centaurus.Domain/exchange/OrderMatcher.cs @@ -12,7 +12,7 @@ public OrderMatcher(ExchangeMarket market, string baseAsset, QuantumProcessingIt this.quantumProcessingItem = quantumProcessingItem ?? throw new ArgumentNullException(nameof(quantumProcessingItem)); this.baseAsset = baseAsset ?? throw new ArgumentNullException(nameof(baseAsset)); this.market = market ?? throw new ArgumentNullException(nameof(market)); - var orderRequest = (OrderRequest)((RequestQuantumBase)quantumProcessingItem.Quantum).RequestMessage; + var orderRequest = (OrderRequest)((ClientRequestQuantumBase)quantumProcessingItem.Quantum).RequestMessage; takerOrder = new OrderWrapper( new Order diff --git a/Centaurus.Domain/helpers/MajorityHelper.cs b/Centaurus.Domain/helpers/MajorityHelper.cs index 3b67d6b1..6815f6d4 100644 --- a/Centaurus.Domain/helpers/MajorityHelper.cs +++ b/Centaurus.Domain/helpers/MajorityHelper.cs @@ -21,7 +21,7 @@ public static int GetTotalAuditorsCount(this ExecutionContext context) { if (context == null) throw new ArgumentNullException(nameof(context)); - return context.Constellation.Auditors.Count; + return context.NodesManager.AllNodes.Count; } public static int GetMajorityCount(int totalAuditorsCount) @@ -43,9 +43,6 @@ public static bool HasMajority(this ExecutionContext context, int auditorsCount, if (context == null) throw new ArgumentNullException(nameof(context)); - if (context.Constellation == null) - return false; - if (!isCurrentIncluded) //+1 is current auditor auditorsCount++; diff --git a/Centaurus.Models.Extensions/MessageEnvelopeExtenstions.cs b/Centaurus.Models.Extensions/MessageEnvelopeExtenstions.cs index 5beb8d29..2a718435 100644 --- a/Centaurus.Models.Extensions/MessageEnvelopeExtenstions.cs +++ b/Centaurus.Models.Extensions/MessageEnvelopeExtenstions.cs @@ -114,7 +114,7 @@ private static TResultMessage CreateResult(this MessageEnvelopeB throw new ArgumentNullException(nameof(envelope)); var resultMessage = Activator.CreateInstance(); var messageId = envelope.Message.MessageId; - if (envelope.Message is RequestQuantumBase request) + if (envelope.Message is ClientRequestQuantumBase request) messageId = request.RequestMessage.MessageId; resultMessage.OriginalMessageId = messageId; resultMessage.Status = status; @@ -127,8 +127,8 @@ public static ResultMessageBase CreateResult(this MessageEnvelopeBase envelope, throw new ArgumentNullException(nameof(envelope)); var messageType = envelope.Message; - if (envelope.Message is RequestQuantumBase) - messageType = ((RequestQuantumBase)envelope.Message).RequestEnvelope.Message; + if (envelope.Message is ClientRequestQuantumBase) + messageType = ((ClientRequestQuantumBase)envelope.Message).RequestEnvelope.Message; //for not Success result return generic message if (status == ResultStatusCode.Success) diff --git a/Centaurus.Models/Common/AuditorSignatureBase.cs b/Centaurus.Models/Common/AuditorSignatureBase.cs index e23e40ed..a2d6c404 100644 --- a/Centaurus.Models/Common/AuditorSignatureBase.cs +++ b/Centaurus.Models/Common/AuditorSignatureBase.cs @@ -11,7 +11,7 @@ namespace Centaurus.Models public class AuditorSignatureBase { [XdrField(0)] - public int AuditorId { get; set; } + public int NodeId { get; set; } [XdrField(1)] public TinySignature PayloadSignature { get; set; } diff --git a/Centaurus.Models/InternalMessages/ConstellationQuantum.cs b/Centaurus.Models/InternalMessages/ConstellationQuantum.cs index 6d85a00a..d7a306c4 100644 --- a/Centaurus.Models/InternalMessages/ConstellationQuantum.cs +++ b/Centaurus.Models/InternalMessages/ConstellationQuantum.cs @@ -5,11 +5,8 @@ namespace Centaurus.Models { - public class ConstellationQuantum: Quantum + public class ConstellationQuantum: RequestQuantumBase { - [XdrField(0)] - public ConstellationMessageEnvelope RequestEnvelope { get; set; } - public Message RequestMessage => RequestEnvelope.Message; } } \ No newline at end of file diff --git a/Centaurus.Models/InternalMessages/ConstellationRequestMessage.cs b/Centaurus.Models/InternalMessages/ConstellationRequestMessage.cs index dad12898..75897b15 100644 --- a/Centaurus.Models/InternalMessages/ConstellationRequestMessage.cs +++ b/Centaurus.Models/InternalMessages/ConstellationRequestMessage.cs @@ -1,4 +1,5 @@ -using System; +using Centaurus.Xdr; +using System; using System.Collections.Generic; using System.Text; @@ -6,5 +7,7 @@ namespace Centaurus.Models { public abstract class ConstellationRequestMessage: Message { + [XdrField(0)] + public ulong LastUpdateApex { get; set; } } } diff --git a/Centaurus.Models/InternalMessages/QuantumMajoritySignaturesBatch.cs b/Centaurus.Models/InternalMessages/MajoritySignaturesBatch.cs similarity index 57% rename from Centaurus.Models/InternalMessages/QuantumMajoritySignaturesBatch.cs rename to Centaurus.Models/InternalMessages/MajoritySignaturesBatch.cs index ec586e39..506c1609 100644 --- a/Centaurus.Models/InternalMessages/QuantumMajoritySignaturesBatch.cs +++ b/Centaurus.Models/InternalMessages/MajoritySignaturesBatch.cs @@ -6,21 +6,18 @@ namespace Centaurus.Models { [XdrContract] - public class QuantumMajoritySignaturesBatch : Message + public class MajoritySignaturesBatch : Message { [XdrField(0)] - public List Signatures { get; set; } + public List Items { get; set; } } [XdrContract] - public class QuantumSignatures: IApex + public class MajoritySignaturesBatchItem: IApex { [XdrField(0)] public ulong Apex { get; set; } - /// - /// Contains all quantum's signatures except Alpha - /// [XdrField(1)] public List Signatures { get; set; } } diff --git a/Centaurus.Models/Results/AuditorResult.cs b/Centaurus.Models/InternalMessages/SingleNodeSignaturesBatch.cs similarity index 58% rename from Centaurus.Models/Results/AuditorResult.cs rename to Centaurus.Models/InternalMessages/SingleNodeSignaturesBatch.cs index c1885611..1e7d390f 100644 --- a/Centaurus.Models/Results/AuditorResult.cs +++ b/Centaurus.Models/InternalMessages/SingleNodeSignaturesBatch.cs @@ -5,8 +5,15 @@ namespace Centaurus.Models { + public class SingleNodeSignaturesBatch : Message + { + [XdrField(0)] + public List Items { get; set; } + } + + [XdrContract] - public class AuditorResult + public class SingleNodeSignaturesBatchItem: IApex { [XdrField(0)] public ulong Apex { get; set; } @@ -14,4 +21,4 @@ public class AuditorResult [XdrField(1)] public NodeSignatureInternal Signature { get; set; } } -} +} \ No newline at end of file diff --git a/Centaurus.Models/InternalMessages/StateMessage.cs b/Centaurus.Models/InternalMessages/StateMessage.cs index 9c9a5ec9..9de8a41a 100644 --- a/Centaurus.Models/InternalMessages/StateMessage.cs +++ b/Centaurus.Models/InternalMessages/StateMessage.cs @@ -18,6 +18,6 @@ public class StateMessage : Message public State State { get; set; } [XdrField(4)] - public DateTime UpdateDate { get; set; } + public long UpdateDate { get; set; } } } \ No newline at end of file diff --git a/Centaurus.Models/InternalMessages/SyncCursor.cs b/Centaurus.Models/InternalMessages/SyncCursor.cs index eb32c525..fa446d3f 100644 --- a/Centaurus.Models/InternalMessages/SyncCursor.cs +++ b/Centaurus.Models/InternalMessages/SyncCursor.cs @@ -5,7 +5,6 @@ namespace Centaurus.Models { - //TODO: convert ulong to ulong? after migration to MessagePack [XdrContract] public class SyncCursor { @@ -25,6 +24,7 @@ public class SyncCursor public enum XdrSyncCursorType { Quanta = 0, - Signatures = 1 + MajoritySignatures = 1, + SingleNodeSignatures = 2 } } diff --git a/Centaurus.Models/InternalMessages/SyncQuantaBatch.cs b/Centaurus.Models/InternalMessages/SyncQuantaBatch.cs index 1f3edb8e..9d471512 100644 --- a/Centaurus.Models/InternalMessages/SyncQuantaBatch.cs +++ b/Centaurus.Models/InternalMessages/SyncQuantaBatch.cs @@ -19,9 +19,6 @@ public class SyncQuantaBatchItem : IApex [XdrField(0)] public Message Quantum { get; set; } - [XdrField(1)] - public NodeSignatureInternal AlphaSignature { get; set; } - public ulong Apex => ((Quantum)Quantum).Apex; } } diff --git a/Centaurus.Models/Messages/Message.cs b/Centaurus.Models/Messages/Message.cs index 8994894a..6a5d2c12 100644 --- a/Centaurus.Models/Messages/Message.cs +++ b/Centaurus.Models/Messages/Message.cs @@ -16,16 +16,16 @@ namespace Centaurus.Models [XdrUnion((int)MessageTypes.PaymentRequest, typeof(PaymentRequest))] [XdrUnion((int)MessageTypes.OrderRequest, typeof(OrderRequest))] [XdrUnion((int)MessageTypes.OrderCancellationRequest, typeof(OrderCancellationRequest))] - [XdrUnion((int)MessageTypes.RequestQuantum, typeof(RequestQuantum))] + [XdrUnion((int)MessageTypes.RequestQuantum, typeof(ClientRequestQuantum))] [XdrUnion((int)MessageTypes.WithdrawalRequestQuantum, typeof(WithdrawalRequestQuantum))] [XdrUnion((int)MessageTypes.DepositQuantum, typeof(DepositQuantum))] [XdrUnion((int)MessageTypes.StateUpdate, typeof(StateMessage))] - [XdrUnion((int)MessageTypes.AuditorSignaturesBatch, typeof(AuditorSignaturesBatch))] + [XdrUnion((int)MessageTypes.SingleNodeSignaturesBatch, typeof(SingleNodeSignaturesBatch))] [XdrUnion((int)MessageTypes.AlphaUpdate, typeof(AlphaUpdate))] [XdrUnion((int)MessageTypes.ConstellationUpdate, typeof(ConstellationUpdate))] [XdrUnion((int)MessageTypes.ConstellationQuantum, typeof(ConstellationQuantum))] [XdrUnion((int)MessageTypes.SyncCursorReset, typeof(SyncCursorReset))] - [XdrUnion((int)MessageTypes.QuantumMajoritySignaturesBatch, typeof(QuantumMajoritySignaturesBatch))] + [XdrUnion((int)MessageTypes.MajoritySignaturesBatch, typeof(MajoritySignaturesBatch))] [XdrUnion((int)MessageTypes.SyncQuantaBatch, typeof(SyncQuantaBatch))] [XdrUnion((int)MessageTypes.RequestQuantaBatch, typeof(RequestQuantaBatch))] [XdrUnion((int)MessageTypes.CatchupQuantaBatchRequest, typeof(CatchupQuantaBatchRequest))] diff --git a/Centaurus.Models/Messages/MessageTypes.cs b/Centaurus.Models/Messages/MessageTypes.cs index da59a596..1e1ec5b3 100644 --- a/Centaurus.Models/Messages/MessageTypes.cs +++ b/Centaurus.Models/Messages/MessageTypes.cs @@ -122,11 +122,11 @@ public enum MessageTypes /// /// Contains majority of signatures for quanta (except Alpha signatures, it will be send with quantum) /// - QuantumMajoritySignaturesBatch = 215, + MajoritySignaturesBatch = 215, /// /// Contains an array of the current auditor's quantum signatures /// - AuditorSignaturesBatch = 216, + SingleNodeSignaturesBatch = 216, /// /// Contains an array of the client requests that must be resent to Alpha /// diff --git a/Centaurus.Models/Messages/RequestQuantum.cs b/Centaurus.Models/Messages/RequestQuantum.cs new file mode 100644 index 00000000..fdc981bb --- /dev/null +++ b/Centaurus.Models/Messages/RequestQuantum.cs @@ -0,0 +1,16 @@ +using Centaurus.Xdr; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Centaurus.Models +{ + public class RequestQuantumBase: Quantum + { + /// + /// Contains original operation request. + /// + [XdrField(0)] + public MessageEnvelopeBase RequestEnvelope { get; set; } + } +} diff --git a/Centaurus.Models/Requests/AccountDataRequestQuantum.cs b/Centaurus.Models/Requests/AccountDataRequestQuantum.cs index 5d5efb3d..8c994aef 100644 --- a/Centaurus.Models/Requests/AccountDataRequestQuantum.cs +++ b/Centaurus.Models/Requests/AccountDataRequestQuantum.cs @@ -5,7 +5,7 @@ namespace Centaurus.Models { - public class AccountDataRequestQuantum : RequestQuantumBase + public class AccountDataRequestQuantum : ClientRequestQuantumBase { [XdrField(0)] public byte[] PayloadHash { get; set; } diff --git a/Centaurus.Models/Requests/ClientRequestQuantumBase.cs b/Centaurus.Models/Requests/ClientRequestQuantumBase.cs new file mode 100644 index 00000000..de2ec9fb --- /dev/null +++ b/Centaurus.Models/Requests/ClientRequestQuantumBase.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; +using Centaurus.Xdr; + +namespace Centaurus.Models +{ + public abstract class ClientRequestQuantumBase : RequestQuantumBase + { + public SequentialRequestMessage RequestMessage => (SequentialRequestMessage)RequestEnvelope.Message; + } +} \ No newline at end of file diff --git a/Centaurus.Models/Requests/RequestQuantum.cs b/Centaurus.Models/Requests/RequestQuantum.cs index abbc91b9..806c6927 100644 --- a/Centaurus.Models/Requests/RequestQuantum.cs +++ b/Centaurus.Models/Requests/RequestQuantum.cs @@ -4,7 +4,7 @@ namespace Centaurus.Models { - public class RequestQuantum : RequestQuantumBase + public class ClientRequestQuantum : ClientRequestQuantumBase { } } diff --git a/Centaurus.Models/Requests/RequestQuantumBase.cs b/Centaurus.Models/Requests/RequestQuantumBase.cs deleted file mode 100644 index 0f34f82a..00000000 --- a/Centaurus.Models/Requests/RequestQuantumBase.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System; -using System.Collections.Generic; -using Centaurus.Xdr; - -namespace Centaurus.Models -{ - public abstract class RequestQuantumBase : Quantum - { - /// - /// Contains original operation request received from a client. - /// - [XdrField(0)] - public MessageEnvelopeBase RequestEnvelope { get; set; } - - public SequentialRequestMessage RequestMessage => (SequentialRequestMessage)RequestEnvelope.Message; - } -} \ No newline at end of file diff --git a/Centaurus.Models/Requests/WithdrawalRequestQuantum.cs b/Centaurus.Models/Requests/WithdrawalRequestQuantum.cs index 84dfc82c..dc738c5c 100644 --- a/Centaurus.Models/Requests/WithdrawalRequestQuantum.cs +++ b/Centaurus.Models/Requests/WithdrawalRequestQuantum.cs @@ -5,7 +5,7 @@ namespace Centaurus.Models /// /// Transaction must be build on Alpha and shared with the constellation. Each auditor can build different transactions. /// - public class WithdrawalRequestQuantum: RequestQuantumBase + public class WithdrawalRequestQuantum: ClientRequestQuantumBase { [XdrField(0)] public byte[] Transaction { get; set; } diff --git a/Centaurus.Models/Results/AuditorSignaturesBatch.cs b/Centaurus.Models/Results/AuditorSignaturesBatch.cs deleted file mode 100644 index 5b6c31a5..00000000 --- a/Centaurus.Models/Results/AuditorSignaturesBatch.cs +++ /dev/null @@ -1,13 +0,0 @@ -using Centaurus.Xdr; -using System; -using System.Collections.Generic; -using System.Text; - -namespace Centaurus.Models -{ - public class AuditorSignaturesBatch : Message - { - [XdrField(0)] - public List AuditorResultMessages { get; set; } - } -} \ No newline at end of file diff --git a/Centaurus.PersistentStorage.Abstraction/Query/StateQuery.cs b/Centaurus.PersistentStorage.Abstraction/Query/StateQuery.cs index f574c4b8..568ffc2d 100644 --- a/Centaurus.PersistentStorage.Abstraction/Query/StateQuery.cs +++ b/Centaurus.PersistentStorage.Abstraction/Query/StateQuery.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; namespace Centaurus.PersistentStorage { @@ -21,7 +22,7 @@ public AccountPersistentModel LoadAccount(byte[] accountPubkey) public SettingsPersistentModel LoadSettings(ulong fromApex) { - return storage.Find(QueryOrder.Desc).From(UlongConverter.Encode(fromApex)).First(); + return storage.Find(QueryOrder.Desc).From(UlongConverter.Encode(fromApex)).FirstOrDefault(); } public CursorsPersistentModel LoadCursors() diff --git a/Centaurus.PersistentStorage/Storage/StorageIterator.cs b/Centaurus.PersistentStorage/Storage/StorageIterator.cs index 93c31a52..992816d1 100644 --- a/Centaurus.PersistentStorage/Storage/StorageIterator.cs +++ b/Centaurus.PersistentStorage/Storage/StorageIterator.cs @@ -75,6 +75,16 @@ public T First() return res; } + public T FirstOrDefault() + { + var value = iterator.Value(); + if (value == null || value.Length < 1) + return default(T); + var res = ParseCurrent(iterator.Key()); + Next(); + return res; + } + private StorageIterator SetBoundaryCheck(Func checkKeyIsWithinBoundaries) { isKeyWithinBoundaries = checkKeyIsWithinBoundaries; diff --git a/Centaurus.Test.Domain/AlphaMessageHandlersTests.cs b/Centaurus.Test.Domain/AlphaMessageHandlersTests.cs index 1de67ba4..2318057c 100644 --- a/Centaurus.Test.Domain/AlphaMessageHandlersTests.cs +++ b/Centaurus.Test.Domain/AlphaMessageHandlersTests.cs @@ -59,7 +59,7 @@ public async Task HandshakeTest(KeyPair clientKeyPair, State alphaState, Type ex var isHandled = await context.MessageHandlers.HandleMessage(connection, inMessage); Assert.IsTrue(isHandled); - Assert.AreEqual(connection.ConnectionState, ConnectionState.Ready); + Assert.IsTrue(connection.IsAuthenticated); } else Assert.ThrowsAsync(expectedException, async () => await context.MessageHandlers.HandleMessage(connection, inMessage)); @@ -86,17 +86,17 @@ public void HandshakeInvalidDataTest() static object[] QuantaBatchTestCases = { - new object[] { TestEnvironment.Client1KeyPair, ConnectionState.Ready, typeof(UnauthorizedException) }, - new object[] { TestEnvironment.Auditor1KeyPair, ConnectionState.Ready, null } + new object[] { TestEnvironment.Client1KeyPair, true, typeof(UnauthorizedException) }, + new object[] { TestEnvironment.Auditor1KeyPair, true, null } }; [Test] [TestCaseSource(nameof(QuantaBatchTestCases))] - public async Task CatchupQuantaBatchTest(KeyPair clientKeyPair, ConnectionState state, Type excpectedException) + public async Task CatchupQuantaBatchTest(KeyPair clientKeyPair, bool isAuthenticated, Type excpectedException) { context.SetState(State.Rising); - var clientConnection = GetIncomingConnection(context, clientKeyPair.PublicKey, state); + var clientConnection = GetIncomingConnection(context, clientKeyPair.PublicKey, isAuthenticated); var envelope = new CatchupQuantaBatch { @@ -112,19 +112,19 @@ public async Task CatchupQuantaBatchTest(KeyPair clientKeyPair, ConnectionState static object[] QuantumMajoritySignaturesBatchCases = { - new object[] { TestEnvironment.Client1KeyPair, ConnectionState.Ready, typeof(UnauthorizedException) }, - new object[] { TestEnvironment.Auditor1KeyPair, ConnectionState.Connected, typeof(InvalidStateException) }, - new object[] { TestEnvironment.Auditor1KeyPair, ConnectionState.Ready, null } + new object[] { TestEnvironment.Client1KeyPair, true, typeof(UnauthorizedException) }, + new object[] { TestEnvironment.Auditor1KeyPair, false, typeof(UnauthorizedException) }, + new object[] { TestEnvironment.Auditor1KeyPair, true, null } }; - public async Task QuantumMajoritySignaturesBatchTest(KeyPair clientKeyPair, ConnectionState state, Type excpectedException) + public async Task QuantumMajoritySignaturesBatchTest(KeyPair clientKeyPair, bool isAuthenticated, Type excpectedException) { context.SetState(State.Ready); - var clientConnection = GetIncomingConnection(context, clientKeyPair.PublicKey, state); + var clientConnection = GetIncomingConnection(context, clientKeyPair.PublicKey, isAuthenticated); - var envelope = new QuantumMajoritySignaturesBatch + var envelope = new MajoritySignaturesBatch { - Signatures = new List() + Items = new List() }.CreateEnvelope(); envelope.Sign(clientKeyPair); @@ -138,17 +138,17 @@ public async Task QuantumMajoritySignaturesBatchTest(KeyPair clientKeyPair, Conn static object[] OrderTestCases = { - new object[] { ConnectionState.Connected, typeof(InvalidStateException) }, - new object[] { ConnectionState.Ready, null } + new object[] { false, typeof(UnauthorizedException) }, + new object[] { true, null } }; [Test] [TestCaseSource(nameof(OrderTestCases))] - public async Task OrderTest(ConnectionState state, Type excpectedException) + public async Task OrderTest(bool isAuthenticated, Type excpectedException) { context.SetState(State.Ready); - var clientConnection = GetIncomingConnection(context, TestEnvironment.Client1KeyPair.PublicKey, state); + var clientConnection = GetIncomingConnection(context, TestEnvironment.Client1KeyPair.PublicKey, isAuthenticated); var account = context.AccountStorage.GetAccount(TestEnvironment.Client1KeyPair); @@ -157,7 +157,7 @@ public async Task OrderTest(ConnectionState state, Type excpectedException) Account = account.Pubkey, RequestId = DateTime.UtcNow.Ticks, Amount = 100, - Asset = context.Constellation.Assets.First(a => !a.IsQuoteAsset).Code, + Asset = context.ConstellationSettingsManager.Current.Assets.First(a => !a.IsQuoteAsset).Code, Price = 1, Side = OrderSide.Buy }.CreateEnvelope(); @@ -170,17 +170,17 @@ public async Task OrderTest(ConnectionState state, Type excpectedException) static object[] AccountDataTestRequestCases = { - new object[] { ConnectionState.Connected, typeof(InvalidStateException) }, - new object[] { ConnectionState.Ready, null } + new object[] { false, typeof(UnauthorizedException) }, + new object[] {true, null } }; [Test] [TestCaseSource(nameof(AccountDataTestRequestCases))] - public async Task AccountDataRequestTest(ConnectionState state, Type excpectedException) + public async Task AccountDataRequestTest(bool isAuthenticated, Type excpectedException) { context.SetState(State.Ready); - var clientConnection = (IncomingClientConnection)GetIncomingConnection(context, TestEnvironment.Client1KeyPair.PublicKey, state); + var clientConnection = (IncomingClientConnection)GetIncomingConnection(context, TestEnvironment.Client1KeyPair.PublicKey, isAuthenticated); var envelope = new AccountDataRequest { @@ -218,12 +218,12 @@ public async Task AccountDataRequestTest(ConnectionState state, Type excpectedEx // MinuteLimit = (uint)requestLimit.Value // } // }; - // var effectProcessor = new RequestRateLimitUpdateEffectProcessor(effect, account, context.Constellation.RequestRateLimits); + // var effectProcessor = new RequestRateLimitUpdateEffectProcessor(effect, account, context.ConstellationSettingsManager.Current.RequestRateLimits); // effectProcessor.CommitEffect(); // } // var clientConnection = GetIncomingConnection(context, clientKeyPair.PublicKey, ConnectionState.Ready); - // var minuteLimit = (account.RequestRateLimits ?? context.Constellation.RequestRateLimits).MinuteLimit; + // var minuteLimit = (account.RequestRateLimits ?? context.ConstellationSettingsManager.Current.RequestRateLimits).MinuteLimit; // var minuteIterCount = minuteLimit + 1; // for (var i = 0; i < minuteIterCount; i++) // { @@ -246,19 +246,19 @@ public async Task AccountDataRequestTest(ConnectionState state, Type excpectedEx static object[] EffectsRequestTestCases = { - new object[] { TestEnvironment.Client1KeyPair, ConnectionState.Connected, typeof(InvalidStateException) }, - new object[] { TestEnvironment.Client1KeyPair, ConnectionState.Ready, null } + new object[] { TestEnvironment.Client1KeyPair, false, typeof(UnauthorizedException) }, + new object[] { TestEnvironment.Client1KeyPair, true, null } }; [Test] [TestCaseSource(nameof(EffectsRequestTestCases))] - public async Task EffectsRequestTest(KeyPair client, ConnectionState state, Type excpectedException) + public async Task EffectsRequestTest(KeyPair client, bool isAuthenticated, Type excpectedException) { context.SetState(State.Ready); var account = context.AccountStorage.GetAccount(client); - var clientConnection = GetIncomingConnection(context, client.PublicKey, state); + var clientConnection = GetIncomingConnection(context, client.PublicKey, isAuthenticated); var envelope = new QuantumInfoRequest { Account = account.Pubkey }.CreateEnvelope(); envelope.Sign(client); diff --git a/Centaurus.Test.Domain/AuditorMessageHandlersTests.cs b/Centaurus.Test.Domain/AuditorMessageHandlersTests.cs index ace98ca6..fb06c1d7 100644 --- a/Centaurus.Test.Domain/AuditorMessageHandlersTests.cs +++ b/Centaurus.Test.Domain/AuditorMessageHandlersTests.cs @@ -42,21 +42,19 @@ public async Task HandshakeTest() static object[] QuantaBatchTestCases = { - new object[] { TestEnvironment.Client1KeyPair, ConnectionState.Ready, typeof(UnauthorizedException) }, - new object[] { TestEnvironment.AlphaKeyPair, ConnectionState.Connected, typeof(InvalidStateException) }, - new object[] { TestEnvironment.AlphaKeyPair, ConnectionState.Ready, null } + new object[] { TestEnvironment.AlphaKeyPair, null } }; [Test] [TestCaseSource(nameof(QuantaBatchTestCases))] - public async Task QuantaBatchTest(KeyPair alphaKeyPair, ConnectionState state, Type excpectedException) + public async Task QuantaBatchTest(KeyPair alphaKeyPair, Type excpectedException) { context.SetState(State.Ready); var connection = default(ConnectionBase); try { - connection = GetOutgoingConnection(context, alphaKeyPair, state); + connection = GetOutgoingConnection(context, alphaKeyPair); } catch (UnauthorizedException) { diff --git a/Centaurus.Test.Domain/BaseMessageHandlerTests.cs b/Centaurus.Test.Domain/BaseMessageHandlerTests.cs index f5ac7eda..aaea5efd 100644 --- a/Centaurus.Test.Domain/BaseMessageHandlerTests.cs +++ b/Centaurus.Test.Domain/BaseMessageHandlerTests.cs @@ -27,7 +27,7 @@ protected async Task AssertMessageHandling(T connection, IncomingMessage mess }); } - protected IncomingConnectionBase GetIncomingConnection(ExecutionContext context, RawPubKey pubKey, ConnectionState? state = null) + protected IncomingConnectionBase GetIncomingConnection(ExecutionContext context, RawPubKey pubKey, bool isAuthenticated = false) { var connection = default(IncomingConnectionBase); if (context.NodesManager.TryGetNode(pubKey, out var node)) @@ -35,17 +35,19 @@ protected IncomingConnectionBase GetIncomingConnection(ExecutionContext context, else connection = new IncomingClientConnection(context, pubKey, new DummyWebSocket(), "127.0.0.1"); - if (state != null) - connection.ConnectionState = state.Value; + if (isAuthenticated) + { + var connectionType = typeof(IncomingConnectionBase); + var authField = connectionType.GetField("isAuthenticated", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic); + authField.SetValue(connection, true); + } return connection; } - protected OutgoingConnection GetOutgoingConnection(ExecutionContext context, RawPubKey pubKey, ConnectionState? state = null) + protected OutgoingConnection GetOutgoingConnection(ExecutionContext context, RawPubKey pubKey) { context.NodesManager.TryGetNode(pubKey, out var node); - var connection = new OutgoingConnection(context, node, new OutgoingMessageStorage(), new DummyConnectionWrapper(new DummyWebSocket())); - if (state != null) - connection.ConnectionState = state.Value; + var connection = new OutgoingConnection(context, node, new DummyConnectionWrapper(new DummyWebSocket())); return connection; } } diff --git a/Centaurus.Test.Domain/BaseQuantumHandlerTests.cs b/Centaurus.Test.Domain/BaseQuantumHandlerTests.cs index cfb7a73e..1b4b6a1e 100644 --- a/Centaurus.Test.Domain/BaseQuantumHandlerTests.cs +++ b/Centaurus.Test.Domain/BaseQuantumHandlerTests.cs @@ -43,7 +43,7 @@ public async Task DepositQuantumTest(int cursor, string asset, Type excpectedExc var depositAmount = (ulong)new Random().Next(10, 1000); - var providerSettings = context.Constellation.Providers.First(); + var providerSettings = context.ConstellationSettingsManager.Current.Providers.First(); var paymentNotification = new DepositNotificationModel { @@ -105,7 +105,7 @@ public async Task OrderQuantumTest(bool useFirstAccount, int nonceInc, string as var envelope = order.CreateEnvelope(); envelope.Sign(accountKeypair); - var quantum = new RequestQuantum + var quantum = new ClientRequestQuantum { RequestEnvelope = envelope }; @@ -136,7 +136,7 @@ public async Task OrderCancellationQuantumTest(string asset, ulong amount, Order }; var envelope = order.CreateEnvelope().Sign(TestEnvironment.Client1KeyPair); - var quantum = new RequestQuantum { RequestEnvelope = envelope }; + var quantum = new ClientRequestQuantum { RequestEnvelope = envelope }; await AssertQuantumHandling(quantum); @@ -152,7 +152,7 @@ public async Task OrderCancellationQuantumTest(string asset, ulong amount, Order var ordersCount = acc.Orders.Count; envelope = orderCancellation.CreateEnvelope().Sign(useFakeSigner ? TestEnvironment.Client2KeyPair : TestEnvironment.Client1KeyPair); - quantum = new RequestQuantum { RequestEnvelope = envelope }; + quantum = new ClientRequestQuantum { RequestEnvelope = envelope }; await AssertQuantumHandling(quantum, excpectedException); if (excpectedException != null) @@ -160,7 +160,7 @@ public async Task OrderCancellationQuantumTest(string asset, ulong amount, Order //remove created order orderCancellation.OrderId = orderId - apexMod; envelope = orderCancellation.CreateEnvelope().Sign(TestEnvironment.Client1KeyPair); - quantum = new RequestQuantum { RequestEnvelope = envelope }; + quantum = new ClientRequestQuantum { RequestEnvelope = envelope }; await AssertQuantumHandling(quantum); return; } @@ -177,13 +177,13 @@ public async Task OrderCancellationQuantumTest(string asset, ulong amount, Order public async Task WithdrawalQuantumTest(ulong amount, bool useFakeSigner, Type excpectedException) { var account = context.AccountStorage.GetAccount(TestEnvironment.Client1KeyPair); - var providerSettings = context.Constellation.Providers.First(); + var providerSettings = context.ConstellationSettingsManager.Current.Providers.First(); var withdrawal = new WithdrawalRequest { Account = account.Pubkey, RequestId = (long)account.Nonce + 1, Amount = amount, - Asset = context.Constellation.QuoteAsset.Code, + Asset = context.ConstellationSettingsManager.Current.QuoteAsset.Code, Destination = KeyPair.Random().PublicKey, Provider = PaymentProviderBase.GetProviderId(providerSettings.Provider, providerSettings.Name) }; @@ -207,7 +207,7 @@ public async Task PaymentQuantumTest(ulong amount, bool useFakeSigner, Type excp { Account = account.Pubkey, RequestId = (long)account.Nonce + 1, - Asset = context.Constellation.QuoteAsset.Code, + Asset = context.ConstellationSettingsManager.Current.QuoteAsset.Code, Destination = TestEnvironment.Client2KeyPair, Amount = amount }; @@ -215,9 +215,9 @@ public async Task PaymentQuantumTest(ulong amount, bool useFakeSigner, Type excp var envelope = withdrawal.CreateEnvelope(); envelope.Sign(useFakeSigner ? TestEnvironment.Client2KeyPair : TestEnvironment.Client1KeyPair); - var quantum = new RequestQuantum { RequestEnvelope = envelope }; + var quantum = new ClientRequestQuantum { RequestEnvelope = envelope }; - var baseAsset = context.Constellation.QuoteAsset.Code; + var baseAsset = context.ConstellationSettingsManager.Current.QuoteAsset.Code; var expextedBalance = account.GetBalance(baseAsset).Amount - amount; await AssertQuantumHandling(quantum, excpectedException); @@ -264,7 +264,7 @@ public async Task AccountDataRequestTest(ulong nonceInc, Type excpectedException // if (requestLimit.HasValue) // account.RequestRateLimits = new RequestRateLimits { HourLimit = (uint)requestLimit.Value, MinuteLimit = (uint)requestLimit.Value }; - // var minuteLimit = (account.RequestRateLimits ?? context.Constellation.RequestRateLimits).MinuteLimit; + // var minuteLimit = (account.RequestRateLimits ?? context.ConstellationSettingsManager.Current.RequestRateLimits).MinuteLimit; // var minuteIterCount = minuteLimit + 1; // for (var i = 0; i < minuteIterCount; i++) // { diff --git a/Centaurus.Test.Domain/OrderbookTests.cs b/Centaurus.Test.Domain/OrderbookTests.cs index 65b7c0c4..71ad20d4 100644 --- a/Centaurus.Test.Domain/OrderbookTests.cs +++ b/Centaurus.Test.Domain/OrderbookTests.cs @@ -97,12 +97,12 @@ public void TearDown() private void ExecuteWithOrderbook(int iterations, bool useNormalDistribution, Action executor) { var rnd = new Random(); - var asset = context.Constellation.Assets[1].Code; + var asset = context.ConstellationSettingsManager.Current.Assets[1].Code; var market = context.Exchange.GetMarket(asset); var orderRequestProcessor = new OrderRequestProcessor(context); - var testTradeResults = new Dictionary(); + var testTradeResults = new Dictionary(); for (var i = 1; i < iterations; i++) { var price = useNormalDistribution ? rnd.NextNormallyDistributed() + 50 : rnd.NextDouble() * 100; @@ -127,7 +127,7 @@ private void ExecuteWithOrderbook(int iterations, bool useNormalDistribution, Ac request.Side = OrderSide.Sell; } - var trade = new RequestQuantum + var trade = new ClientRequestQuantum { Apex = (ulong)i, RequestEnvelope = new MessageEnvelope @@ -140,7 +140,7 @@ private void ExecuteWithOrderbook(int iterations, bool useNormalDistribution, Ac testTradeResults.Add(trade, new QuantumProcessingItem(trade, System.Threading.Tasks.Task.FromResult(true)) { Initiator = initiator }); } - var baseAsset = context.Constellation.QuoteAsset.Code; + var baseAsset = context.ConstellationSettingsManager.Current.QuoteAsset.Code; var xlmStartBalance = account1.GetBalance(baseAsset).Amount + account2.GetBalance(baseAsset).Amount; var assetStartBalance = account1.GetBalance(asset).Amount + account2.GetBalance(asset).Amount; @@ -192,7 +192,7 @@ public void OrderbookPerformanceTest(int iterations, bool useNormalDistribution { ExecuteWithOrderbook(iterations, useNormalDistribution, executeOrders => PerfCounter.MeasureTime(executeOrders, () => { - var market = context.Exchange.GetMarket(context.Constellation.Assets[1].Code); + var market = context.Exchange.GetMarket(context.ConstellationSettingsManager.Current.Assets[1].Code); return $"{iterations} iterations, orderbook size: {market.Bids.Count} bids, {market.Asks.Count} asks, {market.Bids.GetBestPrice().ToString("G3")}/{market.Asks.GetBestPrice().ToString("G3")} spread."; })); } @@ -206,7 +206,7 @@ public void OrderbookRemoveTest(bool isOrderedByPrice) var random = new Random(); var price = 1D; var side = OrderSide.Buy; - var asset = context.Constellation.Assets[1].Code; + var asset = context.ConstellationSettingsManager.Current.Assets[1].Code; var orderbook = context.Exchange.GetOrderbook(asset, side); var ordersCount = 1000; diff --git a/Centaurus.Test.Domain/PendingQuantaPersistentTest.cs b/Centaurus.Test.Domain/PendingQuantaPersistentTest.cs index 92584a51..0bc47790 100644 --- a/Centaurus.Test.Domain/PendingQuantaPersistentTest.cs +++ b/Centaurus.Test.Domain/PendingQuantaPersistentTest.cs @@ -45,6 +45,7 @@ public async Task PendingQuantaTest() await Task.Factory.StartNew(() => { context.Complete(); + context.Dispose(); }); //verify that quantum is in the storage @@ -63,6 +64,10 @@ await Task.Factory.StartNew(() => context = new ExecutionContext(settings, storage, new MockPaymentProviderFactory(), new DummyConnectionWrapperFactory()); Assert.AreEqual(State.Rising, context.NodesManager.CurrentNode.State); + await context.Catchup.AddNodeBatch(TestEnvironment.Auditor1KeyPair, new CatchupQuantaBatch { Quanta = new List(), HasMore = false }); + + Assert.AreEqual(State.Running, context.NodesManager.CurrentNode.State); + var targetQuantum = context.SyncStorage.GetQuanta(quantum.Apex - 1, 1).FirstOrDefault(); Assert.NotNull(targetQuantum); @@ -72,9 +77,6 @@ await Task.Factory.StartNew(() => Assert.AreEqual(quantumFromStorage.Apex, result.Apex); Assert.IsInstanceOf(targetQuantum.Quantum); - //context.NodesManager.CurrentNode.Rised(); - Assert.AreEqual(State.Running, context.NodesManager.CurrentNode.State); - Assert.IsNull(context.PersistentStorage.LoadPendingQuanta()); } } diff --git a/Centaurus.Test.Domain/QuantumHandlerPerformanceTest.cs b/Centaurus.Test.Domain/QuantumHandlerPerformanceTest.cs index 8e1b9c8a..cbe73ce1 100644 --- a/Centaurus.Test.Domain/QuantumHandlerPerformanceTest.cs +++ b/Centaurus.Test.Domain/QuantumHandlerPerformanceTest.cs @@ -29,10 +29,10 @@ public async Task PerformanceTest() context.SetState(State.Ready); var accountId = context.AccountStorage.GetAccount(TestEnvironment.Client1KeyPair); - var messages = new Dictionary>(); + var messages = new Dictionary>(); for (var i = 0; i < 100_000; i++) { - var quantumRequest = new RequestQuantum + var quantumRequest = new ClientRequestQuantum { RequestEnvelope = new PaymentRequest { diff --git a/Centaurus.Test.Domain/QuantumStorageTests.cs b/Centaurus.Test.Domain/QuantumStorageTests.cs index da609f92..af6d13f5 100644 --- a/Centaurus.Test.Domain/QuantumStorageTests.cs +++ b/Centaurus.Test.Domain/QuantumStorageTests.cs @@ -45,18 +45,7 @@ public void QuantumStorageTest() Timestamp = DateTime.UtcNow.Ticks }; - context.SyncStorage.AddQuantum(quantum.Apex, new SyncQuantaBatchItem - { - Quantum = quantum, - AlphaSignature = new NodeSignatureInternal - { - AuditorId = 0, - PayloadSignature = new TinySignature - { - Data = new byte[64] - } - } - }); + context.SyncStorage.AddQuantum(new SyncQuantaBatchItem { Quantum = quantum }); items.Add(new QuantumPersistentModel { Apex = quantum.Apex, RawQuantum = XdrConverter.Serialize(quantum), Signatures = new List(), TimeStamp = quantum.Timestamp, Effects = new List() }); } diff --git a/Centaurus.Test.Exchange.Analytics/UpdatesTest.cs b/Centaurus.Test.Exchange.Analytics/UpdatesTest.cs index b44c9942..69fe0331 100644 --- a/Centaurus.Test.Exchange.Analytics/UpdatesTest.cs +++ b/Centaurus.Test.Exchange.Analytics/UpdatesTest.cs @@ -93,11 +93,11 @@ public void OrderbookPerformanceTest(int iterations, bool useNormalDistribution var rnd = new Random(); var asset = "USD"; var orderRequestProcessor = new OrderRequestProcessor(context); - var testTradeResults = new Dictionary(); + var testTradeResults = new Dictionary(); for (var i = 1; i < iterations; i++) { var price = useNormalDistribution ? rnd.NextNormallyDistributed() + 50 : rnd.NextDouble() * 100; - var trade = new RequestQuantum + var trade = new ClientRequestQuantum { Apex = (ulong)i, RequestEnvelope = new MessageEnvelope @@ -122,7 +122,7 @@ public void OrderbookPerformanceTest(int iterations, bool useNormalDistribution PerfCounter.MeasureTime(() => { foreach (var trade in testTradeResults) - context.Exchange.ExecuteOrder(asset, context.Constellation.QuoteAsset.Code, trade.Value); + context.Exchange.ExecuteOrder(asset, context.ConstellationSettingsManager.Current.QuoteAsset.Code, trade.Value); }, () => { var market = context.Exchange.GetMarket(asset); @@ -141,7 +141,7 @@ private void AnalyticsManager_OnUpdate() { try { - foreach (var asset in context.Constellation.Assets) + foreach (var asset in context.ConstellationSettingsManager.Current.Assets) { if (asset.IsQuoteAsset) //base asset continue; diff --git a/Centaurus.Test.Integration/IntegrationTestEnvironment.cs b/Centaurus.Test.Integration/IntegrationTestEnvironment.cs index fa9cffdb..7a4bc8c2 100644 --- a/Centaurus.Test.Integration/IntegrationTestEnvironment.cs +++ b/Centaurus.Test.Integration/IntegrationTestEnvironment.cs @@ -110,7 +110,7 @@ private Settings GetSettings(string secret, List auditors) SyncBatchSize = 500, GenesisAuditors = auditors, ConnectionString = "", - ParticipationLevel = 1 + IsPrimeNode = true }; settings.Build(); return settings; diff --git a/Centaurus.Test.Integration/IntegrationTestEnvironmentExtensions.cs b/Centaurus.Test.Integration/IntegrationTestEnvironmentExtensions.cs index 729d4bd3..1e01c120 100644 --- a/Centaurus.Test.Integration/IntegrationTestEnvironmentExtensions.cs +++ b/Centaurus.Test.Integration/IntegrationTestEnvironmentExtensions.cs @@ -244,7 +244,7 @@ public static async Task ProcessQuantumIsolated(this I var messageType = quantum.GetType().Name; var account = default(Account); - if (quantum is RequestQuantum requestQuantum) + if (quantum is ClientRequestQuantum requestQuantum) { messageType = requestQuantum.RequestMessage.GetMessageType(); account = context.AccountStorage.GetAccount(requestQuantum.RequestMessage.Account); diff --git a/Centaurus.Test.Integration/IntegrationTests.cs b/Centaurus.Test.Integration/IntegrationTests.cs index 728f607b..44f6a8f5 100644 --- a/Centaurus.Test.Integration/IntegrationTests.cs +++ b/Centaurus.Test.Integration/IntegrationTests.cs @@ -137,7 +137,7 @@ public async Task AlphaRestartWithQuantaDelayTest(bool invalidHash, bool invalid .CreateEnvelope() .Sign(clientPk); - var quantum = new RequestQuantum + var quantum = new ClientRequestQuantum { Apex = lastApex + 1, PrevHash = lastHash, @@ -168,7 +168,7 @@ await Task.WhenAll(environment.AuditorWrappers.Select(a => ((Quantum)quantum.Quantum).Timestamp = DateTime.UtcNow.Ticks; if (invalidClientSignature) { - var request = (RequestQuantum)quantum.Quantum; + var request = (ClientRequestQuantum)quantum.Quantum; ((MessageEnvelope)request.RequestEnvelope).Signature = new TinySignature { Data = new byte[64] }; request.RequestEnvelope.Sign(KeyPair.Random()); } @@ -213,20 +213,20 @@ public async Task ScamQuantaTest(bool useFakeClient, bool useFakeAlpha, bool inv var amount = invalidBalance ? client.GetBalance(environment.SDKConstellationInfo.QuoteAsset.Code).Amount + 1 - : environment.AlphaWrapper.Context.Constellation.MinAllowedLotSize + 1; + : environment.AlphaWrapper.Context.ConstellationSettingsManager.Current.MinAllowedLotSize + 1; var sqamRequest = new OrderRequest { Account = client.Pubkey, Amount = amount, Price = 1, - Asset = environment.AlphaWrapper.Context.Constellation.Assets[1].Code, + Asset = environment.AlphaWrapper.Context.ConstellationSettingsManager.Current.Assets[1].Code, RequestId = 1, Side = OrderSide.Buy }.CreateEnvelope().Sign(useFakeClient ? KeyPair.Random() : clientPk); var apex = environment.AlphaWrapper.Context.QuantumHandler.CurrentApex + 1; var lastQuantumHash = environment.AlphaWrapper.Context.QuantumHandler.LastQuantumHash; - var requestQuantum = new RequestQuantum + var requestQuantum = new ClientRequestQuantum { Apex = apex, EffectsProof = new byte[] { }, @@ -235,18 +235,7 @@ public async Task ScamQuantaTest(bool useFakeClient, bool useFakeAlpha, bool inv Timestamp = DateTime.UtcNow.Ticks }; - syncStorage.AddQuantum(apex, new SyncQuantaBatchItem - { - Quantum = requestQuantum, - AlphaSignature = new NodeSignatureInternal - { - AuditorId = environment.AlphaWrapper.Context.AlphaId, - PayloadSignature = new TinySignature - { - Data = environment.AlphaWrapper.Context.Settings.KeyPair.Sign(requestQuantum.GetPayloadHash()) - } - } - }); + syncStorage.AddQuantum(new SyncQuantaBatchItem { Quantum = requestQuantum }); var expectedState = useFakeClient || useFakeAlpha || invalidBalance ? State.Failed : State.Ready; diff --git a/Centaurus.Test.Utils/GlobalInitHelper.cs b/Centaurus.Test.Utils/GlobalInitHelper.cs index 470e4885..fe65ff15 100644 --- a/Centaurus.Test.Utils/GlobalInitHelper.cs +++ b/Centaurus.Test.Utils/GlobalInitHelper.cs @@ -1,5 +1,4 @@ using Centaurus.Domain; -using Centaurus.Domain.StateManagers; using Centaurus.Models; using Centaurus.PaymentProvider; using Centaurus.PaymentProvider.Models; @@ -45,8 +44,9 @@ private static void SetCommonSettings(Settings settings, string secret) settings.CWD = "AppData"; settings.GenesisAuditors = new[] { TestEnvironment.AlphaKeyPair.AccountId, TestEnvironment.Auditor1KeyPair.AccountId }.Select(a => new Settings.Auditor($"{a}={a}.com")).ToList(); settings.Secret = secret; - settings.ParticipationLevel = 1; + settings.IsPrimeNode = true; settings.SyncBatchSize = 500; + settings.CatchupTimeout = 1; } public static List GetPredefinedClients() @@ -128,7 +128,11 @@ public static async Task Setup(List clients, List(); Action addAssetsFn = (acc, asset) => @@ -167,7 +171,11 @@ public static async Task Setup(List clients, List @@ -178,6 +186,21 @@ await Task.Factory.StartNew(() => return context; } + public static MajoritySignaturesBatchItem GetAuditorSignatures(ExecutionContext context, QuantumProcessingItem processingItem) + { + var payloadSignature = processingItem.Quantum.GetPayloadHash().Sign(TestEnvironment.Auditor1KeyPair); + return new MajoritySignaturesBatchItem + { + Apex = processingItem.Apex, + Signatures = new List { + new NodeSignatureInternal { + NodeId = context.NodesManager.NodePubKeys[TestEnvironment.Auditor1KeyPair], + PayloadSignature = payloadSignature + } + } + }; + } + public static void SetState(this ExecutionContext context, State state) { var method = typeof(CurrentNode).GetMethod("SetState", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); diff --git a/Centaurus.Test.Utils/MockStorage.cs b/Centaurus.Test.Utils/MockStorage.cs index 1a5a31a6..8aa4ffdc 100644 --- a/Centaurus.Test.Utils/MockStorage.cs +++ b/Centaurus.Test.Utils/MockStorage.cs @@ -47,7 +47,7 @@ public CursorsPersistentModel LoadCursors() public SettingsPersistentModel LoadSettings(ulong fromApex) { - return settingsCollection.OrderBy(s => s.Apex).FirstOrDefault(s => s.Apex >= fromApex); + return settingsCollection.OrderByDescending(s => s.Apex).FirstOrDefault(s => s.Apex <= fromApex); } public QuantumPersistentModel LoadQuantum(ulong apex) diff --git a/Centaurus.Test.XdrSerialization/MessageSerializationTests.cs b/Centaurus.Test.XdrSerialization/MessageSerializationTests.cs index 3426f92f..2fff101b 100644 --- a/Centaurus.Test.XdrSerialization/MessageSerializationTests.cs +++ b/Centaurus.Test.XdrSerialization/MessageSerializationTests.cs @@ -56,7 +56,7 @@ public void OrderSerializationTest() [Test] public void NullValueSerializationTest() { - var signature = new NodeSignatureInternal { AuditorId = 1, PayloadSignature = new TinySignature { Data = BinaryExtensions.RandomBytes(64) } }; + var signature = new NodeSignatureInternal { NodeId = 1, PayloadSignature = new TinySignature { Data = BinaryExtensions.RandomBytes(64) } }; var rawData = XdrConverter.Serialize(signature); signature = XdrConverter.Deserialize(rawData); Assert.AreEqual(null, signature.TxSignature); diff --git a/Centaurus/Alpha/Controllers/ConstellationController.cs b/Centaurus/Alpha/Controllers/ConstellationController.cs index 7ccf32d2..de737881 100644 --- a/Centaurus/Alpha/Controllers/ConstellationController.cs +++ b/Centaurus/Alpha/Controllers/ConstellationController.cs @@ -39,9 +39,9 @@ public async Task Update(ConstellationMessageEnvelope constellati if (constellationInitEnvelope == null) return StatusCode(415); - await Context.HandleConstellationQuantum(constellationInitEnvelope); + var apex = await Context.HandleConstellationQuantum(constellationInitEnvelope); - return new JsonResult(new InitResult { IsSuccess = true }); + return new JsonResult(new InitResult { IsSuccess = true, Apex = apex }); } catch (Exception exc) { @@ -53,6 +53,8 @@ public class InitResult { public bool IsSuccess { get; set; } + public ulong Apex { get; set; } + public string Error { get; set; } } } diff --git a/Centaurus/Startup.cs b/Centaurus/Startup.cs index f8bf5c44..8ef90ce6 100644 --- a/Centaurus/Startup.cs +++ b/Centaurus/Startup.cs @@ -19,7 +19,7 @@ public Startup(Domain.ExecutionContext context, AlphaHostFactoryBase alphaHostFa if (alphaHostFactory == null) throw new ArgumentNullException(nameof(alphaHostFactory)); - if (context.RoleManager.ParticipationLevel == CentaurusNodeParticipationLevel.Prime) + if (context.Settings.IsPrimeNode) AlphaStartup = new AlphaStartup(context, alphaHostFactory); } From d60ef4e9099aef51c521e3e532d0aff109feb35c Mon Sep 17 00:00:00 2001 From: hawthorne-abendsen <49230725+hawthorne-abendsen@users.noreply.github.com> Date: Tue, 21 Dec 2021 01:00:55 +0200 Subject: [PATCH 06/13] Catchup refactoring; ConstellationSettingsManager --- Centaurus.Domain/Catchups/Catchup.cs | 148 +++++++-------- .../Contexts/ConstellationSettingsManager.cs | 170 +++++++++++++++--- Centaurus.Domain/Contexts/ExecutionContext.cs | 15 +- .../CatchupQuantaBatchHandler.cs | 8 +- .../CatchupQuantaBatchRequestHandler.cs | 10 +- .../Handshake/HandshakeResponseHandler.cs | 12 +- .../MessageHandlers/MessageHandlerBase.cs | 1 + .../MessageHandlers/SyncQuantaBatchHandler.cs | 6 +- .../Nodes/Common/ValueAggregation.cs | 8 +- .../Nodes/Managers/SyncSourceManager.cs | 1 - .../Nodes/RemoteNodes/RemoteNode.cs | 15 +- .../Nodes/RemoteNodes/RemoteNodeConnection.cs | 7 - .../Quanta/Handlers/QuantumHandler.cs | 27 ++- Centaurus.Domain/Quanta/Sync/CursorGroup.cs | 2 + .../Quanta/Sync/SyncQuantaDataWorker.cs | 22 +-- Centaurus.Domain/Quanta/Sync/SyncStorage.cs | 32 +++- .../ResultManagers/ResultManager.cs | 9 +- Centaurus/Startup.cs | 6 +- 18 files changed, 340 insertions(+), 159 deletions(-) diff --git a/Centaurus.Domain/Catchups/Catchup.cs b/Centaurus.Domain/Catchups/Catchup.cs index b1421b0f..ea25f769 100644 --- a/Centaurus.Domain/Catchups/Catchup.cs +++ b/Centaurus.Domain/Catchups/Catchup.cs @@ -123,11 +123,12 @@ private async Task TryApplyAuditorsData() return; int majority = Context.GetMajorityCount(), - totalAuditorsCount = Context.GetTotalAuditorsCount(); + totalAuditorsCount = Context.GetTotalAuditorsCount(); if (Context.HasMajority(validAuditorStates.Count) || !Context.NodesManager.IsAlpha) { - await ApplyAuditorsData(); + var validQuanta = GetValidQuanta(); + await ApplyQuanta(validQuanta); allAuditorStates.Clear(); validAuditorStates.Clear(); Context.NodesManager.CurrentNode.Rised(); @@ -171,35 +172,28 @@ private void ApplyDataTimer_Elapsed(object sender, System.Timers.ElapsedEventArg } } - private async Task ApplyAuditorsData() - { - var validQuanta = GetValidQuanta(); - - await ApplyQuanta(validQuanta); - } - - private async Task ApplyQuanta(List<(Quantum quantum, List signatures)> quanta) + private async Task ApplyQuanta(List quanta) { foreach (var quantumItem in quanta) { - Context.ResultManager.Add(new MajoritySignaturesBatchItem { Apex = quantumItem.quantum.Apex, Signatures = quantumItem.signatures }); + Context.ResultManager.Add(new MajoritySignaturesBatchItem { Apex = quantumItem.Quantum.Apex, Signatures = quantumItem.Signatures }); //compute quantum hash before processing - var originalQuantumHash = quantumItem.quantum.ComputeHash(); + var originalQuantumHash = quantumItem.Quantum.ComputeHash(); - var processingItem = Context.QuantumHandler.HandleAsync(quantumItem.quantum, QuantumSignatureValidator.Validate(quantumItem.quantum)); + var processingItem = Context.QuantumHandler.HandleAsync(quantumItem.Quantum, QuantumSignatureValidator.Validate(quantumItem.Quantum)); await processingItem.OnAcknowledged; //compute quantum hash after processing - var processedQuantumHash = quantumItem.quantum.ComputeHash(); + var processedQuantumHash = quantumItem.Quantum.ComputeHash(); //TODO: do we need some extra checks here? if (!ByteArrayPrimitives.Equals(originalQuantumHash, processedQuantumHash)) throw new Exception("Quantum hash are not equal on restore."); } } - private List<(Quantum quantum, List signatures)> GetValidQuanta() + private List GetValidQuanta() { //group all quanta by their apex var quanta = allAuditorStates.Values @@ -207,7 +201,7 @@ private async Task ApplyQuanta(List<(Quantum quantum, List ((Quantum)q.Quantum).Apex) .OrderBy(q => q.Key); - var validQuanta = new List<(Quantum quantum, List signatures)>(); + var validQuanta = new List(); if (quanta.Count() == 0) return validQuanta; @@ -222,17 +216,14 @@ private async Task ApplyQuanta(List<(Quantum quantum, List n.PubKey).ToList()); } - private (Quantum quantum, List signatures) GetQuantumData(ulong apex, List allQuanta, int alphaId, List auditors) + private ValidatedQuantumData GetQuantumData(ulong apex, List allApexQuanta, int alphaId, List auditors) { - var payloadHash = default(byte[]); - var quantum = default(Quantum); - var signatures = new Dictionary(); + //compute and group quanta by payload hash + var grouped = allApexQuanta + .GroupBy(q => ((Quantum)q.Quantum).GetPayloadHash(), ByteArrayComparer.Default); - foreach (var currentItem in allQuanta) + //validate each quantum data group + var validatedQuantaData = new List(); + foreach (var quantaGroup in grouped) { - var currentQuantum = (Quantum)currentItem.Quantum; + validatedQuantaData.Add(ProcessQuantumGroup(auditors, quantaGroup)); + } + //get majority for current auditors set + var majorityCount = MajorityHelper.GetMajorityCount(auditors.Count); + //get all quanta data with majority + var quantaDataWithMajority = validatedQuantaData.Where(a => a.Signatures.Count > majorityCount).ToList(); + if (quantaDataWithMajority.Count > 1) + throw new Exception($"Conflict found for apex {apex}. {quantaDataWithMajority.Count} quanta data sets with majority."); + else if (quantaDataWithMajority.Count == 0 && validatedQuantaData.Count > 1) + throw new Exception($"Conflict found for apex {apex}. {validatedQuantaData.Count} quanta data sets, but no majority."); + + //if we have quanta data with majority, return it + if (quantaDataWithMajority.Count > 0) + return quantaDataWithMajority.First(); + else + return validatedQuantaData.First(); + } - //compute current quantum payload hash - var currentPayloadHash = currentQuantum.GetPayloadHash(); + private ValidatedQuantumData ProcessQuantumGroup(List auditors, IGrouping quantaGroup) + { + var quantumHash = quantaGroup.Key; + var quantum = (Quantum)quantaGroup.First().Quantum; + var allSignatures = quantaGroup.SelectMany(q => q.Signatures); + var validSignatures = new Dictionary(); - //verify that quantum has alpha signature - if (!currentItem.Signatures.Any(s => s.NodeId == alphaId)) + //validate each signature + foreach (var signature in allSignatures) + { + //skip if current auditor is already presented in signatures + if (validSignatures.ContainsKey(signature.NodeId)) continue; - //validate each signature - foreach (var signature in currentItem.Signatures) - { - //skip if current auditor is already presented in signatures, but force alpha signature to be checked - if (signature.NodeId != alphaId && signatures.ContainsKey(signature.NodeId)) - continue; + //try to get the node public key + var signer = auditors.ElementAtOrDefault(signature.NodeId - 1); //node id is index + 1 - //try get auditor - var signer = auditors.ElementAtOrDefault(signature.NodeId); - //if auditor is not found or it's signature already added, move to the next signature - if (signer == null || !signature.PayloadSignature.IsValid(signer, currentPayloadHash)) - continue; + //if the node is not found or it's signature is invalid, move to the next signature + if (signer == null || !signature.PayloadSignature.IsValid(signer, quantumHash)) + continue; - if (payloadHash != null && !ByteArrayPrimitives.Equals(payloadHash, currentPayloadHash)) - { - //if there are several quanta with same apex but with different hash signed by Alpha - if (alphaId == signature.NodeId) - throw new Exception($"Alpha {signer.GetAccountId()} private key is compromised. Apex {apex}."); + //check transaction and it's signatures + if (quantum is WithdrawalRequestQuantum transactionQuantum) + { + var provider = Context.PaymentProvidersManager.GetManager(transactionQuantum.Provider); + if (!provider.IsTransactionValid(transactionQuantum.Transaction, transactionQuantum.WithdrawalRequest.ToProviderModel(), out var error)) + throw new Exception($"Transaction is invalid.\nReason: {error}"); + + if (!provider.AreSignaturesValid(transactionQuantum.Transaction, new SignatureModel { Signer = signature.TxSigner, Signature = signature.TxSignature })) //skip invalid signature continue; - } - - //check transaction and it's signatures - if (currentItem.Quantum is WithdrawalRequestQuantum transactionQuantum) - { - var provider = Context.PaymentProvidersManager.GetManager(transactionQuantum.Provider); - if (!provider.IsTransactionValid(transactionQuantum.Transaction, transactionQuantum.WithdrawalRequest.ToProviderModel(), out var error)) - throw new Exception($"Transaction is invalid.\nReason: {error}"); - - if (!provider.AreSignaturesValid(transactionQuantum.Transaction, new SignatureModel { Signer = signature.TxSigner, Signature = signature.TxSignature })) - //skip invalid signature - continue; - } - signatures[signature.NodeId] = signature; } - - //continue if quantum already set - if (quantum != null) - continue; - - quantum = currentQuantum; - payloadHash = currentPayloadHash; + //add valid signatures + validSignatures.Add(signature.NodeId, signature); } - - return (quantum, signatures.Values.ToList()); + return new ValidatedQuantumData(quantum, validSignatures.Values.ToList()); } public void Dispose() @@ -319,5 +312,18 @@ public void Dispose() applyDataTimer?.Dispose(); applyDataTimer = null; } + + class ValidatedQuantumData + { + public ValidatedQuantumData(Quantum quantum, List signatures) + { + Quantum = quantum; + Signatures = signatures; + } + + public Quantum Quantum { get; } + + public List Signatures { get; } + } } } diff --git a/Centaurus.Domain/Contexts/ConstellationSettingsManager.cs b/Centaurus.Domain/Contexts/ConstellationSettingsManager.cs index fe318756..ae766ac5 100644 --- a/Centaurus.Domain/Contexts/ConstellationSettingsManager.cs +++ b/Centaurus.Domain/Contexts/ConstellationSettingsManager.cs @@ -1,24 +1,30 @@ using Centaurus.Models; +using Microsoft.Extensions.Caching.Memory; +using NLog; using System; using System.Collections.Generic; using System.Linq; using System.Text; +using System.Timers; namespace Centaurus.Domain { /// /// This class holds recent constellation settings, to be able to obtain relevant node ids /// - public class ConstellationSettingsManager : ContextualBase + public class ConstellationSettingsManager : ContextualBase, IDisposable { + private ConstellationSettingsCollection settingsCache; + public ConstellationSettingsManager(ExecutionContext context) : base(context) { + settingsCache = new ConstellationSettingsCollection(Context.DataProvider.GetConstellationSettings); } public ConstellationSettings Current { get; private set; } - public void Add(ConstellationSettings newSettings) + public void Update(ConstellationSettings newSettings) { if (newSettings == null) throw new ArgumentNullException(nameof(newSettings)); @@ -26,54 +32,160 @@ public void Add(ConstellationSettings newSettings) if (Current != null && Current.Apex >= newSettings.Apex) throw new ArgumentException("New constellation settings apex is lower than the current one.", nameof(newSettings)); + settingsCache.Add(newSettings); + Current = newSettings; + } + + public bool TryGetForApex(ulong apex, out ConstellationSettings apexSettings) + { + var current = Current; + if (apex > current.Apex) + apexSettings = current; + else + settingsCache.TryGetSettings(apex, out apexSettings); + return apexSettings != null; + } + + public void Dispose() + { + settingsCache.Dispose(); + } + } + + class ConstellationSettingsCollection : IDisposable + { + public ConstellationSettingsCollection(Func constellationSettingsDataProvider) + { + getConstellationSettings = constellationSettingsDataProvider ?? throw new ArgumentNullException(nameof(constellationSettingsDataProvider)); + InitCleanupTimer(); + } + + public void Add(ConstellationSettings settings) + { lock (syncRoot) { - settings.Add(newSettings.Apex, newSettings); - Current = newSettings; - Cleanup(); + var settingsWrapper = new ConstellationSettingsWrapper(settings); + settingsCache.Add(settingsWrapper); } } - public bool TryGetForApex(ulong apex, out ConstellationSettings apexSettings) + public bool TryGetSettings(ulong apex, out ConstellationSettings settings) { - apexSettings = null; - if (Current == null) //if current is null, than there is no constellation settings yet + settings = null; + lock (syncRoot) + { + //try to find the settings in cache + for (int i = settingsCache.Count; i-- > 0;) + { + var currentSettings = settingsCache[i]; + if (apex >= currentSettings.Apex) + { + currentSettings.AccessDate = DateTime.UtcNow; + settings = currentSettings.Value; + return true; + } + } + + //try to load from db + var lastLoadedSettingsApex = settingsCache.First().Apex; + while (true) + { + var prevSettingsLastApex = lastLoadedSettingsApex - 1; + var loadedItem = getConstellationSettings(apex); + //db doesn't contains settings for the apex + if (loadedItem == null) + break; + var item = InsertFirst(loadedItem, prevSettingsLastApex); + if (apex >= item.Apex) + { + settings = item.Value; + return true; + } + //set last loaded apex + lastLoadedSettingsApex = item.Apex; + } + //no settings found return false; + } + } + + public void Dispose() + { + cleanupTimer.Dispose(); + } - if (apex >= Current.Apex) + private ConstellationSettingsWrapper InsertFirst(ConstellationSettings settings, ulong validToApex) + { + lock (syncRoot) { - apexSettings = Current; - return true; + var settingsWrapper = new ConstellationSettingsWrapper(settings); + settingsCache.Insert(0, settingsWrapper); + return settingsWrapper; } + } - lock (syncRoot) + private static Logger logger = LogManager.GetCurrentClassLogger(); + + private List settingsCache = new List(); + + private void InitCleanupTimer() + { + cleanupTimer = new Timer(); + cleanupTimer.AutoReset = false; + cleanupTimer.Interval = TimeSpan.FromSeconds(1).TotalMilliseconds; + cleanupTimer.Elapsed += CleanupTimer_Elapsed; + cleanupTimer.Start(); + } + + private void CleanupTimer_Elapsed(object sender, ElapsedEventArgs e) + { + try { - //looking for the first settings where apex is lower or equal to the request apex - foreach (var settingApex in settings.Keys) + lock (syncRoot) { - //the setting is newer than apex - if (settingApex > apex) - continue; - - apexSettings = settings[apex]; - return true; + //cache 1000 items + if (settingsCache.Count < 1000) + return; + var currentDate = DateTime.UtcNow; + foreach (var settings in settingsCache) + { + if (currentDate - settings.AccessDate > TimeSpan.FromSeconds(15)) + { + settingsCache.RemoveAt(0); + continue; + } + //break cleanup to keep settings chain + break; + } } } - return false; + catch (Exception exc) + { + logger.Error(exc, "Error during settings cache cleanup."); + } + finally + { + cleanupTimer.Start(); + } } private object syncRoot = new { }; + private Timer cleanupTimer; + private Func getConstellationSettings; - private SortedDictionary settings = new SortedDictionary(); - - private void Cleanup() + class ConstellationSettingsWrapper { - //remove old data - if (settings.Count > 1000) + public ConstellationSettingsWrapper(ConstellationSettings settings) { - var firstApex = settings.Keys.First(); - settings.Remove(firstApex); + Value = settings ?? throw new ArgumentNullException(nameof(settings)); + AccessDate = DateTime.UtcNow; } + + public ulong Apex => Value.Apex; + + public ConstellationSettings Value { get; } + + public DateTime AccessDate { get; set; } } } -} +} \ No newline at end of file diff --git a/Centaurus.Domain/Contexts/ExecutionContext.cs b/Centaurus.Domain/Contexts/ExecutionContext.cs index 300f3618..3f99278d 100644 --- a/Centaurus.Domain/Contexts/ExecutionContext.cs +++ b/Centaurus.Domain/Contexts/ExecutionContext.cs @@ -5,6 +5,7 @@ using Centaurus.PaymentProvider; using Centaurus.PersistentStorage; using Centaurus.PersistentStorage.Abstraction; +using Microsoft.Extensions.Caching.Memory; using NLog; using System; using System.Collections.Generic; @@ -94,8 +95,7 @@ internal ExecutionContext( else SetNodes().Wait(); - if (persistentData.pendingQuanta != null) - HandlePendingQuanta(persistentData.pendingQuanta); + HandlePendingQuanta(persistentData.pendingQuanta); SyncStorage = new SyncStorage(this, lastApex); @@ -115,7 +115,11 @@ private void CurrentNode_StateChanged(StateChangedEventArgs stateChangedEventArg private void HandlePendingQuanta(List pendingQuanta) { - _ = Catchup.AddNodeBatch(Settings.KeyPair, new CatchupQuantaBatch { Quanta = pendingQuanta, HasMore = false }); + _ = Catchup.AddNodeBatch(Settings.KeyPair, new CatchupQuantaBatch + { + Quanta = pendingQuanta ?? new List(), + HasMore = false + }); } private string GetAbsolutePath(string path) @@ -154,7 +158,8 @@ public void Init(Snapshot snapshot) public async Task UpdateConstellationSettings(ConstellationSettings constellationSettings) { - ConstellationSettingsManager.Add(constellationSettings); + if (constellationSettings != null) + ConstellationSettingsManager.Update(constellationSettings); //update current auditors await SetNodes(); @@ -172,6 +177,8 @@ public void Complete() ); PersistPendingQuanta(); + + OnComplete?.Invoke(); } private void PersistPendingQuanta() diff --git a/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchHandler.cs b/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchHandler.cs index a964b4eb..ed6d4be2 100644 --- a/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchHandler.cs +++ b/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchHandler.cs @@ -6,7 +6,7 @@ namespace Centaurus.Domain { - internal class CatchupQuantaBatchHandler: MessageHandlerBase + internal class CatchupQuantaBatchHandler: MessageHandlerBase { public CatchupQuantaBatchHandler(ExecutionContext context) :base(context) @@ -15,7 +15,11 @@ public CatchupQuantaBatchHandler(ExecutionContext context) public override string SupportedMessageType => typeof(CatchupQuantaBatch).Name; - public override Task HandleMessage(IncomingNodeConnection connection, IncomingMessage message) + public override bool IsAuditorOnly => true; + + public override bool IsAuthenticatedOnly => true; + + public override Task HandleMessage(ConnectionBase connection, IncomingMessage message) { _ = Context.Catchup.AddNodeBatch(connection.PubKey, (CatchupQuantaBatch)message.Envelope.Message); return Task.CompletedTask; diff --git a/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchRequestHandler.cs b/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchRequestHandler.cs index 2aacf621..1b63895f 100644 --- a/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchRequestHandler.cs +++ b/Centaurus.Domain/MessageHandlers/CatchupQuantaBatchRequestHandler.cs @@ -8,7 +8,7 @@ namespace Centaurus.Domain { - internal class CatchupQuantaBatchRequestHandler : MessageHandlerBase + internal class CatchupQuantaBatchRequestHandler : MessageHandlerBase { public CatchupQuantaBatchRequestHandler(ExecutionContext context) : base(context) @@ -17,13 +17,17 @@ public CatchupQuantaBatchRequestHandler(ExecutionContext context) } public override string SupportedMessageType => typeof(CatchupQuantaBatchRequest).Name; - public override async Task HandleMessage(OutgoingConnection connection, IncomingMessage message) + public override bool IsAuditorOnly => true; + + public override bool IsAuthenticatedOnly => true; + + public override async Task HandleMessage(ConnectionBase connection, IncomingMessage message) { var batchRequest = (CatchupQuantaBatchRequest)message.Envelope.Message; await SendQuanta(connection, batchRequest); } - private async Task SendQuanta(OutgoingConnection connection, CatchupQuantaBatchRequest batchRequest) + private async Task SendQuanta(ConnectionBase connection, CatchupQuantaBatchRequest batchRequest) { var aboveApex = batchRequest.LastKnownApex; var batchSize = Context.Settings.SyncBatchSize; diff --git a/Centaurus.Domain/MessageHandlers/Handshake/HandshakeResponseHandler.cs b/Centaurus.Domain/MessageHandlers/Handshake/HandshakeResponseHandler.cs index 5e2cf371..312fd459 100644 --- a/Centaurus.Domain/MessageHandlers/Handshake/HandshakeResponseHandler.cs +++ b/Centaurus.Domain/MessageHandlers/Handshake/HandshakeResponseHandler.cs @@ -52,19 +52,11 @@ private async Task HandleClientHandshake(IncomingClientConnection clientConnecti await clientConnection.SendMessage(envelope.CreateResult(ResultStatusCode.Success)); } - private async Task HandleAuditorHandshake(IncomingNodeConnection auditorConnection) + private Task HandleAuditorHandshake(IncomingNodeConnection auditorConnection) { //register connection auditorConnection.Node.RegisterIncomingConnection(auditorConnection); - - //request quanta on rising - if (Context.NodesManager.CurrentNode.State == State.Rising) - { - await auditorConnection.SendMessage(new CatchupQuantaBatchRequest - { - LastKnownApex = Context.QuantumHandler.CurrentApex - }.CreateEnvelope()); - } + return Task.CompletedTask; } } } diff --git a/Centaurus.Domain/MessageHandlers/MessageHandlerBase.cs b/Centaurus.Domain/MessageHandlers/MessageHandlerBase.cs index b5e8c013..27fe3d3c 100644 --- a/Centaurus.Domain/MessageHandlers/MessageHandlerBase.cs +++ b/Centaurus.Domain/MessageHandlers/MessageHandlerBase.cs @@ -19,6 +19,7 @@ protected MessageHandlerBase(ExecutionContext context) /// public abstract string SupportedMessageType { get; } + //TODO: set it to true if IsAuditorOnly == true /// /// Only messages from the authenticated connections are allowed. /// diff --git a/Centaurus.Domain/MessageHandlers/SyncQuantaBatchHandler.cs b/Centaurus.Domain/MessageHandlers/SyncQuantaBatchHandler.cs index f09f2809..a47f70f9 100644 --- a/Centaurus.Domain/MessageHandlers/SyncQuantaBatchHandler.cs +++ b/Centaurus.Domain/MessageHandlers/SyncQuantaBatchHandler.cs @@ -32,7 +32,7 @@ private async Task AddQuantaToQueue(OutgoingConnection connection, IncomingMessa { var quantaBatch = (SyncQuantaBatch)message.Envelope.Message; var quanta = quantaBatch.Quanta; - var lastKnownApex = Context.QuantumHandler.CurrentApex; + var lastKnownApex = Context.QuantumHandler.LastAddedApex; foreach (var processedQuantum in quanta) { var quantum = (Quantum)processedQuantum.Quantum; @@ -46,11 +46,11 @@ await connection.SendMessage(new SyncCursorReset Cursors = new List { new SyncCursor { Type = XdrSyncCursorType.Quanta, - Cursor = Context.QuantumHandler.CurrentApex + Cursor = Context.QuantumHandler.LastAddedApex } } }.CreateEnvelope()); - logger.Warn($"Batch has invalid quantum apexes (current: {Context.QuantumHandler.CurrentApex}, received: {quantum.Apex}). New apex cursor request sent."); + logger.Warn($"Batch has invalid quantum apexes (current: {Context.QuantumHandler.LastAddedApex}, received: {quantum.Apex}). New apex cursor request sent."); return; } diff --git a/Centaurus.Domain/Nodes/Common/ValueAggregation.cs b/Centaurus.Domain/Nodes/Common/ValueAggregation.cs index bc144564..db7f009e 100644 --- a/Centaurus.Domain/Nodes/Common/ValueAggregation.cs +++ b/Centaurus.Domain/Nodes/Common/ValueAggregation.cs @@ -13,7 +13,7 @@ protected ValueAggregation(int maxItems) throw new ArgumentException("Data must contain at least two items to be able to calculate avg value.", nameof(maxItems)); } - private List> data = new List>(); + private List data = new List(); private object syncRoot = new { }; @@ -21,7 +21,7 @@ public void AddValue(DateTime dateTime, T value) { lock (syncRoot) { - data.Add(new Item { AddedAt = dateTime, Value = value }); + data.Add(new Item { AddedAt = dateTime, Value = value }); LastValue = value; if (data.Count > 20) //remove old data data.RemoveAt(0); @@ -38,13 +38,13 @@ public void Clear() public abstract int GetAvg(); - protected List> GetData() + protected List GetData() { lock (syncRoot) return data.ToList(); } - protected struct Item + protected struct Item { public DateTime AddedAt { get; set; } diff --git a/Centaurus.Domain/Nodes/Managers/SyncSourceManager.cs b/Centaurus.Domain/Nodes/Managers/SyncSourceManager.cs index c0795152..59aa9d62 100644 --- a/Centaurus.Domain/Nodes/Managers/SyncSourceManager.cs +++ b/Centaurus.Domain/Nodes/Managers/SyncSourceManager.cs @@ -60,7 +60,6 @@ private void ChooseNewSyncNode() var candidateNode = Context.NodesManager.GetRemoteNodes() .Where(n => n.IsConnected) //only connected .OrderByDescending(n => n.IsAlpha) //first of all try to set alpha - .ThenByDescending(n => n.IsPrimeNode) //then try to connect to prime node .ThenByDescending(n => n.LastApex) .FirstOrDefault(); diff --git a/Centaurus.Domain/Nodes/RemoteNodes/RemoteNode.cs b/Centaurus.Domain/Nodes/RemoteNodes/RemoteNode.cs index 35f57d9e..49e8793e 100644 --- a/Centaurus.Domain/Nodes/RemoteNodes/RemoteNode.cs +++ b/Centaurus.Domain/Nodes/RemoteNodes/RemoteNode.cs @@ -27,16 +27,27 @@ private void ConnectionManager_OnConnectionClosed() private void ConnectionManager_OnConnected() { + var connection = connectionManager.GetConnection(); + + //request quanta on rising + if (Context.NodesManager.CurrentNode.State == State.Rising) + { + _ = connection.SendMessage(new CatchupQuantaBatchRequest + { + LastKnownApex = Context.QuantumHandler.CurrentApex + }.CreateEnvelope()); + } + //subscribe for the current remote node signatures if (Context.NodesManager.CurrentNode.IsPrimeNode) { - var connection = connectionManager.GetConnection(); _ = connection.SendMessage(new SyncCursorReset { Cursors = new List { new SyncCursor { Type = XdrSyncCursorType.SingleNodeSignatures, - Cursor = Context.PendingUpdatesManager.LastPersistedApex } + Cursor = Context.PendingUpdatesManager.LastPersistedApex + } } }); } diff --git a/Centaurus.Domain/Nodes/RemoteNodes/RemoteNodeConnection.cs b/Centaurus.Domain/Nodes/RemoteNodes/RemoteNodeConnection.cs index 1536f6fc..945bba01 100644 --- a/Centaurus.Domain/Nodes/RemoteNodes/RemoteNodeConnection.cs +++ b/Centaurus.Domain/Nodes/RemoteNodes/RemoteNodeConnection.cs @@ -61,13 +61,6 @@ private void ConnectToAuditor() var connectionAttempts = 0; while (!isAborted) { - //TODO: remove this condition after refactoring result message broadcasting - //wait while all pending quanta will be handled - if (Context.NodesManager.CurrentNode.State == State.Rising) - { - Thread.Sleep(1000); - continue; - } lock (syncRoot) { if (isAborted) diff --git a/Centaurus.Domain/Quanta/Handlers/QuantumHandler.cs b/Centaurus.Domain/Quanta/Handlers/QuantumHandler.cs index b4594b0e..93085f01 100644 --- a/Centaurus.Domain/Quanta/Handlers/QuantumHandler.cs +++ b/Centaurus.Domain/Quanta/Handlers/QuantumHandler.cs @@ -16,7 +16,7 @@ public class QuantumHandler : ContextualBase public QuantumHandler(ExecutionContext context, ulong lastApex, byte[] lastQuantumHash) : base(context) { - CurrentApex = lastApex; + CurrentApex = LastAddedApex = lastApex; LastQuantumHash = lastQuantumHash ?? throw new ArgumentNullException(nameof(lastQuantumHash)); Task.Factory.StartNew(RunQuantumWorker).Unwrap(); } @@ -26,6 +26,8 @@ public QuantumHandler(ExecutionContext context, ulong lastApex, byte[] lastQuant object awaitedQuantaSyncRoot = new { }; Queue awaitedQuanta = new Queue(); + public ulong LastAddedApex { get; private set; } + /// /// Handles the quantum and returns Task. /// @@ -37,8 +39,7 @@ public QuantumProcessingItem HandleAsync(Quantum quantum, Task signatureVa && QuantaThrottlingManager.Current.MaxItemsPerSecond <= awaitedQuanta.Count) throw new TooManyRequestsException("Server is too busy. Try again later."); var processingItem = new QuantumProcessingItem(quantum, signatureValidation); - lock (awaitedQuantaSyncRoot) - awaitedQuanta.Enqueue(processingItem); + EnqueueHandlingItem(processingItem); return processingItem; } @@ -48,10 +49,24 @@ public QuantumProcessingItem HandleAsync(Quantum quantum, Task signatureVa public int QuantaQueueLenght => awaitedQuanta.Count; - private bool TryGetHandlingItem(out QuantumProcessingItem handlingItem) + private void EnqueueHandlingItem(QuantumProcessingItem processingItem) + { + lock (awaitedQuantaSyncRoot) + { + awaitedQuanta.Enqueue(processingItem); + if (processingItem.Apex > LastAddedApex) + LastAddedApex = processingItem.Apex; + } + } + + private bool TryDequeueHandlingItem(out QuantumProcessingItem handlingItem) { lock (awaitedQuantaSyncRoot) - return awaitedQuanta.TryDequeue(out handlingItem); + { + if (CurrentApex > LastAddedApex) + LastAddedApex = CurrentApex; + return awaitedQuanta.TryDequeue(out handlingItem); + } } private async Task RunQuantumWorker() @@ -60,7 +75,7 @@ private async Task RunQuantumWorker() { try { - if (TryGetHandlingItem(out var handlingItem)) + if (TryDequeueHandlingItem(out var handlingItem)) { //after Alpha switch, quanta queue can contain requests that should be send to new Alpha server if (!IsReadyToHandle(handlingItem)) diff --git a/Centaurus.Domain/Quanta/Sync/CursorGroup.cs b/Centaurus.Domain/Quanta/Sync/CursorGroup.cs index 6627e9ff..3d001f6b 100644 --- a/Centaurus.Domain/Quanta/Sync/CursorGroup.cs +++ b/Centaurus.Domain/Quanta/Sync/CursorGroup.cs @@ -20,6 +20,8 @@ public CursorGroup(SyncCursorType cursorType, ulong batchId) public DateTime LastUpdate { get; set; } + public ulong HighestApex { get; set; } + public List Nodes { get; } = new List(); } diff --git a/Centaurus.Domain/Quanta/Sync/SyncQuantaDataWorker.cs b/Centaurus.Domain/Quanta/Sync/SyncQuantaDataWorker.cs index c4d4b631..5ce7ef69 100644 --- a/Centaurus.Domain/Quanta/Sync/SyncQuantaDataWorker.cs +++ b/Centaurus.Domain/Quanta/Sync/SyncQuantaDataWorker.cs @@ -61,6 +61,8 @@ private void SetCursors(Dictionary cursors, RemoteNode node currentCursorGroup.Nodes.Add(new NodeCursorData(node, timeToken, cursor)); if (currentCursorGroup.LastUpdate == default || currentCursorGroup.LastUpdate > timeToken) currentCursorGroup.LastUpdate = timeToken; + if (cursor > currentCursorGroup.HighestApex) + currentCursorGroup.HighestApex = cursor; } const int ForceTimeOut = 100; @@ -96,21 +98,22 @@ private List> ProcessCursorGroup(CursorGroup cursorGr return sendingQuantaTasks; } - private static Task SendSingleBatch(SyncPortion batch, ulong currentCursor, SyncCursorType cursorType, NodeCursorData currentAuditor) + private static Task SendSingleBatch(SyncPortion batch, ulong currentCursor, SyncCursorType cursorType, NodeCursorData nodeCursor) { - var connection = currentAuditor.Node.GetConnection(); - if (currentAuditor.Cursor >= batch.LastDataApex || connection == null) + var connection = nodeCursor.Node.GetConnection(); + if (nodeCursor.Cursor >= batch.LastDataApex || connection == null) return null; var sendMessageTask = connection.SendMessage(batch.Data.AsMemory()); return sendMessageTask.ContinueWith(t => { if (t.IsFaulted) { - HandleFaultedSendTask(t, currentCursor, batch, cursorType, currentAuditor); + HandleFaultedSendTask(t, currentCursor, batch, cursorType, nodeCursor); return null; } - return new NodeSyncCursorUpdate(currentAuditor.Node, - new SyncCursorUpdate(currentAuditor.TimeToken, batch.LastDataApex, cursorType) + logger.Trace($"Cursor update for node {nodeCursor.Node}. Cursor type: {cursorType}, last apex: {batch.LastDataApex}"); + return new NodeSyncCursorUpdate(nodeCursor.Node, + new SyncCursorUpdate(nodeCursor.TimeToken, batch.LastDataApex, cursorType) ); }); } @@ -138,13 +141,12 @@ private bool TryGetBatch(CursorGroup cursorGroup, out SyncPortion batch) private bool ValidateCursorGroup(CursorGroup cursorGroup) { - var currentCursor = cursorGroup.BatchId; var currentApex = Context.QuantumHandler.CurrentApex; - if (currentCursor == currentApex) + if (cursorGroup.HighestApex == currentApex) return false; - if (currentCursor > currentApex && GetIsCurrentNodeReady()) + if (cursorGroup.HighestApex > currentApex && GetIsCurrentNodeReady()) { - var message = $"Node(s) {string.Join(',', cursorGroup.Nodes.Select(a => a.Node.AccountId))} is above current constellation state, cursor {currentCursor}, current apex {currentApex} type {cursorGroup.CursorType}."; + var message = $"Node(s) {string.Join(',', cursorGroup.Nodes.Select(a => a.Node.AccountId))} is above current constellation state, the nodes group highest apex is {cursorGroup.HighestApex}, current apex {currentApex} type {cursorGroup.CursorType}."; if (cursorGroup.Nodes.Count >= Context.GetMajorityCount() - 1) //-1 is current server logger.Error(message); else diff --git a/Centaurus.Domain/Quanta/Sync/SyncStorage.cs b/Centaurus.Domain/Quanta/Sync/SyncStorage.cs index 5fc67777..182cc560 100644 --- a/Centaurus.Domain/Quanta/Sync/SyncStorage.cs +++ b/Centaurus.Domain/Quanta/Sync/SyncStorage.cs @@ -44,7 +44,7 @@ public SyncPortion GetQuanta(ulong from, bool force) { var quantaBatchStart = GetBatchApexStart(from + 1); var batch = GetBatch(quantaBatchStart); - + return batch.Quanta.GetData(from, force); } @@ -255,10 +255,13 @@ private SyncStorageItem LoadBatch(ulong batchStartApex, int batchSize) var quanta = new List(); var majoritySignatures = new List(); var currentNodeSignatures = new List(); + var currentPubKey = (RawPubKey)Context.Settings.KeyPair; foreach (var rawQuantum in rawQuanta) { quanta.Add(rawQuantum.ToBatchItemQuantum()); - majoritySignatures.Add(rawQuantum.ToMajoritySignatures()); + var signatures = rawQuantum.ToMajoritySignatures(); + majoritySignatures.Add(signatures); + currentNodeSignatures.Add(GetCurrentNodeSignature(currentPubKey, rawQuantum, signatures)); } if (batchStartApex == 0) //insert null at 0 position, otherwise index will not be relevant to apex @@ -271,10 +274,31 @@ private SyncStorageItem LoadBatch(ulong batchStartApex, int batchSize) return new SyncStorageItem(Context, batchStartApex, PortionSize, quanta, majoritySignatures, currentNodeSignatures); } - class SyncStorageItem: ContextualBase + private SingleNodeSignaturesBatchItem GetCurrentNodeSignature(RawPubKey currentPubKey, QuantumPersistentModel rawQuantum, MajoritySignaturesBatchItem signatures) + { + try + { + if (!Context.ConstellationSettingsManager.TryGetForApex(rawQuantum.Apex, out var constellation)) + throw new Exception($"Unable to get constellation for apex {rawQuantum.Apex}"); + + var nodeId = constellation.GetNodeId(currentPubKey); + var currentNodeSignature = signatures.Signatures.FirstOrDefault(s => s.NodeId == nodeId); + if (currentNodeSignature == null) + throw new Exception("Unable to find signature for current node."); + + return new SingleNodeSignaturesBatchItem { Apex = rawQuantum.Apex, Signature = currentNodeSignature }; + } + catch (Exception exc) + { + Context.NodesManager.CurrentNode.Failed(exc); + throw; + } + } + + class SyncStorageItem : ContextualBase { public SyncStorageItem(ExecutionContext context, ulong batchId, int portionSize, List quanta, List majoritySignatures, List signatures) - :base(context) + : base(context) { BatchStart = batchId; BatchEnd = batchId + BatchSize - 1; //batch start is inclusive diff --git a/Centaurus.Domain/ResultManagers/ResultManager.cs b/Centaurus.Domain/ResultManagers/ResultManager.cs index 6abce776..e76a936c 100644 --- a/Centaurus.Domain/ResultManagers/ResultManager.cs +++ b/Centaurus.Domain/ResultManagers/ResultManager.cs @@ -245,6 +245,7 @@ public Aggregate(ulong apex, ResultManager manager) private HashSet processedAuditors = new HashSet(); private List signatures = new List(); private List outrunResults = new List(); + private ConstellationSettings constellation; public List GetSignatures() { @@ -269,8 +270,7 @@ public void Add(NodeSignatureInternal signature) } //obtain auditor from constellation - if (!(Manager.Context.ConstellationSettingsManager.TryGetForApex(Apex, out var constellation) - && constellation.TryGetNodePubKey((byte)signature.NodeId, out var nodePubKey))) + if (!(constellation.TryGetNodePubKey((byte)signature.NodeId, out var nodePubKey))) return; //check if signature is valid and tx signature is presented for TxResultMessage @@ -306,6 +306,11 @@ public void Add(QuantumProcessingItem quantumProcessingItem) { lock (syncRoot) { + if (!Manager.Context.ConstellationSettingsManager.TryGetForApex(Apex, out constellation)) + { + Manager.Context.NodesManager.CurrentNode.Failed(new Exception($"Unable to get constellation settings for apex {Apex}.")); + return; + } ProcessingItem = quantumProcessingItem; var signature = AddCurrentNodeSignature(); //mark as acknowledged diff --git a/Centaurus/Startup.cs b/Centaurus/Startup.cs index 8ef90ce6..6ade8ac7 100644 --- a/Centaurus/Startup.cs +++ b/Centaurus/Startup.cs @@ -37,14 +37,18 @@ public void Run(ManualResetEvent resetEvent) private void Context_OnComplete() { + isContextStopped = true; var isSet = resetEvent.WaitOne(0); if (!isSet) resetEvent.Set(); } + bool isContextStopped; + public void Shutdown() { - Context.Complete(); + if (!isContextStopped) + Context.Complete(); if (AlphaStartup != null) AlphaStartup.Shutdown(); } From 2533611d05f10eaa4489e834ea469042df9ad10b Mon Sep 17 00:00:00 2001 From: hawthorne-abendsen <49230725+hawthorne-abendsen@users.noreply.github.com> Date: Wed, 22 Dec 2021 19:51:32 +0200 Subject: [PATCH 07/13] Constellation update refactoring; providers manager refactoring --- Centaurus.Domain/Catchups/Catchup.cs | 44 ++++--- .../Contexts/ConstellationSettingsManager.cs | 4 +- Centaurus.Domain/Contexts/ExecutionContext.cs | 71 +++++------- .../Helpers/PaymentProviderExtensions.cs | 31 +++++ .../Nodes/Managers/NodesManager.cs | 2 +- .../PaymentProvidersManager.cs | 109 +++++++++++++----- .../Quanta/Processors/AlphaUpdateProcessor.cs | 8 +- .../ConstellationQuantumProcessorBase.cs | 23 ++++ .../ConstellationUpdateProcessor.cs | 49 +++++--- .../ConstellationUpdateProcessorBase.cs | 31 +++++ .../Quanta/Processors/DepositProcessor.cs | 1 - .../Processors/OrderRequestProcessor.cs | 7 +- .../Payments/PaymentRequestProcessor.cs | 5 +- .../Payments/WithdrawalRequestProcessor.cs | 2 +- .../ResultManagers/ResultManager.cs | 2 +- 15 files changed, 268 insertions(+), 121 deletions(-) create mode 100644 Centaurus.Domain/Helpers/PaymentProviderExtensions.cs create mode 100644 Centaurus.Domain/Quanta/Processors/ConstellationQuantumProcessorBase.cs create mode 100644 Centaurus.Domain/Quanta/Processors/ConstellationUpdateProcessorBase.cs diff --git a/Centaurus.Domain/Catchups/Catchup.cs b/Centaurus.Domain/Catchups/Catchup.cs index ea25f769..1c537957 100644 --- a/Centaurus.Domain/Catchups/Catchup.cs +++ b/Centaurus.Domain/Catchups/Catchup.cs @@ -33,10 +33,10 @@ public async Task AddNodeBatch(RawPubKey pubKey, CatchupQuantaBatch nodeBatch) await semaphoreSlim.WaitAsync(); try { - logger.Trace($"Auditor state from {pubKey.GetAccountId()} received by AlphaCatchup."); + logger.Trace($"Catchup quanta batch from {pubKey.GetAccountId()} received by Catchup."); if (Context.NodesManager.CurrentNode.State != State.Rising) { - logger.Warn($"Auditor state messages can be only handled when Alpha is in rising state. State sent by {((KeyPair)pubKey).AccountId}"); + logger.Warn($"Catchup quanta batch messages can be only handled when Alpha is in rising state. State sent by {((KeyPair)pubKey).AccountId}"); return; } @@ -125,7 +125,7 @@ private async Task TryApplyAuditorsData() int majority = Context.GetMajorityCount(), totalAuditorsCount = Context.GetTotalAuditorsCount(); - if (Context.HasMajority(validAuditorStates.Count) || !Context.NodesManager.IsAlpha) + if (Context.HasMajority(validAuditorStates.Count)) { var validQuanta = GetValidQuanta(); await ApplyQuanta(validQuanta); @@ -216,7 +216,8 @@ private List GetValidQuanta() if (lastQuantumApex + 1 != currentQuantaGroup.Key) throw new Exception("A quantum is missing"); - var validatedQuantumData = GetQuantumData(currentQuantaGroup.Key, currentQuantaGroup.ToList(), auditorsSettings.alphaId, auditorsSettings.auditors); + if (!TryGetQuantumData(currentQuantaGroup.Key, currentQuantaGroup.ToList(), auditorsSettings, out var validatedQuantumData)) //if we unable to get quanta with majority, than stop processing newer quanta + break; validQuanta.Add(validatedQuantumData); @@ -233,40 +234,45 @@ private List GetValidQuanta() return validQuanta; } - private (int alphaId, List auditors) GetAuditorsSettings(ConstellationSettings constellation) + private List GetAuditorsSettings(ConstellationSettings constellation) { if (Context.NodesManager.AlphaNode == null) throw new ArgumentNullException("Alpha node is not connected."); - return (Context.NodesManager.AlphaNode.Id, Context.NodesManager.AllNodes.Select(n => n.PubKey).ToList()); + return constellation.Auditors.Select(n => n.PubKey).ToList(); } - private ValidatedQuantumData GetQuantumData(ulong apex, List allApexQuanta, int alphaId, List auditors) + private bool TryGetQuantumData(ulong apex, List allApexQuanta, List auditors, out ValidatedQuantumData validatedQuantumData) { + validatedQuantumData = null; //compute and group quanta by payload hash var grouped = allApexQuanta .GroupBy(q => ((Quantum)q.Quantum).GetPayloadHash(), ByteArrayComparer.Default); //validate each quantum data group - var validatedQuantaData = new List(); - foreach (var quantaGroup in grouped) - { - validatedQuantaData.Add(ProcessQuantumGroup(auditors, quantaGroup)); - } + var validatedQuantaData = GetValidatedQuantaData(auditors, grouped); + //get majority for current auditors set var majorityCount = MajorityHelper.GetMajorityCount(auditors.Count); //get all quanta data with majority - var quantaDataWithMajority = validatedQuantaData.Where(a => a.Signatures.Count > majorityCount).ToList(); + var quantaDataWithMajority = validatedQuantaData.Where(a => a.Signatures.Count >= majorityCount).ToList(); if (quantaDataWithMajority.Count > 1) + { + //throw exception. The case must be investigated throw new Exception($"Conflict found for apex {apex}. {quantaDataWithMajority.Count} quanta data sets with majority."); - else if (quantaDataWithMajority.Count == 0 && validatedQuantaData.Count > 1) - throw new Exception($"Conflict found for apex {apex}. {validatedQuantaData.Count} quanta data sets, but no majority."); + } //if we have quanta data with majority, return it - if (quantaDataWithMajority.Count > 0) - return quantaDataWithMajority.First(); - else - return validatedQuantaData.First(); + validatedQuantumData = quantaDataWithMajority.FirstOrDefault(); + return validatedQuantumData != null; + } + + private List GetValidatedQuantaData(List auditors, IEnumerable> grouped) + { + var validatedQuantaData = new List(); + foreach (var quantaGroup in grouped) + validatedQuantaData.Add(ProcessQuantumGroup(auditors, quantaGroup)); + return validatedQuantaData; } private ValidatedQuantumData ProcessQuantumGroup(List auditors, IGrouping quantaGroup) diff --git a/Centaurus.Domain/Contexts/ConstellationSettingsManager.cs b/Centaurus.Domain/Contexts/ConstellationSettingsManager.cs index ae766ac5..e9b90e04 100644 --- a/Centaurus.Domain/Contexts/ConstellationSettingsManager.cs +++ b/Centaurus.Domain/Contexts/ConstellationSettingsManager.cs @@ -16,10 +16,12 @@ public class ConstellationSettingsManager : ContextualBase, IDisposable { private ConstellationSettingsCollection settingsCache; - public ConstellationSettingsManager(ExecutionContext context) + public ConstellationSettingsManager(ExecutionContext context, ConstellationSettings currentSettings) : base(context) { settingsCache = new ConstellationSettingsCollection(Context.DataProvider.GetConstellationSettings); + if (currentSettings != null) + Update(currentSettings); } public ConstellationSettings Current { get; private set; } diff --git a/Centaurus.Domain/Contexts/ExecutionContext.cs b/Centaurus.Domain/Contexts/ExecutionContext.cs index 3f99278d..c4768683 100644 --- a/Centaurus.Domain/Contexts/ExecutionContext.cs +++ b/Centaurus.Domain/Contexts/ExecutionContext.cs @@ -69,9 +69,10 @@ internal ExecutionContext( Catchup = new Catchup(this); - ConstellationSettingsManager = new ConstellationSettingsManager(this); - var persistentData = DataProvider.GetPersistentData(); + + ConstellationSettingsManager = new ConstellationSettingsManager(this, persistentData.snapshot?.ConstellationSettings); + var currentState = persistentData == default ? State.WaitingForInit : State.Rising; NodesManager = new NodesManager(this, currentState); NodesManager.CurrentNode.StateChanged += CurrentNode_StateChanged; @@ -89,17 +90,33 @@ internal ExecutionContext( PerformanceStatisticsManager = new PerformanceStatisticsManager(this); + SyncStorage = new SyncStorage(this, lastApex); + + SyncQuantaDataWorker = new SyncQuantaDataWorker(this); + + PaymentProvidersManager = new PaymentProvidersManager(PaymentProviderFactory, GetAbsolutePath(Settings.PaymentConfigPath)); + PaymentProvidersManager.OnRegistered += PaymentProvidersManager_OnRegistered; + PaymentProvidersManager.OnRemoved += PaymentProvidersManager_OnRemoved; + + SetNodes().Wait(); + //apply snapshot if not null if (persistentData.snapshot != null) Init(persistentData.snapshot); - else - SetNodes().Wait(); HandlePendingQuanta(persistentData.pendingQuanta); + } - SyncStorage = new SyncStorage(this, lastApex); + private void PaymentProvidersManager_OnRemoved(PaymentProviderBase provider) + { + provider.OnPaymentCommit -= PaymentProvider_OnPaymentCommit; + provider.OnError -= PaymentProvider_OnError; + } - SyncQuantaDataWorker = new SyncQuantaDataWorker(this); + private void PaymentProvidersManager_OnRegistered(PaymentProviderBase provider) + { + provider.OnPaymentCommit += PaymentProvider_OnPaymentCommit; + provider.OnError += PaymentProvider_OnError; } private void CurrentNode_StateChanged(StateChangedEventArgs stateChangedEventArgs) @@ -133,13 +150,11 @@ private string GetAbsolutePath(string path) public void Init(Snapshot snapshot) { - UpdateConstellationSettings(snapshot.ConstellationSettings).Wait(); - AccountStorage = new AccountStorage(snapshot.Accounts); Exchange = Exchange.RestoreExchange(ConstellationSettingsManager.Current.Assets, snapshot.Orders, Settings.IsPrimeNode); - SetupPaymentProviders(snapshot.Cursors); + PaymentProvidersManager.RegisterProviders(ConstellationSettingsManager.Current, snapshot.Cursors); AnalyticsManager = new AnalyticsManager( PersistentStorage, @@ -156,15 +171,6 @@ public void Init(Snapshot snapshot) Exchange.OnUpdates += Exchange_OnUpdates; } - public async Task UpdateConstellationSettings(ConstellationSettings constellationSettings) - { - if (constellationSettings != null) - ConstellationSettingsManager.Update(constellationSettings); - - //update current auditors - await SetNodes(); - } - public void Complete() { NodesManager.CurrentNode.Stopped(); @@ -206,28 +212,7 @@ private async Task SetNodes() var auditors = ConstellationSettingsManager.Current != null ? ConstellationSettingsManager.Current.Auditors.ToList() : Settings.GenesisAuditors.Select(a => new Auditor { PubKey = a.PubKey, Address = a.Address }).ToList(); - await NodesManager.SetAuditors(auditors); - } - - private void SetupPaymentProviders(Dictionary cursors) - { - PaymentProvidersManager?.Dispose(); - - var settings = ConstellationSettingsManager.Current.Providers.Select(p => - { - var providerId = PaymentProviderBase.GetProviderId(p.Provider, p.Name); - cursors.TryGetValue(providerId, out var cursor); - var settings = p.ToProviderModel(cursor); - return settings; - }).ToList(); - - PaymentProvidersManager = new PaymentProvidersManager(PaymentProviderFactory, settings, GetAbsolutePath(Settings.PaymentConfigPath)); - - foreach (var paymentProvider in PaymentProvidersManager.GetAll()) - { - paymentProvider.OnPaymentCommit += PaymentProvider_OnPaymentCommit; - paymentProvider.OnError += PaymentProvider_OnError; - } + await NodesManager.SetNodes(auditors); } private void PaymentProvider_OnPaymentCommit(PaymentProviderBase paymentProvider, PaymentProvider.Models.DepositNotificationModel notification) @@ -246,7 +231,9 @@ private void PaymentProvider_OnError(PaymentProviderBase paymentProvider, Except public void Dispose() { ExtensionsManager?.Dispose(); - PaymentProvidersManager?.Dispose(); + PaymentProvidersManager.Dispose(); + + SyncQuantaDataWorker.Dispose(); ResultManager.Dispose(); DisposeAnalyticsManager(); @@ -297,7 +284,7 @@ public void Dispose() public MessageHandlers MessageHandlers { get; } - public PaymentProvidersManager PaymentProvidersManager { get; private set; } + public PaymentProvidersManager PaymentProvidersManager { get; } public Exchange Exchange { get; private set; } diff --git a/Centaurus.Domain/Helpers/PaymentProviderExtensions.cs b/Centaurus.Domain/Helpers/PaymentProviderExtensions.cs new file mode 100644 index 00000000..3de6a589 --- /dev/null +++ b/Centaurus.Domain/Helpers/PaymentProviderExtensions.cs @@ -0,0 +1,31 @@ +using Centaurus.Models; +using Centaurus.PaymentProvider; +using Centaurus.PaymentProvider.Models; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Centaurus.Domain +{ + public static class PaymentProviderExtensions + { + public static void RegisterProviders(this PaymentProvidersManager providersManager, ConstellationSettings constellation, Dictionary cursors) + { + var settings = constellation.Providers.Select(p => + { + var providerId = PaymentProviderBase.GetProviderId(p.Provider, p.Name); + cursors.TryGetValue(providerId, out var cursor); + var settings = p.ToProviderModel(cursor); + return settings; + }).ToList(); + + foreach (var providerSettings in settings) + { + if (providersManager.TryGetManager(providerSettings.Id, out _)) //already registered + continue; + providersManager.Register(providerSettings); + } + } + } +} diff --git a/Centaurus.Domain/Nodes/Managers/NodesManager.cs b/Centaurus.Domain/Nodes/Managers/NodesManager.cs index 78276e9f..c29aeb3f 100644 --- a/Centaurus.Domain/Nodes/Managers/NodesManager.cs +++ b/Centaurus.Domain/Nodes/Managers/NodesManager.cs @@ -54,7 +54,7 @@ public List GetRemoteNodes() return nodes.GetAllNodes(); } - public async Task SetAuditors(List auditors) + public async Task SetNodes(List auditors) { var invalidatedNodes = await UpdateNodes(auditors); RemoveNodes(invalidatedNodes); diff --git a/Centaurus.Domain/PaymentProvider/PaymentProvidersManager.cs b/Centaurus.Domain/PaymentProvider/PaymentProvidersManager.cs index 23fd0f11..1bae28a2 100644 --- a/Centaurus.Domain/PaymentProvider/PaymentProvidersManager.cs +++ b/Centaurus.Domain/PaymentProvider/PaymentProvidersManager.cs @@ -11,38 +11,26 @@ namespace Centaurus.Domain { public class PaymentProvidersManager : IDisposable { - public PaymentProvidersManager(PaymentProvidersFactoryBase paymentProviderFactory, List settings, string configPath) + public PaymentProvidersManager(PaymentProvidersFactoryBase paymentProviderFactory, string configPath) { - var config = GetConfig(configPath); - var providers = new Dictionary(); - foreach (var provider in settings) - { - if (providers.ContainsKey(provider.Id)) - throw new Exception($"Payments manager for provider {provider.Id} is already registered."); - - var currentProviderRawConfig = default(string); - var assemblyPath = default(string); - if (config != null && config.RootElement.TryGetProperty(provider.Id, out var currentProviderElement)) - { - if (!currentProviderElement.TryGetProperty("assemblyPath", out var assemblyPathProperty)) - throw new ArgumentNullException("Path property is missing."); - assemblyPath = assemblyPathProperty.GetString(); + this.configPath = configPath; + this.paymentProviderFactory = paymentProviderFactory; + } - if (currentProviderElement.TryGetProperty("config", out var providerConfig)) - currentProviderRawConfig = providerConfig.GetRawText(); - } + public event Action OnRegistered; - providers.Add(provider.Id, paymentProviderFactory.GetProvider(provider, assemblyPath, currentProviderRawConfig)); - } + public event Action OnRemoved; - paymentProviders = providers.ToImmutableDictionary(); + public void Register(SettingsModel settings) + { + var config = GetConfig(configPath); + RegisterProvider(settings, config); } - private ImmutableDictionary paymentProviders; - public bool TryGetManager(string provider, out PaymentProviderBase paymentProvider) { - return paymentProviders.TryGetValue(provider, out paymentProvider); + lock (syncRoot) + return paymentProviders.TryGetValue(provider, out paymentProvider); } public PaymentProviderBase GetManager(string provider) @@ -54,7 +42,42 @@ public PaymentProviderBase GetManager(string provider) public List GetAll() { - return paymentProviders.Values.ToList(); + lock (syncRoot) + return paymentProviders.Values.ToList(); + } + + public void Dispose() + { + RemoveAndDisposeAllProviders(); + } + + private Dictionary paymentProviders = new Dictionary(); + private string configPath; + private PaymentProvidersFactoryBase paymentProviderFactory; + + private void RegisterProvider(SettingsModel settings, JsonDocument config) + { + if (TryGetManager(settings.Id, out _)) + throw new Exception($"Provider {settings.Id} already registered."); + var provider = GetProvider(config, settings); + AddProvider(provider); + } + + private PaymentProviderBase GetProvider(JsonDocument config, SettingsModel settings) + { + var currentProviderRawConfig = default(string); + var assemblyPath = default(string); + if (config != null && config.RootElement.TryGetProperty(settings.Id, out var currentProviderElement)) + { + if (!currentProviderElement.TryGetProperty("assemblyPath", out var assemblyPathProperty)) + throw new ArgumentNullException("Path property is missing."); + assemblyPath = assemblyPathProperty.GetString(); + + if (currentProviderElement.TryGetProperty("config", out var providerConfig)) + currentProviderRawConfig = providerConfig.GetRawText(); + } + + return paymentProviderFactory.GetProvider(settings, assemblyPath, currentProviderRawConfig); } private JsonDocument GetConfig(string configPath) @@ -64,10 +87,40 @@ private JsonDocument GetConfig(string configPath) return JsonDocument.Parse(File.ReadAllText(configPath)); } - public void Dispose() + private void RemoveAndDisposeAllProviders() { - foreach (var txM in paymentProviders.Values) - txM.Dispose(); + if (paymentProviders == null) + return; + lock (syncRoot) + { + var providerIds = paymentProviders.Keys.ToList(); + foreach (var providerId in providerIds) + RemoveAndDisposeProvider(providerId); + } } + + private void RemoveAndDisposeProvider(string providerId) + { + lock (syncRoot) + { + if (!paymentProviders.Remove(providerId, out var provider)) + return; + OnRemoved?.Invoke(provider); + provider.Dispose(); + } + } + + private void AddProvider(PaymentProviderBase provider) + { + lock (syncRoot) + { + if (paymentProviders.ContainsKey(provider.Id)) + throw new Exception($"Payments manager for provider {provider.Id} is already registered."); + paymentProviders.Add(provider.Id, provider); + OnRegistered?.Invoke(provider); + } + } + + private object syncRoot = new { }; } } diff --git a/Centaurus.Domain/Quanta/Processors/AlphaUpdateProcessor.cs b/Centaurus.Domain/Quanta/Processors/AlphaUpdateProcessor.cs index f280d571..44023f90 100644 --- a/Centaurus.Domain/Quanta/Processors/AlphaUpdateProcessor.cs +++ b/Centaurus.Domain/Quanta/Processors/AlphaUpdateProcessor.cs @@ -9,7 +9,7 @@ namespace Centaurus.Domain { - public class AlphaUpdateProcessor : QuantumProcessorBase + public class AlphaUpdateProcessor : ConstellationUpdateProcessorBase { public AlphaUpdateProcessor(ExecutionContext context) : base(context) @@ -29,9 +29,7 @@ public override async Task Process(QuantumProcessingIt newConstellationSettings.Apex = processingItem.Apex; newConstellationSettings.Alpha = alphaUpdate.Alpha; - processingItem.AddConstellationUpdate(Context.ConstellationSettingsManager.Current, Context.ConstellationSettingsManager.Current); - - await Context.UpdateConstellationSettings(newConstellationSettings); + await UpdateConstellationSettings(processingItem, newConstellationSettings); return (QuantumResultMessageBase)processingItem.Quantum.CreateEnvelope().CreateResult(ResultStatusCode.Success); } @@ -42,7 +40,7 @@ public override Task Validate(QuantumProcessingItem processingItem) if (currentState == State.Undefined || currentState == State.WaitingForInit) throw new InvalidOperationException($"ConstellationSettingsManager.Current is not initialized yet."); - ((ConstellationQuantum)processingItem.Quantum).Validate(Context); + base.Validate(processingItem); var alphaUpdate = (AlphaUpdate)((ConstellationQuantum)processingItem.Quantum).RequestMessage; diff --git a/Centaurus.Domain/Quanta/Processors/ConstellationQuantumProcessorBase.cs b/Centaurus.Domain/Quanta/Processors/ConstellationQuantumProcessorBase.cs new file mode 100644 index 00000000..f1af1aba --- /dev/null +++ b/Centaurus.Domain/Quanta/Processors/ConstellationQuantumProcessorBase.cs @@ -0,0 +1,23 @@ +using Centaurus.Models; +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; + +namespace Centaurus.Domain +{ + public abstract class ConstellationQuantumProcessorBase : QuantumProcessorBase + { + public ConstellationQuantumProcessorBase(ExecutionContext context) + :base(context) + { + + } + + public override Task Validate(QuantumProcessingItem processingItem) + { + ((ConstellationQuantum)processingItem.Quantum).Validate(Context); + return Task.CompletedTask; + } + } +} diff --git a/Centaurus.Domain/Quanta/Processors/ConstellationUpdateProcessor.cs b/Centaurus.Domain/Quanta/Processors/ConstellationUpdateProcessor.cs index 9cc6ae5e..a5c0a5cf 100644 --- a/Centaurus.Domain/Quanta/Processors/ConstellationUpdateProcessor.cs +++ b/Centaurus.Domain/Quanta/Processors/ConstellationUpdateProcessor.cs @@ -8,7 +8,7 @@ namespace Centaurus.Domain { - public class ConstellationUpdateProcessor : QuantumProcessorBase + public class ConstellationUpdateProcessor : ConstellationUpdateProcessorBase { public ConstellationUpdateProcessor(ExecutionContext context) : base(context) @@ -18,29 +18,36 @@ public ConstellationUpdateProcessor(ExecutionContext context) public override string SupportedMessageType { get; } = typeof(ConstellationUpdate).Name; - public override Task Process(QuantumProcessingItem processingItem) + public override async Task Process(QuantumProcessingItem processingItem) { var updateQuantum = (ConstellationUpdate)((ConstellationQuantum)processingItem.Quantum).RequestMessage; var settings = updateQuantum.ToConstellationSettings(processingItem.Apex); - processingItem.AddConstellationUpdate(settings, Context.ConstellationSettingsManager.Current); - - var updateSnapshot = settings.ToSnapshot( - processingItem.Apex, - Context.AccountStorage?.GetAll().ToList() ?? new List(), - Context.Exchange?.OrderMap.GetAllOrders().ToList() ?? new List(), - GetCursors(settings.Providers), - processingItem.Quantum.ComputeHash() - ); - Context.Init(updateSnapshot); + await UpdateConstellationSettings(processingItem, settings); var currentNode = Context.NodesManager.CurrentNode; - //if state is undefined, than we need to init it + var cursors = GetCursors(settings.Providers); + //if state is WaitingForInit, than we need to init it if (currentNode.State == State.WaitingForInit) + { + var updateSnapshot = settings.ToSnapshot( + processingItem.Apex, + Context.AccountStorage?.GetAll().ToList() ?? new List(), + Context.Exchange?.OrderMap.GetAllOrders().ToList() ?? new List(), + cursors, + processingItem.Quantum.ComputeHash() + ); + + Context.Init(updateSnapshot); currentNode.Init(State.Running); + } + else + { + Context.PaymentProvidersManager.RegisterProviders(settings, cursors); + } - return Task.FromResult((QuantumResultMessageBase)processingItem.Quantum.CreateEnvelope().CreateResult(ResultStatusCode.Success)); + return (QuantumResultMessageBase)processingItem.Quantum.CreateEnvelope().CreateResult(ResultStatusCode.Success); } private Dictionary GetCursors(List providers) @@ -63,7 +70,9 @@ private Dictionary GetCursors(List providers) public override Task Validate(QuantumProcessingItem processingItem) { - ((ConstellationQuantum)processingItem.Quantum).Validate(Context); + base.Validate(processingItem); + + var currentConstellation = Context.ConstellationSettingsManager.Current; var requestEnvelope = ((ConstellationQuantum)processingItem.Quantum).RequestEnvelope; @@ -73,7 +82,7 @@ public override Task Validate(QuantumProcessingItem processingItem) if (constellationUpdate.Auditors == null || constellationUpdate.Auditors.Count() < minAuditorsCount) throw new ArgumentException($"Min auditors count is {minAuditorsCount}"); - if (constellationUpdate.LastUpdateApex != (Context.ConstellationSettingsManager.Current?.Apex ?? 0ul)) + if (constellationUpdate.LastUpdateApex != (currentConstellation?.Apex ?? 0ul)) throw new InvalidOperationException($"Last update apex is invalid."); if (!constellationUpdate.Auditors.All(a => a.Address == null || Uri.TryCreate($"http://{a.Address}", UriKind.Absolute, out _))) @@ -101,6 +110,14 @@ public override Task Validate(QuantumProcessingItem processingItem) if (constellationUpdate.RequestRateLimits == null || constellationUpdate.RequestRateLimits.HourLimit < 1 || constellationUpdate.RequestRateLimits.MinuteLimit < 1) throw new ArgumentException("Request rate limit values should be greater than 0"); + if (currentConstellation != null) + { + if (!currentConstellation.Assets.All(a => constellationUpdate.Assets.Any(cua => cua.Code == a.Code))) + throw new ArgumentException("Constellation update doesn't contain one or more assets."); + + if (!currentConstellation.Providers.All(p => constellationUpdate.Providers.Any(cua => cua.Name == p.Name && cua.Provider == p.Provider))) + throw new ArgumentException("Constellation update doesn't contain one or more payment provider."); + } return Task.CompletedTask; } } diff --git a/Centaurus.Domain/Quanta/Processors/ConstellationUpdateProcessorBase.cs b/Centaurus.Domain/Quanta/Processors/ConstellationUpdateProcessorBase.cs new file mode 100644 index 00000000..193837dd --- /dev/null +++ b/Centaurus.Domain/Quanta/Processors/ConstellationUpdateProcessorBase.cs @@ -0,0 +1,31 @@ +using Centaurus.Models; +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; + +namespace Centaurus.Domain +{ + public abstract class ConstellationUpdateProcessorBase: ConstellationQuantumProcessorBase + { + public ConstellationUpdateProcessorBase(ExecutionContext context) + :base(context) + { + } + + /// + /// Registers new constellation settings and updates nodes + /// + /// + /// + /// + protected async Task UpdateConstellationSettings(QuantumProcessingItem processingItem, ConstellationSettings newSettings) + { + processingItem.AddConstellationUpdate(newSettings, Context.ConstellationSettingsManager.Current); + + Context.ConstellationSettingsManager.Update(newSettings); + + await Context.NodesManager.SetNodes(newSettings.Auditors); + } + } +} diff --git a/Centaurus.Domain/Quanta/Processors/DepositProcessor.cs b/Centaurus.Domain/Quanta/Processors/DepositProcessor.cs index ab132fd0..11ef5f16 100644 --- a/Centaurus.Domain/Quanta/Processors/DepositProcessor.cs +++ b/Centaurus.Domain/Quanta/Processors/DepositProcessor.cs @@ -12,7 +12,6 @@ public class DepositProcessor : QuantumProcessorBase public DepositProcessor(ExecutionContext context) : base(context) { - } public override string SupportedMessageType { get; } = typeof(DepositQuantum).Name; diff --git a/Centaurus.Domain/Quanta/Processors/OrderRequestProcessor.cs b/Centaurus.Domain/Quanta/Processors/OrderRequestProcessor.cs index 34dc55ec..e16692ae 100644 --- a/Centaurus.Domain/Quanta/Processors/OrderRequestProcessor.cs +++ b/Centaurus.Domain/Quanta/Processors/OrderRequestProcessor.cs @@ -12,7 +12,6 @@ public class OrderRequestProcessor : QuantumProcessorBase public OrderRequestProcessor(ExecutionContext context) :base(context) { - } public override string SupportedMessageType { get; } = typeof(OrderRequest).Name; @@ -43,8 +42,8 @@ public override Task Validate(QuantumProcessingItem processingItem) throw new BadRequestException("Order asset must be different from quote asset."); var orderAsset = Context.ConstellationSettingsManager.Current.Assets.FirstOrDefault(a => a.Code == orderRequest.Asset); - if (orderAsset == null) - throw new BadRequestException("Invalid asset identifier: " + orderRequest.Asset); + if (orderAsset == null || orderAsset.IsSuspended) + throw new BadRequestException($"Asset {orderRequest.Asset} is not supported or suspended."); if (processingItem.Initiator.Orders.Any(o => o.Value.Asset == orderRequest.Asset && o.Value.Side != orderRequest.Side)) throw new BadRequestException("You cannot place order that crosses own order."); @@ -91,7 +90,7 @@ private void ValidateCounterOrdersCount(OrderRequest orderRequest, Orderbook ord if (counterOrdersSum >= orderRequest.Amount) break; if (counterOrdersCount > MaxCrossOrdersCount) - throw new BadRequestException("Failed to execute order. Maximum crossed orders length exceeded"); + throw new BadRequestException("Failed to execute order. Maximum crossed orders length exceeded."); } } } diff --git a/Centaurus.Domain/Quanta/Processors/Payments/PaymentRequestProcessor.cs b/Centaurus.Domain/Quanta/Processors/Payments/PaymentRequestProcessor.cs index 20d9e025..16f3826a 100644 --- a/Centaurus.Domain/Quanta/Processors/Payments/PaymentRequestProcessor.cs +++ b/Centaurus.Domain/Quanta/Processors/Payments/PaymentRequestProcessor.cs @@ -67,8 +67,9 @@ public override Task Validate(QuantumProcessingItem quantumProcessingItem) if (payment.Amount <= 0) throw new BadRequestException("Amount should be greater than 0"); - if (!Context.ConstellationSettingsManager.Current.Assets.Any(a => a.Code == payment.Asset)) - throw new BadRequestException($"Asset {payment.Asset} is not supported"); + var paymentAsset = Context.ConstellationSettingsManager.Current.Assets.FirstOrDefault(a => a.Code == payment.Asset); + if (paymentAsset == null || paymentAsset.IsSuspended) + throw new BadRequestException($"Asset {payment.Asset} is not supported or suspended."); var minBalance = payment.Asset == baseAsset ? Context.ConstellationSettingsManager.Current.MinAccountBalance : 0; if (!(quantumProcessingItem.Initiator.GetBalance(payment.Asset)?.HasSufficientBalance(payment.Amount, minBalance) ?? false)) diff --git a/Centaurus.Domain/Quanta/Processors/Payments/WithdrawalRequestProcessor.cs b/Centaurus.Domain/Quanta/Processors/Payments/WithdrawalRequestProcessor.cs index d50cbe78..6d1ae655 100644 --- a/Centaurus.Domain/Quanta/Processors/Payments/WithdrawalRequestProcessor.cs +++ b/Centaurus.Domain/Quanta/Processors/Payments/WithdrawalRequestProcessor.cs @@ -37,7 +37,7 @@ public override Task Validate(QuantumProcessingItem quantumProcessingItem) var withdrawalRequest = (WithdrawalRequest)withdrawalQuantum.RequestMessage; var centaurusAsset = Context.ConstellationSettingsManager.Current.Assets.FirstOrDefault(a => a.Code == withdrawalRequest.Asset); - if (centaurusAsset == null || centaurusAsset.IsSuspended) + if (centaurusAsset == null) throw new BadRequestException($"Constellation doesn't support asset '{withdrawalRequest.Asset}'."); if (!Context.PaymentProvidersManager.TryGetManager(withdrawalRequest.Provider, out var paymentProvider)) diff --git a/Centaurus.Domain/ResultManagers/ResultManager.cs b/Centaurus.Domain/ResultManagers/ResultManager.cs index e76a936c..6f5dd42e 100644 --- a/Centaurus.Domain/ResultManagers/ResultManager.cs +++ b/Centaurus.Domain/ResultManagers/ResultManager.cs @@ -63,7 +63,7 @@ public bool TryGetResult(ulong apex, out QuantumProcessingItem processingItem) private void OnPostEviction(object key, object value, EvictionReason reason, object state) { - logger.Info($"Batch {key} with {((List)value).Count} is removed because of {reason}. State: {state}."); + logger.Info($"Batch {key} is removed because of {reason}. State: {state}."); } private ulong GetBatchApexStart(ulong apex) From 004573304ccfeecce7335c65feda26280cf1a253 Mon Sep 17 00:00:00 2001 From: hawthorne-abendsen <49230725+hawthorne-abendsen@users.noreply.github.com> Date: Fri, 24 Dec 2021 20:15:53 +0200 Subject: [PATCH 08/13] Tests updates --- .../BaseMessageHandlerTests.cs | 12 +++++-- Centaurus.Test.Domain/OrderbookTests.cs | 32 +++++++++++-------- .../PendingQuantaPersistentTest.cs | 18 ++++++++++- .../UpdatesTest.cs | 28 +++++++++------- .../IntegrationTestEnvironment.cs | 3 +- 5 files changed, 62 insertions(+), 31 deletions(-) diff --git a/Centaurus.Test.Domain/BaseMessageHandlerTests.cs b/Centaurus.Test.Domain/BaseMessageHandlerTests.cs index aaea5efd..b815051a 100644 --- a/Centaurus.Test.Domain/BaseMessageHandlerTests.cs +++ b/Centaurus.Test.Domain/BaseMessageHandlerTests.cs @@ -37,9 +37,7 @@ protected IncomingConnectionBase GetIncomingConnection(ExecutionContext context, if (isAuthenticated) { - var connectionType = typeof(IncomingConnectionBase); - var authField = connectionType.GetField("isAuthenticated", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic); - authField.SetValue(connection, true); + MarkAsAuthenticated(connection); } return connection; } @@ -48,7 +46,15 @@ protected OutgoingConnection GetOutgoingConnection(ExecutionContext context, Raw { context.NodesManager.TryGetNode(pubKey, out var node); var connection = new OutgoingConnection(context, node, new DummyConnectionWrapper(new DummyWebSocket())); + MarkAsAuthenticated(connection); return connection; } + + private void MarkAsAuthenticated(ConnectionBase connection) + { + var connectionType = typeof(ConnectionBase); + var authField = connectionType.GetProperty("IsAuthenticated", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic); + authField.SetValue(connection, true); + } } } diff --git a/Centaurus.Test.Domain/OrderbookTests.cs b/Centaurus.Test.Domain/OrderbookTests.cs index 71ad20d4..02027bec 100644 --- a/Centaurus.Test.Domain/OrderbookTests.cs +++ b/Centaurus.Test.Domain/OrderbookTests.cs @@ -50,14 +50,9 @@ public void Setup() account2.CreateBalance(secondAsset); account2.GetBalance(secondAsset).UpdateBalance(10000000000, UpdateSign.Plus); - context.Init(new Snapshot + var constellation = new ConstellationSettings { - Accounts = new List { account1, account2 }, - Apex = 0, - Orders = new List(), - ConstellationSettings = new ConstellationSettings - { - Providers = new List { + Providers = new List { new ProviderSettings { Assets = new List { new ProviderAsset { CentaurusAsset = baseAsset, Token = "native" } }, InitCursor = "0", @@ -67,19 +62,28 @@ public void Setup() Vault = KeyPair.Random().AccountId } }, - Assets = new List { new AssetSettings { Code = baseAsset, IsQuoteAsset = true }, new AssetSettings { Code = secondAsset } }, - Alpha = TestEnvironment.AlphaKeyPair, - Auditors = new[] { TestEnvironment.AlphaKeyPair, TestEnvironment.Auditor1KeyPair } + Assets = new List { new AssetSettings { Code = baseAsset, IsQuoteAsset = true }, new AssetSettings { Code = secondAsset } }, + Alpha = TestEnvironment.AlphaKeyPair, + Auditors = new[] { TestEnvironment.AlphaKeyPair, TestEnvironment.Auditor1KeyPair } .Select(pk => new Auditor { PubKey = pk, Address = $"{pk.AccountId}.com" }) .ToList(), - MinAccountBalance = 1, - MinAllowedLotSize = 1, - RequestRateLimits = requestRateLimits - }, + MinAccountBalance = 1, + MinAllowedLotSize = 1, + RequestRateLimits = requestRateLimits + }; + + context.ConstellationSettingsManager.Update(constellation); + + context.Init(new Snapshot + { + Accounts = new List { account1, account2 }, + Apex = 0, + Orders = new List(), + ConstellationSettings = constellation, Cursors = new Dictionary { { "Stellar-Main", "0" } } }); diff --git a/Centaurus.Test.Domain/PendingQuantaPersistentTest.cs b/Centaurus.Test.Domain/PendingQuantaPersistentTest.cs index 0bc47790..2b2932e8 100644 --- a/Centaurus.Test.Domain/PendingQuantaPersistentTest.cs +++ b/Centaurus.Test.Domain/PendingQuantaPersistentTest.cs @@ -64,7 +64,23 @@ await Task.Factory.StartNew(() => context = new ExecutionContext(settings, storage, new MockPaymentProviderFactory(), new DummyConnectionWrapperFactory()); Assert.AreEqual(State.Rising, context.NodesManager.CurrentNode.State); - await context.Catchup.AddNodeBatch(TestEnvironment.Auditor1KeyPair, new CatchupQuantaBatch { Quanta = new List(), HasMore = false }); + await context.Catchup.AddNodeBatch(TestEnvironment.AlphaKeyPair, new CatchupQuantaBatch + { + Quanta = new List { item }, + HasMore = false + }); + + item.Signatures.Add(new NodeSignatureInternal + { + NodeId = 2, + PayloadSignature = quantum.GetPayloadHash().Sign(TestEnvironment.Auditor1KeyPair) + }); + + await context.Catchup.AddNodeBatch(TestEnvironment.Auditor1KeyPair, new CatchupQuantaBatch + { + Quanta = new List { item }, + HasMore = false + }); Assert.AreEqual(State.Running, context.NodesManager.CurrentNode.State); diff --git a/Centaurus.Test.Exchange.Analytics/UpdatesTest.cs b/Centaurus.Test.Exchange.Analytics/UpdatesTest.cs index 69fe0331..b09ecbaf 100644 --- a/Centaurus.Test.Exchange.Analytics/UpdatesTest.cs +++ b/Centaurus.Test.Exchange.Analytics/UpdatesTest.cs @@ -65,25 +65,29 @@ public void Setup() PaymentSubmitDelay = 0 }; - context.Init(new Snapshot + var constellation = new ConstellationSettings { - Accounts = new List { account1, account2 }, - Apex = 0, - Orders = new List(), - ConstellationSettings = new ConstellationSettings - { - Providers = new List { stellarPaymentProvider }, - Assets = new List { new AssetSettings { Code = "XLM", IsQuoteAsset = true }, new AssetSettings { Code = "USD" } }, - RequestRateLimits = new RequestRateLimits { HourLimit = 1000, MinuteLimit = 100 }, - Alpha = TestEnvironment.AlphaKeyPair, - Auditors = new[] { TestEnvironment.AlphaKeyPair, TestEnvironment.Auditor1KeyPair } + Providers = new List { stellarPaymentProvider }, + Assets = new List { new AssetSettings { Code = "XLM", IsQuoteAsset = true }, new AssetSettings { Code = "USD" } }, + RequestRateLimits = new RequestRateLimits { HourLimit = 1000, MinuteLimit = 100 }, + Alpha = TestEnvironment.AlphaKeyPair, + Auditors = new[] { TestEnvironment.AlphaKeyPair, TestEnvironment.Auditor1KeyPair } .Select(pk => new Auditor { PubKey = pk, Address = $"{pk.AccountId}.com" }) .ToList() - }, + }; + + context.ConstellationSettingsManager.Update(constellation); + + context.Init(new Snapshot + { + Accounts = new List { account1, account2 }, + Apex = 0, + Orders = new List(), + ConstellationSettings = constellation, Cursors = new[] { stellarPaymentProvider }.ToDictionary(p => PaymentProviderBase.GetProviderId(p.Provider, p.Name), p => p.InitCursor) }); } diff --git a/Centaurus.Test.Integration/IntegrationTestEnvironment.cs b/Centaurus.Test.Integration/IntegrationTestEnvironment.cs index 7a4bc8c2..7607a44a 100644 --- a/Centaurus.Test.Integration/IntegrationTestEnvironment.cs +++ b/Centaurus.Test.Integration/IntegrationTestEnvironment.cs @@ -110,7 +110,8 @@ private Settings GetSettings(string secret, List auditors) SyncBatchSize = 500, GenesisAuditors = auditors, ConnectionString = "", - IsPrimeNode = true + IsPrimeNode = true, + CatchupTimeout = 5 }; settings.Build(); return settings; From 29f26714089778f8e5e35d0769bd1c7316498003 Mon Sep 17 00:00:00 2001 From: hawthorne-abendsen <49230725+hawthorne-abendsen@users.noreply.github.com> Date: Tue, 4 Jan 2022 16:24:57 +0200 Subject: [PATCH 09/13] Participation level added --- Centaurus.Common/Common/SettingsExtensions.cs | 17 +++++++++++++++++ Centaurus.Common/Settings/ParticipationLevel.cs | 12 ++++++++++++ Centaurus.Common/Settings/Settings.cs | 4 ++-- Centaurus.Domain/Contexts/ExecutionContext.cs | 2 +- .../Nodes/CurrentNode/CurrentNode.cs | 2 +- .../IntegrationTestEnvironment.cs | 2 +- Centaurus.Test.Utils/GlobalInitHelper.cs | 2 +- Centaurus/Startup.cs | 2 +- 8 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 Centaurus.Common/Common/SettingsExtensions.cs create mode 100644 Centaurus.Common/Settings/ParticipationLevel.cs diff --git a/Centaurus.Common/Common/SettingsExtensions.cs b/Centaurus.Common/Common/SettingsExtensions.cs new file mode 100644 index 00000000..367fd2a8 --- /dev/null +++ b/Centaurus.Common/Common/SettingsExtensions.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Centaurus +{ + public static class SettingsExtensions + { + public static bool IsPrimeNode(this Settings settings) + { + if (settings == null) + throw new ArgumentNullException(nameof(settings)); + + return settings.ParticipationLevel == ParticipationLevel.Prime; + } + } +} diff --git a/Centaurus.Common/Settings/ParticipationLevel.cs b/Centaurus.Common/Settings/ParticipationLevel.cs new file mode 100644 index 00000000..3ec6d1b9 --- /dev/null +++ b/Centaurus.Common/Settings/ParticipationLevel.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Centaurus +{ + public enum ParticipationLevel + { + Prime = 1, + Auditor = 2 + } +} diff --git a/Centaurus.Common/Settings/Settings.cs b/Centaurus.Common/Settings/Settings.cs index 90506ea0..dc11572d 100644 --- a/Centaurus.Common/Settings/Settings.cs +++ b/Centaurus.Common/Settings/Settings.cs @@ -45,8 +45,8 @@ public class Settings [Option("catchup_timeout", Default = 15, HelpText = "Catchup timeout in seconds.")] public int CatchupTimeout { get; set; } - [Option('p', "prime_node", Default = false, HelpText = "Centaurus node participation level. If true the node is Prime, otherwise the node is Auditor")] - public bool IsPrimeNode { get; set; } + [Option("participation_level", Required = true, HelpText = "Centaurus node participation level. '1' or 'Prime' for the prime nodes, and '2' or 'Auditor' for the auditor nodes.")] + public ParticipationLevel ParticipationLevel { get; set; } [Option("payment_config", Required = true, HelpText = "Payment providers config path.")] public string PaymentConfigPath { get; set; } diff --git a/Centaurus.Domain/Contexts/ExecutionContext.cs b/Centaurus.Domain/Contexts/ExecutionContext.cs index c4768683..f04987f1 100644 --- a/Centaurus.Domain/Contexts/ExecutionContext.cs +++ b/Centaurus.Domain/Contexts/ExecutionContext.cs @@ -152,7 +152,7 @@ public void Init(Snapshot snapshot) { AccountStorage = new AccountStorage(snapshot.Accounts); - Exchange = Exchange.RestoreExchange(ConstellationSettingsManager.Current.Assets, snapshot.Orders, Settings.IsPrimeNode); + Exchange = Exchange.RestoreExchange(ConstellationSettingsManager.Current.Assets, snapshot.Orders, Settings.IsPrimeNode()); PaymentProvidersManager.RegisterProviders(ConstellationSettingsManager.Current, snapshot.Cursors); diff --git a/Centaurus.Domain/Nodes/CurrentNode/CurrentNode.cs b/Centaurus.Domain/Nodes/CurrentNode/CurrentNode.cs index 2775efd0..237f6f54 100644 --- a/Centaurus.Domain/Nodes/CurrentNode/CurrentNode.cs +++ b/Centaurus.Domain/Nodes/CurrentNode/CurrentNode.cs @@ -12,7 +12,7 @@ public CurrentNode(ExecutionContext context, State initState) State = initState; } - public override bool IsPrimeNode => Context.Settings.IsPrimeNode; + public override bool IsPrimeNode => Context.Settings.IsPrimeNode(); public event Action StateChanged; diff --git a/Centaurus.Test.Integration/IntegrationTestEnvironment.cs b/Centaurus.Test.Integration/IntegrationTestEnvironment.cs index 7607a44a..f285ee16 100644 --- a/Centaurus.Test.Integration/IntegrationTestEnvironment.cs +++ b/Centaurus.Test.Integration/IntegrationTestEnvironment.cs @@ -110,7 +110,7 @@ private Settings GetSettings(string secret, List auditors) SyncBatchSize = 500, GenesisAuditors = auditors, ConnectionString = "", - IsPrimeNode = true, + ParticipationLevel = ParticipationLevel.Prime, CatchupTimeout = 5 }; settings.Build(); diff --git a/Centaurus.Test.Utils/GlobalInitHelper.cs b/Centaurus.Test.Utils/GlobalInitHelper.cs index fe65ff15..92ff7282 100644 --- a/Centaurus.Test.Utils/GlobalInitHelper.cs +++ b/Centaurus.Test.Utils/GlobalInitHelper.cs @@ -44,7 +44,7 @@ private static void SetCommonSettings(Settings settings, string secret) settings.CWD = "AppData"; settings.GenesisAuditors = new[] { TestEnvironment.AlphaKeyPair.AccountId, TestEnvironment.Auditor1KeyPair.AccountId }.Select(a => new Settings.Auditor($"{a}={a}.com")).ToList(); settings.Secret = secret; - settings.IsPrimeNode = true; + settings.ParticipationLevel = ParticipationLevel.Prime; settings.SyncBatchSize = 500; settings.CatchupTimeout = 1; } diff --git a/Centaurus/Startup.cs b/Centaurus/Startup.cs index 6ade8ac7..ebbd5f65 100644 --- a/Centaurus/Startup.cs +++ b/Centaurus/Startup.cs @@ -19,7 +19,7 @@ public Startup(Domain.ExecutionContext context, AlphaHostFactoryBase alphaHostFa if (alphaHostFactory == null) throw new ArgumentNullException(nameof(alphaHostFactory)); - if (context.Settings.IsPrimeNode) + if (context.Settings.IsPrimeNode()) AlphaStartup = new AlphaStartup(context, alphaHostFactory); } From 4a0dccd261e8c8bdf33092464b76e269f4aa227e Mon Sep 17 00:00:00 2001 From: hawthorne-abendsen <49230725+hawthorne-abendsen@users.noreply.github.com> Date: Wed, 5 Jan 2022 01:14:47 +0200 Subject: [PATCH 10/13] Catchup for the auditor participation level --- Centaurus.Domain/Catchups/Catchup.cs | 45 +++++++++++-------- .../Quanta/Processors/DepositProcessor.cs | 7 ++- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/Centaurus.Domain/Catchups/Catchup.cs b/Centaurus.Domain/Catchups/Catchup.cs index 1c537957..62e868a8 100644 --- a/Centaurus.Domain/Catchups/Catchup.cs +++ b/Centaurus.Domain/Catchups/Catchup.cs @@ -57,10 +57,7 @@ public async Task AddNodeBatch(RawPubKey pubKey, CatchupQuantaBatch nodeBatch) validAuditorStates.Add(pubKey, aggregatedNodeBatch); - int majority = Context.GetMajorityCount(), - totalAuditorsCount = Context.GetTotalAuditorsCount(); - var completedStatesCount = allAuditorStates.Count(s => !s.Value.HasMore); - if (completedStatesCount == totalAuditorsCount) + if (AreAllNodesConnected()) await TryApplyAuditorsData(); } catch (Exception exc) @@ -122,24 +119,19 @@ private async Task TryApplyAuditorsData() if (Context.NodesManager.CurrentNode.State != State.Rising) return; - int majority = Context.GetMajorityCount(), - totalAuditorsCount = Context.GetTotalAuditorsCount(); - - if (Context.HasMajority(validAuditorStates.Count)) - { - var validQuanta = GetValidQuanta(); - await ApplyQuanta(validQuanta); - allAuditorStates.Clear(); - validAuditorStates.Clear(); - Context.NodesManager.CurrentNode.Rised(); - logger.Trace($"Alpha is risen."); - } - else + if (!IsReadyToApplyQuantumData()) { var connectedAccounts = allAuditorStates.Keys.Select(a => a.GetAccountId()); var validatedAccounts = validAuditorStates.Keys.Select(a => a.GetAccountId()); - throw new Exception($"Unable to raise. Connected auditors: {string.Join(',', connectedAccounts)}; validated auditors: {string.Join(',', validatedAccounts)}; majority is {majority}."); + throw new Exception($"Unable to raise. Connected auditors: {string.Join(',', connectedAccounts)}; validated auditors: {string.Join(',', validatedAccounts)}; majority is {Context.GetMajorityCount()}."); } + + var validQuanta = GetValidQuanta(); + await ApplyQuanta(validQuanta); + allAuditorStates.Clear(); + validAuditorStates.Clear(); + Context.NodesManager.CurrentNode.Rised(); + logger.Trace($"Alpha is risen."); } catch (Exception exc) { @@ -151,6 +143,23 @@ private async Task TryApplyAuditorsData() } } + private bool AreAllNodesConnected() + { + var totalAuditorsCount = Context.GetTotalAuditorsCount(); + var completedStatesCount = allAuditorStates.Count(s => !s.Value.HasMore); + return completedStatesCount == totalAuditorsCount; + } + + private bool IsReadyToApplyQuantumData() + { + //a prime node should be connected with the all nodes + if (Context.Settings.IsPrimeNode()) + return Context.HasMajority(validAuditorStates.Count); + + //connect to at least one prime node + return validAuditorStates.Count > 1; + } + private void InitTimer() { applyDataTimer = new System.Timers.Timer(); diff --git a/Centaurus.Domain/Quanta/Processors/DepositProcessor.cs b/Centaurus.Domain/Quanta/Processors/DepositProcessor.cs index 11ef5f16..d6377b27 100644 --- a/Centaurus.Domain/Quanta/Processors/DepositProcessor.cs +++ b/Centaurus.Domain/Quanta/Processors/DepositProcessor.cs @@ -24,11 +24,14 @@ public override Task Process(QuantumProcessingItem qua if (!Context.PaymentProvidersManager.TryGetManager(depositNotification.Provider, out var paymentProvider)) throw new Exception($"Payment provider {paymentProvider} is not registered."); + var notification = default(DepositNotification); if (depositQuantum.Source == null - || !TryGetNotification(paymentProvider, out var notification) + || !TryGetNotification(paymentProvider, out notification) || !ByteArrayPrimitives.Equals(notification.ComputeHash(), depositQuantum.Source.ComputeHash())) { - throw new InvalidOperationException("Unexpected tx notification."); + var notificationInfo = $"Deposit source cursor: {depositQuantum.Source?.Cursor ?? "n\\a"}; " + + $"provider cursor: {notification?.Cursor ?? "n\\a"}"; + throw new InvalidOperationException($"Unexpected tx notification. {notificationInfo}."); } quantumProcessingItem.AddCursorUpdate(paymentProvider.NotificationsManager, depositNotification.Provider, depositNotification.Cursor, paymentProvider.Cursor); From 9d7d385b95e900c44b4ea68a5cd31fa5ddc2de13 Mon Sep 17 00:00:00 2001 From: hawthorne-abendsen <49230725+hawthorne-abendsen@users.noreply.github.com> Date: Wed, 5 Jan 2022 19:33:02 +0200 Subject: [PATCH 11/13] Batch sync bug fix --- .../Quanta/Sync/ApexItemsBatchPortion.cs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Centaurus.Domain/Quanta/Sync/ApexItemsBatchPortion.cs b/Centaurus.Domain/Quanta/Sync/ApexItemsBatchPortion.cs index dfa07335..c0b247a7 100644 --- a/Centaurus.Domain/Quanta/Sync/ApexItemsBatchPortion.cs +++ b/Centaurus.Domain/Quanta/Sync/ApexItemsBatchPortion.cs @@ -10,7 +10,7 @@ class ApexItemsBatchPortion { public ApexItemsBatchPortion(ulong start, int size, ApexItemsBatch source) { - Start = lastSerializedBatchApex = start; + Start = start; Size = size; LastApex = start + (ulong)size; this.source = source ?? throw new ArgumentNullException(nameof(source)); @@ -26,12 +26,13 @@ public SyncPortion GetBatch(bool force) if (batch != null && lastSerializedBatchApex == LastApex) return batch; - if (force && source.LastApex >= lastSerializedBatchApex || source.LastApex >= LastApex) - if (lastSerializedBatchApex < source.LastApex) - { - batch = GetBatchData(); - lastSerializedBatchApex = batch.LastDataApex; - } + if (source.LastApex >= Start //the source has data for the batch + && source.LastApex > lastSerializedBatchApex //the source has fresh data + && (force || source.LastApex >= LastApex)) //if force or all data for the batch is ready + { + batch = GetBatchData(); + lastSerializedBatchApex = batch.LastDataApex; + } return batch; } From 05ca2021201d887b9a981e1b3fdd2e5313503150 Mon Sep 17 00:00:00 2001 From: hawthorne-abendsen <49230725+hawthorne-abendsen@users.noreply.github.com> Date: Wed, 5 Jan 2022 19:41:27 +0200 Subject: [PATCH 12/13] Failed state handling --- Centaurus.Domain/Contexts/ExecutionContext.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Centaurus.Domain/Contexts/ExecutionContext.cs b/Centaurus.Domain/Contexts/ExecutionContext.cs index f04987f1..2af259a6 100644 --- a/Centaurus.Domain/Contexts/ExecutionContext.cs +++ b/Centaurus.Domain/Contexts/ExecutionContext.cs @@ -128,6 +128,8 @@ private void CurrentNode_StateChanged(StateChangedEventArgs stateChangedEventArg if (prevState == State.Rising && (state == State.Running || state == State.Ready)) //after node successfully started, the pending quanta can be deleted PersistentStorage.DeletePendingQuanta(); + if (state == State.Failed) + Complete(); } private void HandlePendingQuanta(List pendingQuanta) From 6ee6e7a5057394e86f47dbd9d2c012a470faf74b Mon Sep 17 00:00:00 2001 From: hawthorne-abendsen <49230725+hawthorne-abendsen@users.noreply.github.com> Date: Wed, 5 Jan 2022 19:49:51 +0200 Subject: [PATCH 13/13] Handling pending quanta only on Rising state --- Centaurus.Domain/Contexts/ExecutionContext.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Centaurus.Domain/Contexts/ExecutionContext.cs b/Centaurus.Domain/Contexts/ExecutionContext.cs index 2af259a6..205b25ff 100644 --- a/Centaurus.Domain/Contexts/ExecutionContext.cs +++ b/Centaurus.Domain/Contexts/ExecutionContext.cs @@ -134,11 +134,12 @@ private void CurrentNode_StateChanged(StateChangedEventArgs stateChangedEventArg private void HandlePendingQuanta(List pendingQuanta) { - _ = Catchup.AddNodeBatch(Settings.KeyPair, new CatchupQuantaBatch - { - Quanta = pendingQuanta ?? new List(), - HasMore = false - }); + if (NodesManager.CurrentNode.State == State.Rising) + _ = Catchup.AddNodeBatch(Settings.KeyPair, new CatchupQuantaBatch + { + Quanta = pendingQuanta ?? new List(), + HasMore = false + }); } private string GetAbsolutePath(string path)