From f0d87c9cbc02928547773ee94b0a5fc65c4d8c72 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Mon, 21 Jul 2025 21:09:30 +0300 Subject: [PATCH 01/22] Added initial version of the app --- Apps/MispConnectorApp/App.cs | 367 ++++++++++++++++++ Apps/MispConnectorApp/MispConnectorApp.csproj | 43 ++ Apps/MispConnectorApp/dnsApp.config | 8 + DnsServer.sln | 12 + 4 files changed, 430 insertions(+) create mode 100644 Apps/MispConnectorApp/App.cs create mode 100644 Apps/MispConnectorApp/MispConnectorApp.csproj create mode 100644 Apps/MispConnectorApp/dnsApp.config diff --git a/Apps/MispConnectorApp/App.cs b/Apps/MispConnectorApp/App.cs new file mode 100644 index 00000000..9f84d9b9 --- /dev/null +++ b/Apps/MispConnectorApp/App.cs @@ -0,0 +1,367 @@ +/* +Technitium DNS Server +Copyright (C) 2024 Shreyas Zare (shreyas@technitium.com) +Copyright (C) 2025 Zafer Balkan (zafer@zaferbalkan.com) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +using DnsServerCore.ApplicationCommon; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Security; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using TechnitiumLibrary.Net.Dns; +using TechnitiumLibrary.Net.Dns.ResourceRecords; +using TechnitiumLibrary.Net.Http.Client; + +namespace MispConnector +{ + public sealed class App : IDnsApplication, IDnsRequestBlockingHandler + { + #region variables + + readonly object _blocklistLock = new object(); + + readonly HashSet _globalBlocklist = new HashSet(StringComparer.OrdinalIgnoreCase); + + readonly Random _random = new Random(); + + bool _allowTxtBlockingReport; + + string _cacheFilePath; + + IDnsServer _dnsServer; + bool _enableBlocking; + + HttpClient _httpClient; + + string _maxIocAge; + + string _mispApiKey; + + Uri _mispApiUrl; + + DnsSOARecordData _soaRecord; + TimeSpan _updateInterval; + + Timer _updateTimer; + #endregion + + #region IDisposable + + public void Dispose() + { + _updateTimer?.Dispose(); + _httpClient?.Dispose(); + } + + public async Task InitializeAsync(IDnsServer dnsServer, string config) + { + _dnsServer = dnsServer; + try + { + string configDir = _dnsServer.ApplicationFolder; + Directory.CreateDirectory(configDir); + _cacheFilePath = Path.Combine(configDir, "misp_domain_cache.txt"); + + _soaRecord = new DnsSOARecordData(_dnsServer.ServerDomain, _dnsServer.ResponsiblePerson.Address, 1, 14400, 3600, 604800, 60); + + using JsonDocument jsonDocument = JsonDocument.Parse(config); + JsonElement jsonConfig = jsonDocument.RootElement; + + _enableBlocking = jsonConfig.GetProperty("enableBlocking").GetBoolean(); + _allowTxtBlockingReport = jsonConfig.GetProperty("allowTxtBlockingReport").GetBoolean(); + Uri mispServerUrl = new Uri(jsonConfig.GetProperty("mispServerUrl").GetString()); + _mispApiKey = jsonConfig.GetProperty("mispApiKey").GetString(); + bool disableTlsValidation = jsonConfig.GetProperty("disableTlsValidation").GetBoolean(); + + string updateIntervalString = jsonConfig.GetProperty("updateInterval").GetString(); + _updateInterval = ParseUpdateInterval(updateIntervalString); + + _maxIocAge = jsonConfig.GetProperty("maxIocAge").GetString(); + + _mispApiUrl = new Uri(mispServerUrl, "/attributes/restSearch"); + _httpClient = CreateHttpClient(mispServerUrl, disableTlsValidation); + + await LoadBlocklistFromCacheAsync(); + await using Timer _ = _updateTimer = new Timer(async _ => + { + await UpdateIocsAsync(); + }, null, TimeSpan.FromSeconds(10), Timeout.InfiniteTimeSpan); + } + catch (Exception ex) + { + _dnsServer.WriteLog($"FATAL: MISP Connector failed to initialize. Check configuration. Error: {ex.Message}"); + _dnsServer.WriteLog(ex); + } + } + + #endregion + + #region public + + public Task IsAllowedAsync(DnsDatagram request, IPEndPoint remoteEP) + { + return Task.FromResult(false); + } + + public Task ProcessRequestAsync(DnsDatagram request, IPEndPoint remoteEP) + { + if (!_enableBlocking) + return Task.FromResult(null); + + DnsQuestionRecord question = request.Question[0]; + if (!IsDomainBlocked(question.Name, out string blockedDomain)) + { + return Task.FromResult(null); + } + + if (_allowTxtBlockingReport && question.Type == DnsResourceRecordType.TXT) + { + DnsResourceRecord[] answer = new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, 60, new DnsTXTRecordData($"source=misp-connector;domain={blockedDomain}")) }; + return Task.FromResult(new DnsDatagram( + ID: request.Identifier, + isResponse: true, + OPCODE: DnsOpcode.StandardQuery, + authoritativeAnswer: false, + truncation: false, + recursionDesired: request.RecursionDesired, + recursionAvailable: true, + authenticData: false, + checkingDisabled: false, + RCODE: DnsResponseCode.NoError, + question: request.Question, + answer: answer, + authority: null, + additional: null + )); + } + + DnsResourceRecord[] authority = new DnsResourceRecord[] { new DnsResourceRecord(GetParentZone(blockedDomain) ?? string.Empty, DnsResourceRecordType.SOA, question.Class, 60, _soaRecord) }; + return Task.FromResult(new DnsDatagram( + ID: request.Identifier, + isResponse: true, + OPCODE: DnsOpcode.StandardQuery, + authoritativeAnswer: true, + truncation: false, + recursionDesired: request.RecursionDesired, + recursionAvailable: true, + authenticData: false, + checkingDisabled: false, + RCODE: DnsResponseCode.NxDomain, + question: request.Question, + answer: null, + authority: authority, + additional: null + )); + } + + #endregion + + #region private + private static string GetParentZone(string domain) + { + int i = domain.IndexOf('.'); + return (i > -1) ? domain.Substring(i + 1) : null; + } + + private static TimeSpan ParseUpdateInterval(string interval) + { + if (string.IsNullOrWhiteSpace(interval) || interval.Length < 2) + { + throw new FormatException("Update interval is not in a valid format (e.g., '60m', '2h', '7d')."); + } + + string unit = interval.Substring(interval.Length - 1).ToLowerInvariant(); + string valueString = interval.Substring(0, interval.Length - 1); + + if (!int.TryParse(valueString, NumberStyles.Integer, CultureInfo.InvariantCulture, out int value) || value <= 0) + { + throw new FormatException($"Invalid numeric value '{valueString}' in update interval."); + } + + switch (unit) + { + case "m": + return TimeSpan.FromMinutes(value); + case "h": + return TimeSpan.FromHours(value); + case "d": + return TimeSpan.FromDays(value); + default: + throw new FormatException($"Invalid unit '{unit}' in update interval. Allowed units are 'm', 'h', 'd'."); + } + } + + private HttpClient CreateHttpClient(Uri serverUrl, bool disableTlsValidation) + { + SocketsHttpHandler handler = new SocketsHttpHandler + { + Proxy = _dnsServer.Proxy, + UseProxy = _dnsServer.Proxy != null, + SslOptions = new SslClientAuthenticationOptions() + }; + + if (disableTlsValidation) + { + handler.SslOptions.RemoteCertificateValidationCallback = (sender, certificate, chain, sslPolicyErrors) => + { + return true; + }; + _dnsServer.WriteLog($"WARNING: TLS certificate validation is DISABLED for MISP server: {serverUrl}"); + } + + return new HttpClient(new HttpClientNetworkHandler(handler, _dnsServer.PreferIPv6 ? HttpClientNetworkType.PreferIPv6 : HttpClientNetworkType.Default, _dnsServer)); + } + + private async Task> FetchDomainsFromMispAsync() + { + var requestBody = new + { + type = "domain", + to_ids = true, + deleted = false, + last = _maxIocAge + }; + + StringContent requestContent = new StringContent(JsonSerializer.Serialize(requestBody), Encoding.UTF8, "application/json"); + + using HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, _mispApiUrl) + { + Content = requestContent + }; + + request.Headers.Add("Authorization", _mispApiKey); + request.Headers.Add("Accept", "application/json"); + + using HttpResponseMessage response = await _httpClient.SendAsync(request); + response.EnsureSuccessStatusCode(); + + using Stream responseStream = await response.Content.ReadAsStreamAsync(); + using JsonDocument jsonDoc = await JsonDocument.ParseAsync(responseStream); + + HashSet domains = new HashSet(StringComparer.OrdinalIgnoreCase); + if (jsonDoc.RootElement.TryGetProperty("response", out JsonElement responseElement) && + responseElement.TryGetProperty("Attribute", out JsonElement attributeArray) && + attributeArray.ValueKind == JsonValueKind.Array) + { + foreach (JsonElement attributeElement in attributeArray.EnumerateArray()) + { + if (attributeElement.TryGetProperty("value", out JsonElement valueElement) && valueElement.ValueKind == JsonValueKind.String) + { + string domain = valueElement.GetString()?.Trim().ToLowerInvariant(); + if (!string.IsNullOrEmpty(domain) && DnsClient.IsDomainNameValid(domain)) + { + domains.Add(domain); + } + } + } + } + return domains; + } + + private bool IsDomainBlocked(string domain, out string foundZone) + { + lock (_blocklistLock) + { + string currentDomain = domain.ToLowerInvariant(); + do + { + if (_globalBlocklist.Contains(currentDomain)) + { + foundZone = currentDomain; + return true; + } + currentDomain = GetParentZone(currentDomain); + } while (currentDomain != null); + } + foundZone = null; + return false; + } + + private async Task LoadBlocklistFromCacheAsync() + { + if (!File.Exists(_cacheFilePath)) return; + try + { + HashSet domains = (await File.ReadAllLinesAsync(_cacheFilePath)).ToHashSet(StringComparer.OrdinalIgnoreCase); + ReloadBlocklist(domains); + _dnsServer.WriteLog($"MISP Connector: Loaded {domains.Count} domains from cache."); + } + catch (IOException ex) + { + _dnsServer.WriteLog($"ERROR: Failed to read cache file '{_cacheFilePath}'. Error: {ex.Message}"); + } + } + + private void ReloadBlocklist(HashSet domains) + { + lock (_blocklistLock) + { + _globalBlocklist.Clear(); + foreach (string domain in domains) + { + _globalBlocklist.Add(domain); + } + } + } + + private async Task UpdateIocsAsync() + { + try + { + _dnsServer.WriteLog("MISP Connector: Starting IOC update..."); + HashSet domains = await FetchDomainsFromMispAsync(); + await WriteDomainsToCacheAsync(domains); + ReloadBlocklist(domains); + _dnsServer.WriteLog($"MISP Connector: Successfully updated blocklist with {domains.Count} domains."); + } + catch (Exception ex) + { + _dnsServer.WriteLog($"ERROR: MISP Connector failed to update IOCs. Error: {ex.Message}"); + } + finally + { + TimeSpan nextInterval = _updateInterval + TimeSpan.FromSeconds(_random.Next(0, 60)); + _updateTimer?.Change(nextInterval, Timeout.InfiniteTimeSpan); + } + } + private async Task WriteDomainsToCacheAsync(HashSet domains) + { + string tempPath = _cacheFilePath + ".tmp"; + await File.WriteAllLinesAsync(tempPath, domains); + File.Move(tempPath, _cacheFilePath, true); + } + #endregion + + #region properties + public string Description + { + get + { + return "A focused connector that imports domain IOCs from a MISP server to block malicious domains using direct REST API calls."; + } + } + #endregion + } +} diff --git a/Apps/MispConnectorApp/MispConnectorApp.csproj b/Apps/MispConnectorApp/MispConnectorApp.csproj new file mode 100644 index 00000000..acba681d --- /dev/null +++ b/Apps/MispConnectorApp/MispConnectorApp.csproj @@ -0,0 +1,43 @@ + + + + net8.0 + false + 8.0 + false + Technitium + Technitium DNS Server + Zafer Balkan + MispConnectorApp + MispConnector + https://technitium.com/dns/ + https://github.com/TechnitiumSoftware/DnsServer + Block domain names extracted MISP instance by querying IOCs.\n\nNote! This app works independent of the DNS server's built-in blocking feature. The options configured in DNS server Settings section does not apply to this app. + false + Library + + + + + false + + + + + + ..\..\..\TechnitiumLibrary\bin\TechnitiumLibrary.dll + false + + + ..\..\..\TechnitiumLibrary\bin\TechnitiumLibrary.Net.dll + false + + + + + + PreserveNewest + + + + diff --git a/Apps/MispConnectorApp/dnsApp.config b/Apps/MispConnectorApp/dnsApp.config new file mode 100644 index 00000000..cff8512a --- /dev/null +++ b/Apps/MispConnectorApp/dnsApp.config @@ -0,0 +1,8 @@ +{ + "enableBlocking": true, + "mispServerUrl": "https://misp.example.com", + "mispApiKey": "YourMispApiKeyHere", + "disableTlsValidation": false, + "updateInterval": "2h", + "maxIocAge": "90d" +} diff --git a/DnsServer.sln b/DnsServer.sln index 24ad9414..05fc6280 100644 --- a/DnsServer.sln +++ b/DnsServer.sln @@ -67,6 +67,13 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QueryLogsSqlServerApp", "Ap EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QueryLogsMySqlApp", "Apps\QueryLogsMySqlApp\QueryLogsMySqlApp.csproj", "{699E2A1D-D917-4825-939E-65CDB2B16A96}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MispConnectorApp", "Apps\MispConnectorApp\MispConnectorApp.csproj", "{83C8180A-0F86-F9A0-8F41-6FD61FAC41CB}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{02EA681E-C7D8-13C7-8484-4AC65E1B71E8}" + ProjectSection(SolutionItems) = preProject + .editorconfig = .editorconfig + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -197,6 +204,10 @@ Global {699E2A1D-D917-4825-939E-65CDB2B16A96}.Debug|Any CPU.Build.0 = Debug|Any CPU {699E2A1D-D917-4825-939E-65CDB2B16A96}.Release|Any CPU.ActiveCfg = Release|Any CPU {699E2A1D-D917-4825-939E-65CDB2B16A96}.Release|Any CPU.Build.0 = Release|Any CPU + {83C8180A-0F86-F9A0-8F41-6FD61FAC41CB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {83C8180A-0F86-F9A0-8F41-6FD61FAC41CB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {83C8180A-0F86-F9A0-8F41-6FD61FAC41CB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {83C8180A-0F86-F9A0-8F41-6FD61FAC41CB}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -228,6 +239,7 @@ Global {0A9B7F39-80DA-4084-AD47-8707576927ED} = {938BF8EF-74B9-4FE0-B46F-11EBB7A4B3D2} {6F655C97-FD43-4FE1-B15A-6C783D2D91C9} = {938BF8EF-74B9-4FE0-B46F-11EBB7A4B3D2} {699E2A1D-D917-4825-939E-65CDB2B16A96} = {938BF8EF-74B9-4FE0-B46F-11EBB7A4B3D2} + {83C8180A-0F86-F9A0-8F41-6FD61FAC41CB} = {938BF8EF-74B9-4FE0-B46F-11EBB7A4B3D2} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {6747BB6D-2826-4356-A213-805FBCCF9201} From d4e80129bfe0e37b4ae12c5bfff2a362fc1f3382 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Tue, 22 Jul 2025 11:41:44 +0300 Subject: [PATCH 02/22] Added allowTxtBlockingReport bool --- Apps/MispConnectorApp/App.cs | 8 ++++---- Apps/MispConnectorApp/dnsApp.config | 13 +++++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Apps/MispConnectorApp/App.cs b/Apps/MispConnectorApp/App.cs index 9f84d9b9..c18535e7 100644 --- a/Apps/MispConnectorApp/App.cs +++ b/Apps/MispConnectorApp/App.cs @@ -75,6 +75,10 @@ public void Dispose() _httpClient?.Dispose(); } + #endregion + + #region public + public async Task InitializeAsync(IDnsServer dnsServer, string config) { _dnsServer = dnsServer; @@ -116,10 +120,6 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) } } - #endregion - - #region public - public Task IsAllowedAsync(DnsDatagram request, IPEndPoint remoteEP) { return Task.FromResult(false); diff --git a/Apps/MispConnectorApp/dnsApp.config b/Apps/MispConnectorApp/dnsApp.config index cff8512a..256b55fa 100644 --- a/Apps/MispConnectorApp/dnsApp.config +++ b/Apps/MispConnectorApp/dnsApp.config @@ -1,8 +1,9 @@ { - "enableBlocking": true, - "mispServerUrl": "https://misp.example.com", - "mispApiKey": "YourMispApiKeyHere", - "disableTlsValidation": false, - "updateInterval": "2h", - "maxIocAge": "90d" +"enableBlocking": true, +"mispServerUrl": "https://misp.example.com", +"mispApiKey": "YourMispApiKeyHere", +"disableTlsValidation": false, +"updateInterval": "2h", +"maxIocAge": "90d", +"allowTxtBlockingReport": true } From 10681487ed4579041a4d6ab5a20ae5a171e23232 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Tue, 22 Jul 2025 12:54:58 +0300 Subject: [PATCH 03/22] Improved error handling --- Apps/MispConnectorApp/App.cs | 89 ++++++++++++++++++++++++++---------- 1 file changed, 65 insertions(+), 24 deletions(-) diff --git a/Apps/MispConnectorApp/App.cs b/Apps/MispConnectorApp/App.cs index c18535e7..f03b5957 100644 --- a/Apps/MispConnectorApp/App.cs +++ b/Apps/MispConnectorApp/App.cs @@ -108,10 +108,18 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) _httpClient = CreateHttpClient(mispServerUrl, disableTlsValidation); await LoadBlocklistFromCacheAsync(); - await using Timer _ = _updateTimer = new Timer(async _ => + _updateTimer = new Timer(async _ => { - await UpdateIocsAsync(); - }, null, TimeSpan.FromSeconds(10), Timeout.InfiniteTimeSpan); + try + { + await UpdateIocsAsync(); + } + catch (Exception ex) + { + _dnsServer.WriteLog($"FATAL: The MispConnector update task failed unexpectedly. Error: {ex.Message}"); + _dnsServer.WriteLog(ex); + } + }, null, TimeSpan.FromSeconds(1), Timeout.InfiniteTimeSpan); } catch (Exception ex) { @@ -236,48 +244,81 @@ private HttpClient CreateHttpClient(Uri serverUrl, bool disableTlsValidation) private async Task> FetchDomainsFromMispAsync() { + // The request body can be constructed once. var requestBody = new { type = "domain", to_ids = true, deleted = false, + limit = 1000, last = _maxIocAge }; - StringContent requestContent = new StringContent(JsonSerializer.Serialize(requestBody), Encoding.UTF8, "application/json"); - using HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, _mispApiUrl) + try { - Content = requestContent - }; + using HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, _mispApiUrl) + { + Content = requestContent + }; - request.Headers.Add("Authorization", _mispApiKey); - request.Headers.Add("Accept", "application/json"); + request.Headers.Add("Authorization", _mispApiKey); + request.Headers.Add("Accept", "application/json"); - using HttpResponseMessage response = await _httpClient.SendAsync(request); - response.EnsureSuccessStatusCode(); + _dnsServer.WriteLog($"Sending API request to {_mispApiUrl}..."); + using HttpResponseMessage response = await _httpClient.SendAsync(request); - using Stream responseStream = await response.Content.ReadAsStreamAsync(); - using JsonDocument jsonDoc = await JsonDocument.ParseAsync(responseStream); + if (!response.IsSuccessStatusCode) + { + string errorBody = await response.Content.ReadAsStringAsync(); + _dnsServer.WriteLog($"ERROR: MISP API returned a non-success status code: {(int)response.StatusCode} {response.ReasonPhrase}. Response Body: {errorBody}"); - HashSet domains = new HashSet(StringComparer.OrdinalIgnoreCase); - if (jsonDoc.RootElement.TryGetProperty("response", out JsonElement responseElement) && - responseElement.TryGetProperty("Attribute", out JsonElement attributeArray) && - attributeArray.ValueKind == JsonValueKind.Array) - { - foreach (JsonElement attributeElement in attributeArray.EnumerateArray()) + throw new HttpRequestException($"MISP API request failed with status code {response.StatusCode}.", null, response.StatusCode); + } + + _dnsServer.WriteLog("API request successful. Parsing response..."); + await using Stream responseStream = await response.Content.ReadAsStreamAsync(); + using JsonDocument jsonDoc = await JsonDocument.ParseAsync(responseStream); + + HashSet domains = new HashSet(StringComparer.OrdinalIgnoreCase); + if (jsonDoc.RootElement.TryGetProperty("response", out JsonElement responseElement) && + responseElement.TryGetProperty("Attribute", out JsonElement attributeArray) && + attributeArray.ValueKind == JsonValueKind.Array) { - if (attributeElement.TryGetProperty("value", out JsonElement valueElement) && valueElement.ValueKind == JsonValueKind.String) + foreach (JsonElement attributeElement in attributeArray.EnumerateArray()) { - string domain = valueElement.GetString()?.Trim().ToLowerInvariant(); - if (!string.IsNullOrEmpty(domain) && DnsClient.IsDomainNameValid(domain)) + if (attributeElement.TryGetProperty("value", out JsonElement valueElement) && valueElement.ValueKind == JsonValueKind.String) { - domains.Add(domain); + string domain = valueElement.GetString()?.Trim().ToLowerInvariant(); + if (!string.IsNullOrEmpty(domain) && DnsClient.IsDomainNameValid(domain)) + { + domains.Add(domain); + } } } } + else + { + _dnsServer.WriteLog("WARNING: MISP API response was successful but did not contain the expected 'response.Attribute' array structure."); + } + + return domains; + } + catch (HttpRequestException ex) + { + _dnsServer.WriteLog($"ERROR: A network or HTTP error occurred while communicating with MISP. Error: {ex.Message}"); + throw; + } + catch (JsonException ex) + { + _dnsServer.WriteLog($"ERROR: Failed to parse the JSON response from MISP. The response may not be valid JSON. Error: {ex.Message}"); + throw; + } + catch (Exception ex) + { + _dnsServer.WriteLog($"ERROR: An unexpected error occurred during the fetch process. Error: {ex.Message}"); + throw; } - return domains; } private bool IsDomainBlocked(string domain, out string foundZone) From 060959a100fa630420022d2555d5960dd6df9935 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Tue, 22 Jul 2025 14:11:22 +0300 Subject: [PATCH 04/22] Used POCO for config --- Apps/MispConnectorApp/App.cs | 76 ++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 24 deletions(-) diff --git a/Apps/MispConnectorApp/App.cs b/Apps/MispConnectorApp/App.cs index f03b5957..ace9d3b7 100644 --- a/Apps/MispConnectorApp/App.cs +++ b/Apps/MispConnectorApp/App.cs @@ -20,6 +20,7 @@ You should have received a copy of the GNU General Public License using DnsServerCore.ApplicationCommon; using System; using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; using System.Globalization; using System.IO; using System.Linq; @@ -28,6 +29,7 @@ You should have received a copy of the GNU General Public License using System.Net.Security; using System.Text; using System.Text.Json; +using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; using TechnitiumLibrary.Net.Dns; @@ -40,25 +42,19 @@ public sealed class App : IDnsApplication, IDnsRequestBlockingHandler { #region variables + Config _config; + readonly object _blocklistLock = new object(); readonly HashSet _globalBlocklist = new HashSet(StringComparer.OrdinalIgnoreCase); readonly Random _random = new Random(); - bool _allowTxtBlockingReport; - string _cacheFilePath; IDnsServer _dnsServer; - bool _enableBlocking; - HttpClient _httpClient; - string _maxIocAge; - - string _mispApiKey; - Uri _mispApiUrl; DnsSOARecordData _soaRecord; @@ -90,22 +86,22 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) _soaRecord = new DnsSOARecordData(_dnsServer.ServerDomain, _dnsServer.ResponsiblePerson.Address, 1, 14400, 3600, 604800, 60); - using JsonDocument jsonDocument = JsonDocument.Parse(config); - JsonElement jsonConfig = jsonDocument.RootElement; + JsonSerializerOptions options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true }; + _config = JsonSerializer.Deserialize(config, options); + + Validator.ValidateObject(_config, new ValidationContext(_config), validateAllProperties: true); - _enableBlocking = jsonConfig.GetProperty("enableBlocking").GetBoolean(); - _allowTxtBlockingReport = jsonConfig.GetProperty("allowTxtBlockingReport").GetBoolean(); - Uri mispServerUrl = new Uri(jsonConfig.GetProperty("mispServerUrl").GetString()); - _mispApiKey = jsonConfig.GetProperty("mispApiKey").GetString(); - bool disableTlsValidation = jsonConfig.GetProperty("disableTlsValidation").GetBoolean(); + Directory.CreateDirectory(configDir); + _cacheFilePath = Path.Combine(configDir, "misp_domain_cache.txt"); - string updateIntervalString = jsonConfig.GetProperty("updateInterval").GetString(); - _updateInterval = ParseUpdateInterval(updateIntervalString); + _soaRecord = new DnsSOARecordData(_dnsServer.ServerDomain, _dnsServer.ResponsiblePerson.Address, 1, 14400, 3600, 604800, 60); - _maxIocAge = jsonConfig.GetProperty("maxIocAge").GetString(); + _updateInterval = ParseUpdateInterval(_config.UpdateInterval); + Uri mispServerUrl = new Uri(_config.MispServerUrl); _mispApiUrl = new Uri(mispServerUrl, "/attributes/restSearch"); - _httpClient = CreateHttpClient(mispServerUrl, disableTlsValidation); + _httpClient = CreateHttpClient(mispServerUrl, _config.DisableTlsValidation); + _httpClient.Timeout = TimeSpan.FromSeconds(15); await LoadBlocklistFromCacheAsync(); _updateTimer = new Timer(async _ => @@ -119,7 +115,7 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) _dnsServer.WriteLog($"FATAL: The MispConnector update task failed unexpectedly. Error: {ex.Message}"); _dnsServer.WriteLog(ex); } - }, null, TimeSpan.FromSeconds(1), Timeout.InfiniteTimeSpan); + }, null, TimeSpan.FromSeconds(_random.Next(5,30)), Timeout.InfiniteTimeSpan); } catch (Exception ex) { @@ -135,7 +131,7 @@ public Task IsAllowedAsync(DnsDatagram request, IPEndPoint remoteEP) public Task ProcessRequestAsync(DnsDatagram request, IPEndPoint remoteEP) { - if (!_enableBlocking) + if (_config == null || !_config.EnableBlocking) return Task.FromResult(null); DnsQuestionRecord question = request.Question[0]; @@ -144,7 +140,7 @@ public Task ProcessRequestAsync(DnsDatagram request, IPEndPoint rem return Task.FromResult(null); } - if (_allowTxtBlockingReport && question.Type == DnsResourceRecordType.TXT) + if (_config.AllowTxtBlockingReport && question.Type == DnsResourceRecordType.TXT) { DnsResourceRecord[] answer = new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, 60, new DnsTXTRecordData($"source=misp-connector;domain={blockedDomain}")) }; return Task.FromResult(new DnsDatagram( @@ -251,7 +247,7 @@ private async Task> FetchDomainsFromMispAsync() to_ids = true, deleted = false, limit = 1000, - last = _maxIocAge + last = _config.MaxIocAge }; StringContent requestContent = new StringContent(JsonSerializer.Serialize(requestBody), Encoding.UTF8, "application/json"); @@ -262,7 +258,7 @@ private async Task> FetchDomainsFromMispAsync() Content = requestContent }; - request.Headers.Add("Authorization", _mispApiKey); + request.Headers.Add("Authorization", _config.MispApiKey); request.Headers.Add("Accept", "application/json"); _dnsServer.WriteLog($"Sending API request to {_mispApiUrl}..."); @@ -404,5 +400,37 @@ public string Description } } #endregion + + public class Config + { + [JsonPropertyName("enableBlocking")] + public bool EnableBlocking { get; set; } = true; + + [JsonPropertyName("allowTxtBlockingReport")] + public bool AllowTxtBlockingReport { get; set; } = true; + + [JsonPropertyName("mispServerUrl")] + [Required(ErrorMessage = "mispServerUrl is a required configuration property.")] + [Url(ErrorMessage = "mispServerUrl must be a valid URL.")] + public string MispServerUrl { get; set; } + + [JsonPropertyName("mispApiKey")] + [Required(ErrorMessage = "mispApiKey is a required configuration property.")] + [MinLength(1, ErrorMessage = "mispApiKey cannot be empty.")] + public string MispApiKey { get; set; } + + [JsonPropertyName("disableTlsValidation")] + public bool DisableTlsValidation { get; set; } = false; + + [JsonPropertyName("updateInterval")] + [Required(ErrorMessage = "updateInterval is a required configuration property.")] + [RegularExpression(@"^\d+[mhd]$", ErrorMessage = "Invalid interval format. Use a number followed by 'm', 'h', or 'd' (e.g., '90m', '2h', '7d').", MatchTimeoutInMilliseconds = 3000)] + public string UpdateInterval { get; set; } + + [JsonPropertyName("maxIocAge")] + [Required(ErrorMessage = "maxIocAge is a required configuration property.")] + [RegularExpression(@"^\d+[mhd]$", ErrorMessage = "Invalid interval format. Use a number followed by 'm', 'h', or 'd' (e.g., '90m', '2h', '7d').", MatchTimeoutInMilliseconds = 3000)] + public string MaxIocAge { get; set; } + } } } From 150d667f4931ecce631a319e74ad7b10bbcd3bd0 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Tue, 22 Jul 2025 14:32:46 +0300 Subject: [PATCH 05/22] Added TCP port check before HTTP request --- Apps/MispConnectorApp/App.cs | 51 ++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/Apps/MispConnectorApp/App.cs b/Apps/MispConnectorApp/App.cs index ace9d3b7..2c4a5f50 100644 --- a/Apps/MispConnectorApp/App.cs +++ b/Apps/MispConnectorApp/App.cs @@ -27,6 +27,7 @@ You should have received a copy of the GNU General Public License using System.Net; using System.Net.Http; using System.Net.Security; +using System.Net.Sockets; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; @@ -91,17 +92,11 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) Validator.ValidateObject(_config, new ValidationContext(_config), validateAllProperties: true); - Directory.CreateDirectory(configDir); - _cacheFilePath = Path.Combine(configDir, "misp_domain_cache.txt"); - - _soaRecord = new DnsSOARecordData(_dnsServer.ServerDomain, _dnsServer.ResponsiblePerson.Address, 1, 14400, 3600, 604800, 60); - _updateInterval = ParseUpdateInterval(_config.UpdateInterval); Uri mispServerUrl = new Uri(_config.MispServerUrl); _mispApiUrl = new Uri(mispServerUrl, "/attributes/restSearch"); _httpClient = CreateHttpClient(mispServerUrl, _config.DisableTlsValidation); - _httpClient.Timeout = TimeSpan.FromSeconds(15); await LoadBlocklistFromCacheAsync(); _updateTimer = new Timer(async _ => @@ -223,7 +218,8 @@ private HttpClient CreateHttpClient(Uri serverUrl, bool disableTlsValidation) { Proxy = _dnsServer.Proxy, UseProxy = _dnsServer.Proxy != null, - SslOptions = new SslClientAuthenticationOptions() + SslOptions = new SslClientAuthenticationOptions(), + ConnectTimeout = TimeSpan.FromSeconds(15) }; if (disableTlsValidation) @@ -240,7 +236,6 @@ private HttpClient CreateHttpClient(Uri serverUrl, bool disableTlsValidation) private async Task> FetchDomainsFromMispAsync() { - // The request body can be constructed once. var requestBody = new { type = "domain", @@ -363,10 +358,50 @@ private void ReloadBlocklist(HashSet domains) } } + private async Task CheckTcpPortAsync(Uri serverUri) + { + var host = serverUri.DnsSafeHost; + var port = serverUri.Port; + var timeout = TimeSpan.FromSeconds(5); + + _dnsServer.WriteLog($"Performing pre-flight TCP check for {host}:{port} with a {timeout.TotalSeconds}-second timeout..."); + + try + { + using var cts = new CancellationTokenSource(timeout); + using var client = new TcpClient(); + + await client.ConnectAsync(host, port, cts.Token); + + _dnsServer.WriteLog($"Pre-flight TCP check successful for {host}:{port}."); + return true; + } + catch (OperationCanceledException) + { + _dnsServer.WriteLog($"ERROR: Pre-flight TCP check failed: Connection to {host}:{port} timed out after {timeout.TotalSeconds} seconds. Check firewall rules or network route."); + return false; + } + catch (SocketException ex) + { + _dnsServer.WriteLog($"ERROR: Pre-flight TCP check failed: A network error occurred for {host}:{port}. Error: {ex.Message}"); + return false; + } + catch (Exception ex) + { + _dnsServer.WriteLog($"ERROR: An unexpected error occurred during the pre-flight TCP check for {host}:{port}. Error: {ex.Message}"); + return false; + } + } + private async Task UpdateIocsAsync() { try { + if (!await CheckTcpPortAsync(new Uri(_config.MispServerUrl))) + { + return; + } + _dnsServer.WriteLog("MISP Connector: Starting IOC update..."); HashSet domains = await FetchDomainsFromMispAsync(); await WriteDomainsToCacheAsync(domains); From 2d4c6ed7010a0d2e02dd541585787d9924148792 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Thu, 24 Jul 2025 10:50:01 +0300 Subject: [PATCH 06/22] Used pagination instead of loading al the query results --- Apps/MispConnectorApp/App.cs | 146 ++++++++++++++++------------ Apps/MispConnectorApp/dnsApp.config | 15 +-- 2 files changed, 92 insertions(+), 69 deletions(-) diff --git a/Apps/MispConnectorApp/App.cs b/Apps/MispConnectorApp/App.cs index 2c4a5f50..2a246866 100644 --- a/Apps/MispConnectorApp/App.cs +++ b/Apps/MispConnectorApp/App.cs @@ -126,7 +126,7 @@ public Task IsAllowedAsync(DnsDatagram request, IPEndPoint remoteEP) public Task ProcessRequestAsync(DnsDatagram request, IPEndPoint remoteEP) { - if (_config == null || !_config.EnableBlocking) + if (_config?.EnableBlocking != true) return Task.FromResult(null); DnsQuestionRecord question = request.Question[0]; @@ -236,82 +236,83 @@ private HttpClient CreateHttpClient(Uri serverUrl, bool disableTlsValidation) private async Task> FetchDomainsFromMispAsync() { - var requestBody = new - { - type = "domain", - to_ids = true, - deleted = false, - limit = 1000, - last = _config.MaxIocAge - }; - StringContent requestContent = new StringContent(JsonSerializer.Serialize(requestBody), Encoding.UTF8, "application/json"); + HashSet domains = new HashSet(StringComparer.OrdinalIgnoreCase); + int page = 1; + int limit = _config.PaginationLimit; + bool hasMorePages = true; - try + _dnsServer.WriteLog($"Starting paginated fetch from MISP API with a page size of {limit}..."); + + while (hasMorePages) { - using HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, _mispApiUrl) + var requestBody = new { - Content = requestContent + type = "domain", + to_ids = true, + deleted = false, + last = _config.MaxIocAge, + limit, + page }; + StringContent requestContent = new StringContent(JsonSerializer.Serialize(requestBody), Encoding.UTF8, "application/json"); - request.Headers.Add("Authorization", _config.MispApiKey); - request.Headers.Add("Accept", "application/json"); + try + { + using HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, _mispApiUrl) { Content = requestContent }; + request.Headers.Add("Authorization", _config.MispApiKey); + request.Headers.Add("Accept", "application/json"); - _dnsServer.WriteLog($"Sending API request to {_mispApiUrl}..."); - using HttpResponseMessage response = await _httpClient.SendAsync(request); + _dnsServer.WriteLog($"Fetching page {page}..."); + using HttpResponseMessage response = await _httpClient.SendAsync(request); - if (!response.IsSuccessStatusCode) - { - string errorBody = await response.Content.ReadAsStringAsync(); - _dnsServer.WriteLog($"ERROR: MISP API returned a non-success status code: {(int)response.StatusCode} {response.ReasonPhrase}. Response Body: {errorBody}"); + if (!response.IsSuccessStatusCode) + { + string errorBody = await response.Content.ReadAsStringAsync(); + throw new HttpRequestException($"MISP API request failed on page {page} with status code {response.StatusCode}. Body: {errorBody}", null, response.StatusCode); + } - throw new HttpRequestException($"MISP API request failed with status code {response.StatusCode}.", null, response.StatusCode); - } + await using Stream responseStream = await response.Content.ReadAsStreamAsync(); + MispResponse mispResponse = await JsonSerializer.DeserializeAsync(responseStream); - _dnsServer.WriteLog("API request successful. Parsing response..."); - await using Stream responseStream = await response.Content.ReadAsStreamAsync(); - using JsonDocument jsonDoc = await JsonDocument.ParseAsync(responseStream); + List attributes = mispResponse?.Response?.Attribute; + if (attributes?.Count == 0) + { + // No more attributes found, we're done. + hasMorePages = false; + continue; + } - HashSet domains = new HashSet(StringComparer.OrdinalIgnoreCase); - if (jsonDoc.RootElement.TryGetProperty("response", out JsonElement responseElement) && - responseElement.TryGetProperty("Attribute", out JsonElement attributeArray) && - attributeArray.ValueKind == JsonValueKind.Array) - { - foreach (JsonElement attributeElement in attributeArray.EnumerateArray()) + foreach (MispAttribute attribute in attributes) { - if (attributeElement.TryGetProperty("value", out JsonElement valueElement) && valueElement.ValueKind == JsonValueKind.String) + string domain = attribute.Value?.Trim().ToLowerInvariant(); + if (!string.IsNullOrEmpty(domain) && DnsClient.IsDomainNameValid(domain)) { - string domain = valueElement.GetString()?.Trim().ToLowerInvariant(); - if (!string.IsNullOrEmpty(domain) && DnsClient.IsDomainNameValid(domain)) - { - domains.Add(domain); - } + domains.Add(domain); } } + + // Assumption: If we received fewer items than our limit, it must be the last page. + if (attributes.Count < limit) + { + hasMorePages = false; + } + else + { + page++; + } } - else + catch (Exception ex) { - _dnsServer.WriteLog("WARNING: MISP API response was successful but did not contain the expected 'response.Attribute' array structure."); + _dnsServer.WriteLog($"ERROR: Failed while fetching page {page}. Halting update cycle. Error: {ex.Message}"); + throw; } - - return domains; - } - catch (HttpRequestException ex) - { - _dnsServer.WriteLog($"ERROR: A network or HTTP error occurred while communicating with MISP. Error: {ex.Message}"); - throw; - } - catch (JsonException ex) - { - _dnsServer.WriteLog($"ERROR: Failed to parse the JSON response from MISP. The response may not be valid JSON. Error: {ex.Message}"); - throw; - } - catch (Exception ex) - { - _dnsServer.WriteLog($"ERROR: An unexpected error occurred during the fetch process. Error: {ex.Message}"); - throw; } + + _dnsServer.WriteLog($"Finished paginated fetch. Total unique domains collected: {domains.Count}"); + return domains; } + private bool IsDomainBlocked(string domain, out string foundZone) { lock (_blocklistLock) @@ -360,16 +361,16 @@ private void ReloadBlocklist(HashSet domains) private async Task CheckTcpPortAsync(Uri serverUri) { - var host = serverUri.DnsSafeHost; - var port = serverUri.Port; - var timeout = TimeSpan.FromSeconds(5); + string host = serverUri.DnsSafeHost; + int port = serverUri.Port; + TimeSpan timeout = TimeSpan.FromSeconds(5); _dnsServer.WriteLog($"Performing pre-flight TCP check for {host}:{port} with a {timeout.TotalSeconds}-second timeout..."); try { - using var cts = new CancellationTokenSource(timeout); - using var client = new TcpClient(); + using CancellationTokenSource cts = new CancellationTokenSource(timeout); + using TcpClient client = new TcpClient(); await client.ConnectAsync(host, port, cts.Token); @@ -466,6 +467,27 @@ public class Config [Required(ErrorMessage = "maxIocAge is a required configuration property.")] [RegularExpression(@"^\d+[mhd]$", ErrorMessage = "Invalid interval format. Use a number followed by 'm', 'h', or 'd' (e.g., '90m', '2h', '7d').", MatchTimeoutInMilliseconds = 3000)] public string MaxIocAge { get; set; } + + [JsonPropertyName("paginationLimit")] + public int PaginationLimit { get; set; } = 5000; + } + + class MispResponse + { + [JsonPropertyName("response")] + public MispResponseData Response { get; set; } + } + + class MispResponseData + { + [JsonPropertyName("Attribute")] + public List Attribute { get; set; } + } + + class MispAttribute + { + [JsonPropertyName("value")] + public string Value { get; set; } } } } diff --git a/Apps/MispConnectorApp/dnsApp.config b/Apps/MispConnectorApp/dnsApp.config index 256b55fa..74537156 100644 --- a/Apps/MispConnectorApp/dnsApp.config +++ b/Apps/MispConnectorApp/dnsApp.config @@ -1,9 +1,10 @@ { -"enableBlocking": true, -"mispServerUrl": "https://misp.example.com", -"mispApiKey": "YourMispApiKeyHere", -"disableTlsValidation": false, -"updateInterval": "2h", -"maxIocAge": "90d", -"allowTxtBlockingReport": true + "enableBlocking": true, + "mispServerUrl": "https://misp.example.com", + "mispApiKey": "YourMispApiKeyHere", + "disableTlsValidation": false, + "updateInterval": "2h", + "maxIocAge": "90d", + "allowTxtBlockingReport": true, + "paginationLimit": 5000 } From 764ade52f6103e104580f8fc91fc0614e993fc4e Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Thu, 24 Jul 2025 10:58:02 +0300 Subject: [PATCH 07/22] Added Extended DNS Error (RFC 8914) to block responses --- Apps/MispConnectorApp/App.cs | 28 +++++++++++++++++++++++++--- Apps/MispConnectorApp/dnsApp.config | 3 ++- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/Apps/MispConnectorApp/App.cs b/Apps/MispConnectorApp/App.cs index 2a246866..edc63091 100644 --- a/Apps/MispConnectorApp/App.cs +++ b/Apps/MispConnectorApp/App.cs @@ -34,6 +34,7 @@ You should have received a copy of the GNU General Public License using System.Threading; using System.Threading.Tasks; using TechnitiumLibrary.Net.Dns; +using TechnitiumLibrary.Net.Dns.EDnsOptions; using TechnitiumLibrary.Net.Dns.ResourceRecords; using TechnitiumLibrary.Net.Http.Client; @@ -110,7 +111,7 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) _dnsServer.WriteLog($"FATAL: The MispConnector update task failed unexpectedly. Error: {ex.Message}"); _dnsServer.WriteLog(ex); } - }, null, TimeSpan.FromSeconds(_random.Next(5,30)), Timeout.InfiniteTimeSpan); + }, null, TimeSpan.FromSeconds(_random.Next(5, 30)), Timeout.InfiniteTimeSpan); } catch (Exception ex) { @@ -135,6 +136,18 @@ public Task ProcessRequestAsync(DnsDatagram request, IPEndPoint rem return Task.FromResult(null); } + List options = null; + if (_config.AddExtendedDnsError && request.EDNS is not null) + { + options = new List + { + new EDnsOption( + EDnsOptionCode.EXTENDED_DNS_ERROR, + new EDnsExtendedDnsErrorOptionData(EDnsExtendedDnsErrorCode.Blocked, "Blocked by MISP Connector") + ) + }; + } + if (_config.AllowTxtBlockingReport && question.Type == DnsResourceRecordType.TXT) { DnsResourceRecord[] answer = new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, 60, new DnsTXTRecordData($"source=misp-connector;domain={blockedDomain}")) }; @@ -152,7 +165,10 @@ public Task ProcessRequestAsync(DnsDatagram request, IPEndPoint rem question: request.Question, answer: answer, authority: null, - additional: null + additional: null, + udpPayloadSize: request.EDNS is null ? ushort.MinValue : _dnsServer.UdpPayloadSize, + ednsFlags: EDnsHeaderFlags.None, + options: options )); } @@ -171,7 +187,10 @@ public Task ProcessRequestAsync(DnsDatagram request, IPEndPoint rem question: request.Question, answer: null, authority: authority, - additional: null + additional: null, + udpPayloadSize: request.EDNS is null ? ushort.MinValue : _dnsServer.UdpPayloadSize, + ednsFlags: EDnsHeaderFlags.None, + options: options )); } @@ -470,6 +489,9 @@ public class Config [JsonPropertyName("paginationLimit")] public int PaginationLimit { get; set; } = 5000; + + [JsonPropertyName("addExtendedDnsError")] + public bool AddExtendedDnsError { get; set; } = true; } class MispResponse diff --git a/Apps/MispConnectorApp/dnsApp.config b/Apps/MispConnectorApp/dnsApp.config index 74537156..28bdc1fd 100644 --- a/Apps/MispConnectorApp/dnsApp.config +++ b/Apps/MispConnectorApp/dnsApp.config @@ -6,5 +6,6 @@ "updateInterval": "2h", "maxIocAge": "90d", "allowTxtBlockingReport": true, - "paginationLimit": 5000 + "paginationLimit": 5000, + "addExtendedDnsError": true } From 089a95292c6b1b4e2bb4e9471de1466436dec6f6 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Thu, 24 Jul 2025 11:03:46 +0300 Subject: [PATCH 08/22] Used a POCO for MISP request --- Apps/MispConnectorApp/App.cs | 37 +++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/Apps/MispConnectorApp/App.cs b/Apps/MispConnectorApp/App.cs index edc63091..bea5bfc9 100644 --- a/Apps/MispConnectorApp/App.cs +++ b/Apps/MispConnectorApp/App.cs @@ -264,15 +264,14 @@ private async Task> FetchDomainsFromMispAsync() while (hasMorePages) { - var requestBody = new - { - type = "domain", - to_ids = true, - deleted = false, - last = _config.MaxIocAge, - limit, - page - }; + var requestBody = new MispRequestBody(); + requestBody.Type = "domain"; + requestBody.To_ids = true; + requestBody.Deleted = false; + requestBody.Last = _config.MaxIocAge; + requestBody.Limit = limit; + requestBody.Page = page; + StringContent requestContent = new StringContent(JsonSerializer.Serialize(requestBody), Encoding.UTF8, "application/json"); try @@ -511,5 +510,25 @@ class MispAttribute [JsonPropertyName("value")] public string Value { get; set; } } + class MispRequestBody + { + [JsonPropertyName("type")] + public string Type { get; set; } + + [JsonPropertyName("to_ids")] + public bool To_ids { get; set; } + + [JsonPropertyName("deleted")] + public bool Deleted { get; set; } + + [JsonPropertyName("last")] + public string Last { get; set; } + + [JsonPropertyName("limit")] + public int Limit { get; set; } + + [JsonPropertyName("page")] + public int Page { get; set; } + } } } From 1e1d8c22a541ccb78dc8687b27d646dc92f71577 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Thu, 24 Jul 2025 11:16:30 +0300 Subject: [PATCH 09/22] Standardized EDNS and TXT report for blocking --- Apps/MispConnectorApp/App.cs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/Apps/MispConnectorApp/App.cs b/Apps/MispConnectorApp/App.cs index bea5bfc9..4c0a23dd 100644 --- a/Apps/MispConnectorApp/App.cs +++ b/Apps/MispConnectorApp/App.cs @@ -136,21 +136,16 @@ public Task ProcessRequestAsync(DnsDatagram request, IPEndPoint rem return Task.FromResult(null); } - List options = null; + string blockingReport = $"source=misp-connector;domain={blockedDomain}"; + EDnsOption[] options = null; if (_config.AddExtendedDnsError && request.EDNS is not null) { - options = new List - { - new EDnsOption( - EDnsOptionCode.EXTENDED_DNS_ERROR, - new EDnsExtendedDnsErrorOptionData(EDnsExtendedDnsErrorCode.Blocked, "Blocked by MISP Connector") - ) - }; + options = new EDnsOption[] { new EDnsOption(EDnsOptionCode.EXTENDED_DNS_ERROR, new EDnsExtendedDnsErrorOptionData(EDnsExtendedDnsErrorCode.Blocked, blockingReport)) }; } if (_config.AllowTxtBlockingReport && question.Type == DnsResourceRecordType.TXT) { - DnsResourceRecord[] answer = new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, 60, new DnsTXTRecordData($"source=misp-connector;domain={blockedDomain}")) }; + DnsResourceRecord[] answer = new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, 60, new DnsTXTRecordData(blockingReport)) }; return Task.FromResult(new DnsDatagram( ID: request.Identifier, isResponse: true, From 825c052708c578945c596a33b759089a34864725 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Thu, 24 Jul 2025 11:20:32 +0300 Subject: [PATCH 10/22] Used ReadOnlySpan instead of string for temporary allocations --- Apps/MispConnectorApp/App.cs | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/Apps/MispConnectorApp/App.cs b/Apps/MispConnectorApp/App.cs index 4c0a23dd..905196f4 100644 --- a/Apps/MispConnectorApp/App.cs +++ b/Apps/MispConnectorApp/App.cs @@ -328,19 +328,31 @@ private async Task> FetchDomainsFromMispAsync() private bool IsDomainBlocked(string domain, out string foundZone) { + ReadOnlySpan domainSpan = domain.AsSpan(); + lock (_blocklistLock) { - string currentDomain = domain.ToLowerInvariant(); - do + ReadOnlySpan currentSpan = domainSpan; + while (true) { - if (_globalBlocklist.Contains(currentDomain)) + // To look up in a HashSet, we must provide a string. + string key = new string(currentSpan); + if (_globalBlocklist.TryGetValue(key, out foundZone)) { - foundZone = currentDomain; return true; } - currentDomain = GetParentZone(currentDomain); - } while (currentDomain != null); + + int dotIndex = currentSpan.IndexOf('.'); + if (dotIndex == -1) + { + break; // No more parent domains. + } + + // Slice to the parent domain view. No allocation here. + currentSpan = currentSpan.Slice(dotIndex + 1); + } } + foundZone = null; return false; } From 0114b8cc1be6b6a7780c6c2820a8db4ab9a2441f Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Thu, 24 Jul 2025 11:25:41 +0300 Subject: [PATCH 11/22] Used lock-free global blocklist for concurrent scenarios --- Apps/MispConnectorApp/App.cs | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/Apps/MispConnectorApp/App.cs b/Apps/MispConnectorApp/App.cs index 905196f4..589f041d 100644 --- a/Apps/MispConnectorApp/App.cs +++ b/Apps/MispConnectorApp/App.cs @@ -48,7 +48,7 @@ public sealed class App : IDnsApplication, IDnsRequestBlockingHandler readonly object _blocklistLock = new object(); - readonly HashSet _globalBlocklist = new HashSet(StringComparer.OrdinalIgnoreCase); + volatile HashSet _globalBlocklist = new HashSet(StringComparer.OrdinalIgnoreCase); readonly Random _random = new Random(); @@ -259,7 +259,7 @@ private async Task> FetchDomainsFromMispAsync() while (hasMorePages) { - var requestBody = new MispRequestBody(); + MispRequestBody requestBody = new MispRequestBody(); requestBody.Type = "domain"; requestBody.To_ids = true; requestBody.Deleted = false; @@ -328,6 +328,9 @@ private async Task> FetchDomainsFromMispAsync() private bool IsDomainBlocked(string domain, out string foundZone) { + HashSet currentBlocklist = _globalBlocklist; + Thread.MemoryBarrier(); + ReadOnlySpan domainSpan = domain.AsSpan(); lock (_blocklistLock) @@ -337,7 +340,7 @@ private bool IsDomainBlocked(string domain, out string foundZone) { // To look up in a HashSet, we must provide a string. string key = new string(currentSpan); - if (_globalBlocklist.TryGetValue(key, out foundZone)) + if (currentBlocklist.TryGetValue(key, out foundZone)) { return true; } @@ -372,16 +375,10 @@ private async Task LoadBlocklistFromCacheAsync() } } - private void ReloadBlocklist(HashSet domains) + private void ReloadBlocklist(HashSet newBlocklist) { - lock (_blocklistLock) - { - _globalBlocklist.Clear(); - foreach (string domain in domains) - { - _globalBlocklist.Add(domain); - } - } + Thread.MemoryBarrier(); + _globalBlocklist = newBlocklist; } private async Task CheckTcpPortAsync(Uri serverUri) From c6213e22dbb138d11c04416a975ca5dd9d3f7649 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Thu, 24 Jul 2025 11:27:32 +0300 Subject: [PATCH 12/22] Formatting and reorganization --- Apps/MispConnectorApp/App.cs | 179 ++++++++++++++++++----------------- 1 file changed, 90 insertions(+), 89 deletions(-) diff --git a/Apps/MispConnectorApp/App.cs b/Apps/MispConnectorApp/App.cs index 589f041d..034c40a6 100644 --- a/Apps/MispConnectorApp/App.cs +++ b/Apps/MispConnectorApp/App.cs @@ -44,17 +44,12 @@ public sealed class App : IDnsApplication, IDnsRequestBlockingHandler { #region variables - Config _config; - readonly object _blocklistLock = new object(); - - volatile HashSet _globalBlocklist = new HashSet(StringComparer.OrdinalIgnoreCase); - readonly Random _random = new Random(); - string _cacheFilePath; - + Config _config; IDnsServer _dnsServer; + volatile HashSet _globalBlocklist = new HashSet(StringComparer.OrdinalIgnoreCase); HttpClient _httpClient; Uri _mispApiUrl; @@ -63,7 +58,8 @@ public sealed class App : IDnsApplication, IDnsRequestBlockingHandler TimeSpan _updateInterval; Timer _updateTimer; - #endregion + + #endregion variables #region IDisposable @@ -73,7 +69,7 @@ public void Dispose() _httpClient?.Dispose(); } - #endregion + #endregion IDisposable #region public @@ -189,9 +185,10 @@ public Task ProcessRequestAsync(DnsDatagram request, IPEndPoint rem )); } - #endregion + #endregion public #region private + private static string GetParentZone(string domain) { int i = domain.IndexOf('.'); @@ -217,15 +214,53 @@ private static TimeSpan ParseUpdateInterval(string interval) { case "m": return TimeSpan.FromMinutes(value); + case "h": return TimeSpan.FromHours(value); + case "d": return TimeSpan.FromDays(value); + default: throw new FormatException($"Invalid unit '{unit}' in update interval. Allowed units are 'm', 'h', 'd'."); } } + private async Task CheckTcpPortAsync(Uri serverUri) + { + string host = serverUri.DnsSafeHost; + int port = serverUri.Port; + TimeSpan timeout = TimeSpan.FromSeconds(5); + + _dnsServer.WriteLog($"Performing pre-flight TCP check for {host}:{port} with a {timeout.TotalSeconds}-second timeout..."); + + try + { + using CancellationTokenSource cts = new CancellationTokenSource(timeout); + using TcpClient client = new TcpClient(); + + await client.ConnectAsync(host, port, cts.Token); + + _dnsServer.WriteLog($"Pre-flight TCP check successful for {host}:{port}."); + return true; + } + catch (OperationCanceledException) + { + _dnsServer.WriteLog($"ERROR: Pre-flight TCP check failed: Connection to {host}:{port} timed out after {timeout.TotalSeconds} seconds. Check firewall rules or network route."); + return false; + } + catch (SocketException ex) + { + _dnsServer.WriteLog($"ERROR: Pre-flight TCP check failed: A network error occurred for {host}:{port}. Error: {ex.Message}"); + return false; + } + catch (Exception ex) + { + _dnsServer.WriteLog($"ERROR: An unexpected error occurred during the pre-flight TCP check for {host}:{port}. Error: {ex.Message}"); + return false; + } + } + private HttpClient CreateHttpClient(Uri serverUrl, bool disableTlsValidation) { SocketsHttpHandler handler = new SocketsHttpHandler @@ -325,7 +360,6 @@ private async Task> FetchDomainsFromMispAsync() return domains; } - private bool IsDomainBlocked(string domain, out string foundZone) { HashSet currentBlocklist = _globalBlocklist; @@ -380,42 +414,6 @@ private void ReloadBlocklist(HashSet newBlocklist) Thread.MemoryBarrier(); _globalBlocklist = newBlocklist; } - - private async Task CheckTcpPortAsync(Uri serverUri) - { - string host = serverUri.DnsSafeHost; - int port = serverUri.Port; - TimeSpan timeout = TimeSpan.FromSeconds(5); - - _dnsServer.WriteLog($"Performing pre-flight TCP check for {host}:{port} with a {timeout.TotalSeconds}-second timeout..."); - - try - { - using CancellationTokenSource cts = new CancellationTokenSource(timeout); - using TcpClient client = new TcpClient(); - - await client.ConnectAsync(host, port, cts.Token); - - _dnsServer.WriteLog($"Pre-flight TCP check successful for {host}:{port}."); - return true; - } - catch (OperationCanceledException) - { - _dnsServer.WriteLog($"ERROR: Pre-flight TCP check failed: Connection to {host}:{port} timed out after {timeout.TotalSeconds} seconds. Check firewall rules or network route."); - return false; - } - catch (SocketException ex) - { - _dnsServer.WriteLog($"ERROR: Pre-flight TCP check failed: A network error occurred for {host}:{port}. Error: {ex.Message}"); - return false; - } - catch (Exception ex) - { - _dnsServer.WriteLog($"ERROR: An unexpected error occurred during the pre-flight TCP check for {host}:{port}. Error: {ex.Message}"); - return false; - } - } - private async Task UpdateIocsAsync() { try @@ -441,15 +439,18 @@ private async Task UpdateIocsAsync() _updateTimer?.Change(nextInterval, Timeout.InfiniteTimeSpan); } } + private async Task WriteDomainsToCacheAsync(HashSet domains) { string tempPath = _cacheFilePath + ".tmp"; await File.WriteAllLinesAsync(tempPath, domains); File.Move(tempPath, _cacheFilePath, true); } - #endregion + + #endregion private #region properties + public string Description { get @@ -457,71 +458,53 @@ public string Description return "A focused connector that imports domain IOCs from a MISP server to block malicious domains using direct REST API calls."; } } - #endregion + + #endregion properties public class Config { - [JsonPropertyName("enableBlocking")] - public bool EnableBlocking { get; set; } = true; + [JsonPropertyName("addExtendedDnsError")] + public bool AddExtendedDnsError { get; set; } = true; [JsonPropertyName("allowTxtBlockingReport")] public bool AllowTxtBlockingReport { get; set; } = true; - [JsonPropertyName("mispServerUrl")] - [Required(ErrorMessage = "mispServerUrl is a required configuration property.")] - [Url(ErrorMessage = "mispServerUrl must be a valid URL.")] - public string MispServerUrl { get; set; } - - [JsonPropertyName("mispApiKey")] - [Required(ErrorMessage = "mispApiKey is a required configuration property.")] - [MinLength(1, ErrorMessage = "mispApiKey cannot be empty.")] - public string MispApiKey { get; set; } - [JsonPropertyName("disableTlsValidation")] public bool DisableTlsValidation { get; set; } = false; - [JsonPropertyName("updateInterval")] - [Required(ErrorMessage = "updateInterval is a required configuration property.")] - [RegularExpression(@"^\d+[mhd]$", ErrorMessage = "Invalid interval format. Use a number followed by 'm', 'h', or 'd' (e.g., '90m', '2h', '7d').", MatchTimeoutInMilliseconds = 3000)] - public string UpdateInterval { get; set; } - + [JsonPropertyName("enableBlocking")] + public bool EnableBlocking { get; set; } = true; [JsonPropertyName("maxIocAge")] [Required(ErrorMessage = "maxIocAge is a required configuration property.")] [RegularExpression(@"^\d+[mhd]$", ErrorMessage = "Invalid interval format. Use a number followed by 'm', 'h', or 'd' (e.g., '90m', '2h', '7d').", MatchTimeoutInMilliseconds = 3000)] public string MaxIocAge { get; set; } + [JsonPropertyName("mispApiKey")] + [Required(ErrorMessage = "mispApiKey is a required configuration property.")] + [MinLength(1, ErrorMessage = "mispApiKey cannot be empty.")] + public string MispApiKey { get; set; } + + [JsonPropertyName("mispServerUrl")] + [Required(ErrorMessage = "mispServerUrl is a required configuration property.")] + [Url(ErrorMessage = "mispServerUrl must be a valid URL.")] + public string MispServerUrl { get; set; } [JsonPropertyName("paginationLimit")] public int PaginationLimit { get; set; } = 5000; - [JsonPropertyName("addExtendedDnsError")] - public bool AddExtendedDnsError { get; set; } = true; - } - - class MispResponse - { - [JsonPropertyName("response")] - public MispResponseData Response { get; set; } - } - - class MispResponseData - { - [JsonPropertyName("Attribute")] - public List Attribute { get; set; } + [JsonPropertyName("updateInterval")] + [Required(ErrorMessage = "updateInterval is a required configuration property.")] + [RegularExpression(@"^\d+[mhd]$", ErrorMessage = "Invalid interval format. Use a number followed by 'm', 'h', or 'd' (e.g., '90m', '2h', '7d').", MatchTimeoutInMilliseconds = 3000)] + public string UpdateInterval { get; set; } } - class MispAttribute + private class MispAttribute { [JsonPropertyName("value")] public string Value { get; set; } } - class MispRequestBody - { - [JsonPropertyName("type")] - public string Type { get; set; } - - [JsonPropertyName("to_ids")] - public bool To_ids { get; set; } + private class MispRequestBody + { [JsonPropertyName("deleted")] public bool Deleted { get; set; } @@ -533,6 +516,24 @@ class MispRequestBody [JsonPropertyName("page")] public int Page { get; set; } + + [JsonPropertyName("to_ids")] + public bool To_ids { get; set; } + + [JsonPropertyName("type")] + public string Type { get; set; } + } + + private class MispResponse + { + [JsonPropertyName("response")] + public MispResponseData Response { get; set; } + } + + private class MispResponseData + { + [JsonPropertyName("Attribute")] + public List Attribute { get; set; } } } -} +} \ No newline at end of file From 7b8398b7cf70f362487453bb84717bf7502f3cdf Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Thu, 24 Jul 2025 11:41:11 +0300 Subject: [PATCH 13/22] Removed leftover lock from previous commit --- Apps/MispConnectorApp/App.cs | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/Apps/MispConnectorApp/App.cs b/Apps/MispConnectorApp/App.cs index 034c40a6..ad0001b0 100644 --- a/Apps/MispConnectorApp/App.cs +++ b/Apps/MispConnectorApp/App.cs @@ -365,29 +365,25 @@ private bool IsDomainBlocked(string domain, out string foundZone) HashSet currentBlocklist = _globalBlocklist; Thread.MemoryBarrier(); - ReadOnlySpan domainSpan = domain.AsSpan(); + ReadOnlySpan currentSpan = domain.AsSpan(); - lock (_blocklistLock) + while (true) { - ReadOnlySpan currentSpan = domainSpan; - while (true) + // To look up in a HashSet, we must provide a string. + string key = new string(currentSpan); + if (currentBlocklist.TryGetValue(key, out foundZone)) { - // To look up in a HashSet, we must provide a string. - string key = new string(currentSpan); - if (currentBlocklist.TryGetValue(key, out foundZone)) - { - return true; - } - - int dotIndex = currentSpan.IndexOf('.'); - if (dotIndex == -1) - { - break; // No more parent domains. - } + return true; + } - // Slice to the parent domain view. No allocation here. - currentSpan = currentSpan.Slice(dotIndex + 1); + int dotIndex = currentSpan.IndexOf('.'); + if (dotIndex == -1) + { + break; // No more parent domains. } + + // Slice to the parent domain view. No allocation here. + currentSpan = currentSpan.Slice(dotIndex + 1); } foundZone = null; From 34a8fcccefdbaeff30c68544ac6d3a0b9053980b Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Thu, 24 Jul 2025 11:50:25 +0300 Subject: [PATCH 14/22] Removed unnecessary parent domain check --- Apps/MispConnectorApp/App.cs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Apps/MispConnectorApp/App.cs b/Apps/MispConnectorApp/App.cs index ad0001b0..1465a08e 100644 --- a/Apps/MispConnectorApp/App.cs +++ b/Apps/MispConnectorApp/App.cs @@ -163,7 +163,7 @@ public Task ProcessRequestAsync(DnsDatagram request, IPEndPoint rem )); } - DnsResourceRecord[] authority = new DnsResourceRecord[] { new DnsResourceRecord(GetParentZone(blockedDomain) ?? string.Empty, DnsResourceRecordType.SOA, question.Class, 60, _soaRecord) }; + DnsResourceRecord[] authority = { new DnsResourceRecord(blockedDomain, DnsResourceRecordType.SOA, question.Class, 60, _soaRecord) }; return Task.FromResult(new DnsDatagram( ID: request.Identifier, isResponse: true, @@ -189,12 +189,6 @@ public Task ProcessRequestAsync(DnsDatagram request, IPEndPoint rem #region private - private static string GetParentZone(string domain) - { - int i = domain.IndexOf('.'); - return (i > -1) ? domain.Substring(i + 1) : null; - } - private static TimeSpan ParseUpdateInterval(string interval) { if (string.IsNullOrWhiteSpace(interval) || interval.Length < 2) From 6f85301bff4ef566a701f4d6f7ff94effff14802 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Thu, 24 Jul 2025 12:07:25 +0300 Subject: [PATCH 15/22] Made config private --- Apps/MispConnectorApp/App.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Apps/MispConnectorApp/App.cs b/Apps/MispConnectorApp/App.cs index 1465a08e..c1f273b6 100644 --- a/Apps/MispConnectorApp/App.cs +++ b/Apps/MispConnectorApp/App.cs @@ -451,7 +451,7 @@ public string Description #endregion properties - public class Config + private class Config { [JsonPropertyName("addExtendedDnsError")] public bool AddExtendedDnsError { get; set; } = true; From d5fec64f40d6881d9a98f328cc24f86144015212 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Thu, 24 Jul 2025 13:21:46 +0300 Subject: [PATCH 16/22] Used frozen sets instead of hand-made lock-free hashsets --- Apps/MispConnectorApp/App.cs | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/Apps/MispConnectorApp/App.cs b/Apps/MispConnectorApp/App.cs index c1f273b6..92b3e702 100644 --- a/Apps/MispConnectorApp/App.cs +++ b/Apps/MispConnectorApp/App.cs @@ -19,6 +19,7 @@ You should have received a copy of the GNU General Public License using DnsServerCore.ApplicationCommon; using System; +using System.Collections.Frozen; using System.Collections.Generic; using System.ComponentModel.DataAnnotations; using System.Globalization; @@ -44,12 +45,11 @@ public sealed class App : IDnsApplication, IDnsRequestBlockingHandler { #region variables - readonly object _blocklistLock = new object(); readonly Random _random = new Random(); string _cacheFilePath; Config _config; IDnsServer _dnsServer; - volatile HashSet _globalBlocklist = new HashSet(StringComparer.OrdinalIgnoreCase); + private FrozenSet _globalBlocklist = FrozenSet.Empty; HttpClient _httpClient; Uri _mispApiUrl; @@ -277,7 +277,7 @@ private HttpClient CreateHttpClient(Uri serverUrl, bool disableTlsValidation) return new HttpClient(new HttpClientNetworkHandler(handler, _dnsServer.PreferIPv6 ? HttpClientNetworkType.PreferIPv6 : HttpClientNetworkType.Default, _dnsServer)); } - private async Task> FetchDomainsFromMispAsync() + private async Task> FetchDomainsFromMispAsync() { HashSet domains = new HashSet(StringComparer.OrdinalIgnoreCase); int page = 1; @@ -350,14 +350,13 @@ private async Task> FetchDomainsFromMispAsync() } } - _dnsServer.WriteLog($"Finished paginated fetch. Total unique domains collected: {domains.Count}"); - return domains; + _dnsServer.WriteLog($"Finished paginated fetch. Freezing {domains.Count} domains for optimal read performance..."); + return domains.ToFrozenSet(StringComparer.OrdinalIgnoreCase); } private bool IsDomainBlocked(string domain, out string foundZone) { - HashSet currentBlocklist = _globalBlocklist; - Thread.MemoryBarrier(); + FrozenSet currentBlocklist = _globalBlocklist; ReadOnlySpan currentSpan = domain.AsSpan(); @@ -389,7 +388,7 @@ private async Task LoadBlocklistFromCacheAsync() if (!File.Exists(_cacheFilePath)) return; try { - HashSet domains = (await File.ReadAllLinesAsync(_cacheFilePath)).ToHashSet(StringComparer.OrdinalIgnoreCase); + FrozenSet domains = (await File.ReadAllLinesAsync(_cacheFilePath)).ToHashSet(StringComparer.OrdinalIgnoreCase).ToFrozenSet(StringComparer.OrdinalIgnoreCase); ReloadBlocklist(domains); _dnsServer.WriteLog($"MISP Connector: Loaded {domains.Count} domains from cache."); } @@ -399,11 +398,11 @@ private async Task LoadBlocklistFromCacheAsync() } } - private void ReloadBlocklist(HashSet newBlocklist) + private void ReloadBlocklist(FrozenSet newBlocklist) { - Thread.MemoryBarrier(); - _globalBlocklist = newBlocklist; + Interlocked.Exchange(ref _globalBlocklist, newBlocklist); } + private async Task UpdateIocsAsync() { try @@ -414,7 +413,7 @@ private async Task UpdateIocsAsync() } _dnsServer.WriteLog("MISP Connector: Starting IOC update..."); - HashSet domains = await FetchDomainsFromMispAsync(); + FrozenSet domains = await FetchDomainsFromMispAsync(); await WriteDomainsToCacheAsync(domains); ReloadBlocklist(domains); _dnsServer.WriteLog($"MISP Connector: Successfully updated blocklist with {domains.Count} domains."); @@ -430,7 +429,7 @@ private async Task UpdateIocsAsync() } } - private async Task WriteDomainsToCacheAsync(HashSet domains) + private async Task WriteDomainsToCacheAsync(FrozenSet domains) { string tempPath = _cacheFilePath + ".tmp"; await File.WriteAllLinesAsync(tempPath, domains); From 0853baeb2c2e1450295533b727b827949edc5138 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Thu, 24 Jul 2025 14:35:08 +0300 Subject: [PATCH 17/22] Aded a retry mechanism and failure handling when a query fails during pagination --- Apps/MispConnectorApp/App.cs | 112 +++++++++++++++++++++-------------- 1 file changed, 68 insertions(+), 44 deletions(-) diff --git a/Apps/MispConnectorApp/App.cs b/Apps/MispConnectorApp/App.cs index 92b3e702..8e5c885b 100644 --- a/Apps/MispConnectorApp/App.cs +++ b/Apps/MispConnectorApp/App.cs @@ -285,68 +285,92 @@ private async Task> FetchDomainsFromMispAsync() bool hasMorePages = true; _dnsServer.WriteLog($"Starting paginated fetch from MISP API with a page size of {limit}..."); + const int maxRetries = 3; while (hasMorePages) { - MispRequestBody requestBody = new MispRequestBody(); - requestBody.Type = "domain"; - requestBody.To_ids = true; - requestBody.Deleted = false; - requestBody.Last = _config.MaxIocAge; - requestBody.Limit = limit; - requestBody.Page = page; + int attempt = 0; + MispResponse mispResponse = null; - StringContent requestContent = new StringContent(JsonSerializer.Serialize(requestBody), Encoding.UTF8, "application/json"); - - try + while (attempt < maxRetries) { - using HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, _mispApiUrl) { Content = requestContent }; - request.Headers.Add("Authorization", _config.MispApiKey); - request.Headers.Add("Accept", "application/json"); - - _dnsServer.WriteLog($"Fetching page {page}..."); - using HttpResponseMessage response = await _httpClient.SendAsync(request); - - if (!response.IsSuccessStatusCode) + attempt++; + try { - string errorBody = await response.Content.ReadAsStringAsync(); - throw new HttpRequestException($"MISP API request failed on page {page} with status code {response.StatusCode}. Body: {errorBody}", null, response.StatusCode); - } + MispRequestBody requestBody = new MispRequestBody + { + Type = "domain", + To_ids = true, + Deleted = false, + Last = _config.MaxIocAge, + Limit = limit, + Page = page + }; + StringContent requestContent = new StringContent(JsonSerializer.Serialize(requestBody), Encoding.UTF8, "application/json"); + + using HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, _mispApiUrl) { Content = requestContent }; + request.Headers.Add("Authorization", _config.MispApiKey); + request.Headers.Add("Accept", "application/json"); + + _dnsServer.WriteLog($"Fetching page {page}, attempt {attempt}/{maxRetries}..."); + using HttpResponseMessage response = await _httpClient.SendAsync(request); + + if (!response.IsSuccessStatusCode) + { + // This is a definitive failure from the server (e.g., 403, 500). + // We should not retry this. Abort immediately. + string errorBody = await response.Content.ReadAsStringAsync(); + throw new HttpRequestException($"MISP API returned a non-success status code: {(int)response.StatusCode}. Body: {errorBody}", null, response.StatusCode); + } - await using Stream responseStream = await response.Content.ReadAsStreamAsync(); - MispResponse mispResponse = await JsonSerializer.DeserializeAsync(responseStream); + await using Stream responseStream = await response.Content.ReadAsStreamAsync(); + mispResponse = await JsonSerializer.DeserializeAsync(responseStream); - List attributes = mispResponse?.Response?.Attribute; - if (attributes?.Count == 0) - { - // No more attributes found, we're done. - hasMorePages = false; - continue; + break; } - - foreach (MispAttribute attribute in attributes) + catch (Exception ex) when (ex is HttpRequestException || ex is SocketException || ex is OperationCanceledException) { - string domain = attribute.Value?.Trim().ToLowerInvariant(); - if (!string.IsNullOrEmpty(domain) && DnsClient.IsDomainNameValid(domain)) + // These are likely transient network errors, so we should retry. + _dnsServer.WriteLog($"WARNING: A transient network error occurred on page {page}, attempt {attempt}/{maxRetries}. Error: {ex.Message}"); + if (attempt < maxRetries) + { + TimeSpan delay = TimeSpan.FromSeconds(Math.Pow(2, attempt)) + TimeSpan.FromMilliseconds(Random.Shared.Next(0, 1000)); + _dnsServer.WriteLog($"Waiting for {delay.TotalSeconds:F1} seconds before retrying..."); + await Task.Delay(delay); + } + else { - domains.Add(domain); + // All retries have failed for this page. + _dnsServer.WriteLog($"ERROR: Failed to fetch page {page} after {maxRetries} attempts. Aborting entire update cycle."); + throw; } } + } - // Assumption: If we received fewer items than our limit, it must be the last page. - if (attributes.Count < limit) - { - hasMorePages = false; - } - else + List attributes = mispResponse?.Response?.Attribute; + if (attributes == null || attributes.Count == 0) + { + hasMorePages = false; + continue; + } + + foreach (MispAttribute attribute in attributes) + { + string domain = attribute.Value?.Trim().ToLowerInvariant(); + if (!string.IsNullOrEmpty(domain) && DnsClient.IsDomainNameValid(domain)) { - page++; + domains.Add(domain); } } - catch (Exception ex) + + // Assumption: If we received fewer items than our limit, it must be the last page. + if (attributes.Count < limit) + { + hasMorePages = false; + } + else { - _dnsServer.WriteLog($"ERROR: Failed while fetching page {page}. Halting update cycle. Error: {ex.Message}"); - throw; + page++; } } From 74aab9a61d0d6b5a84fff4ca7bb8555edeb15d08 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Thu, 24 Jul 2025 14:47:51 +0300 Subject: [PATCH 18/22] Removed the _random variable to use Random.Shared everywhere in case there is a concurrency issue --- Apps/MispConnectorApp/App.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Apps/MispConnectorApp/App.cs b/Apps/MispConnectorApp/App.cs index 8e5c885b..12b5d801 100644 --- a/Apps/MispConnectorApp/App.cs +++ b/Apps/MispConnectorApp/App.cs @@ -45,7 +45,6 @@ public sealed class App : IDnsApplication, IDnsRequestBlockingHandler { #region variables - readonly Random _random = new Random(); string _cacheFilePath; Config _config; IDnsServer _dnsServer; @@ -107,7 +106,7 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) _dnsServer.WriteLog($"FATAL: The MispConnector update task failed unexpectedly. Error: {ex.Message}"); _dnsServer.WriteLog(ex); } - }, null, TimeSpan.FromSeconds(_random.Next(5, 30)), Timeout.InfiniteTimeSpan); + }, null, TimeSpan.FromSeconds(Random.Shared.Next(5, 30)), Timeout.InfiniteTimeSpan); } catch (Exception ex) { @@ -448,7 +447,7 @@ private async Task UpdateIocsAsync() } finally { - TimeSpan nextInterval = _updateInterval + TimeSpan.FromSeconds(_random.Next(0, 60)); + TimeSpan nextInterval = _updateInterval + TimeSpan.FromSeconds(Random.Shared.Next(0, 60)); _updateTimer?.Change(nextInterval, Timeout.InfiniteTimeSpan); } } From 1b7e064af5c17348221cb9dfa4085474997bfc59 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Thu, 24 Jul 2025 15:05:18 +0300 Subject: [PATCH 19/22] Fixed async code to prevent collision and reentrance if the task takes longer than interval --- Apps/MispConnectorApp/App.cs | 88 +++++++++++++++++++++--------------- 1 file changed, 52 insertions(+), 36 deletions(-) diff --git a/Apps/MispConnectorApp/App.cs b/Apps/MispConnectorApp/App.cs index 12b5d801..de2a3a51 100644 --- a/Apps/MispConnectorApp/App.cs +++ b/Apps/MispConnectorApp/App.cs @@ -56,7 +56,7 @@ public sealed class App : IDnsApplication, IDnsRequestBlockingHandler DnsSOARecordData _soaRecord; TimeSpan _updateInterval; - Timer _updateTimer; + CancellationTokenSource _appShutdownCts; #endregion variables @@ -64,7 +64,8 @@ public sealed class App : IDnsApplication, IDnsRequestBlockingHandler public void Dispose() { - _updateTimer?.Dispose(); + _appShutdownCts?.Cancel(); + _appShutdownCts?.Dispose(); _httpClient?.Dispose(); } @@ -95,18 +96,11 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) _httpClient = CreateHttpClient(mispServerUrl, _config.DisableTlsValidation); await LoadBlocklistFromCacheAsync(); - _updateTimer = new Timer(async _ => - { - try - { - await UpdateIocsAsync(); - } - catch (Exception ex) - { - _dnsServer.WriteLog($"FATAL: The MispConnector update task failed unexpectedly. Error: {ex.Message}"); - _dnsServer.WriteLog(ex); - } - }, null, TimeSpan.FromSeconds(Random.Shared.Next(5, 30)), Timeout.InfiniteTimeSpan); + _appShutdownCts = new CancellationTokenSource(); + + // 2. Start the new, long-running update loop task. + // We do not await this, as it's designed to run for the lifetime of the app. + _ = StartUpdateLoopAsync(_appShutdownCts.Token); } catch (Exception ex) { @@ -187,7 +181,31 @@ public Task ProcessRequestAsync(DnsDatagram request, IPEndPoint rem #endregion public #region private + private async Task StartUpdateLoopAsync(CancellationToken cancellationToken) + { + await Task.Delay(TimeSpan.FromSeconds(Random.Shared.Next(5, 30)), cancellationToken); + using var timer = new PeriodicTimer(_updateInterval); + while (!cancellationToken.IsCancellationRequested) + { + try + { + await UpdateIocsAsync(cancellationToken); + } + catch (OperationCanceledException) + { + _dnsServer.WriteLog("Update loop is shutting down gracefully."); + break; + } + catch (Exception ex) + { + _dnsServer.WriteLog($"FATAL: The MispConnector update task failed unexpectedly. Error: {ex.Message}"); + _dnsServer.WriteLog(ex); + } + + await timer.WaitForNextTickAsync(cancellationToken); + } + } private static TimeSpan ParseUpdateInterval(string interval) { if (string.IsNullOrWhiteSpace(interval) || interval.Length < 2) @@ -219,7 +237,7 @@ private static TimeSpan ParseUpdateInterval(string interval) } } - private async Task CheckTcpPortAsync(Uri serverUri) + private async Task CheckTcpPortAsync(Uri serverUri, CancellationToken cancellationToken) { string host = serverUri.DnsSafeHost; int port = serverUri.Port; @@ -229,7 +247,7 @@ private async Task CheckTcpPortAsync(Uri serverUri) try { - using CancellationTokenSource cts = new CancellationTokenSource(timeout); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, new CancellationTokenSource(timeout).Token); using TcpClient client = new TcpClient(); await client.ConnectAsync(host, port, cts.Token); @@ -276,7 +294,7 @@ private HttpClient CreateHttpClient(Uri serverUrl, bool disableTlsValidation) return new HttpClient(new HttpClientNetworkHandler(handler, _dnsServer.PreferIPv6 ? HttpClientNetworkType.PreferIPv6 : HttpClientNetworkType.Default, _dnsServer)); } - private async Task> FetchDomainsFromMispAsync() + private async Task> FetchDomainsFromMispAsync(CancellationToken cancellationToken) { HashSet domains = new HashSet(StringComparer.OrdinalIgnoreCase); int page = 1; @@ -335,7 +353,7 @@ private async Task> FetchDomainsFromMispAsync() { TimeSpan delay = TimeSpan.FromSeconds(Math.Pow(2, attempt)) + TimeSpan.FromMilliseconds(Random.Shared.Next(0, 1000)); _dnsServer.WriteLog($"Waiting for {delay.TotalSeconds:F1} seconds before retrying..."); - await Task.Delay(delay); + await Task.Delay(delay, cancellationToken); } else { @@ -426,36 +444,34 @@ private void ReloadBlocklist(FrozenSet newBlocklist) Interlocked.Exchange(ref _globalBlocklist, newBlocklist); } - private async Task UpdateIocsAsync() + private async Task UpdateIocsAsync(CancellationToken cancellationToken) { - try + if (!await CheckTcpPortAsync(new Uri(_config.MispServerUrl), cancellationToken)) { - if (!await CheckTcpPortAsync(new Uri(_config.MispServerUrl))) - { - return; - } + return; + } + + _dnsServer.WriteLog("MISP Connector: Starting IOC update..."); + FrozenSet domains = await FetchDomainsFromMispAsync(cancellationToken); + await WriteDomainsToCacheAsync(domains, cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); - _dnsServer.WriteLog("MISP Connector: Starting IOC update..."); - FrozenSet domains = await FetchDomainsFromMispAsync(); - await WriteDomainsToCacheAsync(domains); + if (!domains.SetEquals(_globalBlocklist)) + { + await WriteDomainsToCacheAsync(domains, cancellationToken); ReloadBlocklist(domains); _dnsServer.WriteLog($"MISP Connector: Successfully updated blocklist with {domains.Count} domains."); } - catch (Exception ex) - { - _dnsServer.WriteLog($"ERROR: MISP Connector failed to update IOCs. Error: {ex.Message}"); - } - finally + else { - TimeSpan nextInterval = _updateInterval + TimeSpan.FromSeconds(Random.Shared.Next(0, 60)); - _updateTimer?.Change(nextInterval, Timeout.InfiniteTimeSpan); + _dnsServer.WriteLog("MISP data has not changed. No update to blocklist or cache is necessary."); } } - private async Task WriteDomainsToCacheAsync(FrozenSet domains) + private async Task WriteDomainsToCacheAsync(FrozenSet domains, CancellationToken cancellationToken) { string tempPath = _cacheFilePath + ".tmp"; - await File.WriteAllLinesAsync(tempPath, domains); + await File.WriteAllLinesAsync(tempPath, domains, cancellationToken); File.Move(tempPath, _cacheFilePath, true); } From 0076019c035612fc216efddd588237bcf097e675 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Thu, 24 Jul 2025 15:12:26 +0300 Subject: [PATCH 20/22] Removed forgotten call to WriteDomainsToCacheAsync, causing write twice --- Apps/MispConnectorApp/App.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Apps/MispConnectorApp/App.cs b/Apps/MispConnectorApp/App.cs index de2a3a51..d7be8704 100644 --- a/Apps/MispConnectorApp/App.cs +++ b/Apps/MispConnectorApp/App.cs @@ -453,7 +453,6 @@ private async Task UpdateIocsAsync(CancellationToken cancellationToken) _dnsServer.WriteLog("MISP Connector: Starting IOC update..."); FrozenSet domains = await FetchDomainsFromMispAsync(cancellationToken); - await WriteDomainsToCacheAsync(domains, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); if (!domains.SetEquals(_globalBlocklist)) From 20ae2684d346f6d71ab516b0a77dd742ba1a7fa7 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Mon, 4 Aug 2025 12:43:16 +0300 Subject: [PATCH 21/22] Updated copyright year, added README and an error handler for update task Signed-off-by: Zafer Balkan --- Apps/MispConnectorApp/App.cs | 175 ++++++++++++++++------------ Apps/MispConnectorApp/README.md | 43 +++++++ Apps/MispConnectorApp/dnsApp.config | 2 +- 3 files changed, 144 insertions(+), 76 deletions(-) create mode 100644 Apps/MispConnectorApp/README.md diff --git a/Apps/MispConnectorApp/App.cs b/Apps/MispConnectorApp/App.cs index d7be8704..70e5fe16 100644 --- a/Apps/MispConnectorApp/App.cs +++ b/Apps/MispConnectorApp/App.cs @@ -1,6 +1,6 @@ /* Technitium DNS Server -Copyright (C) 2024 Shreyas Zare (shreyas@technitium.com) +Copyright (C) 2025 Shreyas Zare (shreyas@technitium.com) Copyright (C) 2025 Zafer Balkan (zafer@zaferbalkan.com) This program is free software: you can redistribute it and/or modify @@ -45,19 +45,19 @@ public sealed class App : IDnsApplication, IDnsRequestBlockingHandler { #region variables - string _cacheFilePath; + string _domainCacheFilePath; Config _config; IDnsServer _dnsServer; - private FrozenSet _globalBlocklist = FrozenSet.Empty; + FrozenSet _domainBlocklist = FrozenSet.Empty; HttpClient _httpClient; Uri _mispApiUrl; DnsSOARecordData _soaRecord; TimeSpan _updateInterval; + Task _updateLoopTask; CancellationTokenSource _appShutdownCts; - #endregion variables #region IDisposable @@ -65,8 +65,21 @@ public sealed class App : IDnsApplication, IDnsRequestBlockingHandler public void Dispose() { _appShutdownCts?.Cancel(); - _appShutdownCts?.Dispose(); - _httpClient?.Dispose(); + try + { + if (_updateLoopTask != null) + { + _ = Task.WhenAny(_updateLoopTask, Task.Delay(TimeSpan.FromSeconds(2))).GetAwaiter().GetResult(); + } + } + catch + { + } + finally + { + _appShutdownCts?.Dispose(); + _httpClient?.Dispose(); + } } #endregion IDisposable @@ -78,10 +91,6 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) _dnsServer = dnsServer; try { - string configDir = _dnsServer.ApplicationFolder; - Directory.CreateDirectory(configDir); - _cacheFilePath = Path.Combine(configDir, "misp_domain_cache.txt"); - _soaRecord = new DnsSOARecordData(_dnsServer.ServerDomain, _dnsServer.ResponsiblePerson.Address, 1, 14400, 3600, 604800, 60); JsonSerializerOptions options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true }; @@ -89,6 +98,10 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) Validator.ValidateObject(_config, new ValidationContext(_config), validateAllProperties: true); + string configDir = _dnsServer.ApplicationFolder; + Directory.CreateDirectory(configDir); + _domainCacheFilePath = Path.Combine(configDir, "misp_domain_cache.txt"); + _updateInterval = ParseUpdateInterval(_config.UpdateInterval); Uri mispServerUrl = new Uri(_config.MispServerUrl); @@ -98,9 +111,16 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) await LoadBlocklistFromCacheAsync(); _appShutdownCts = new CancellationTokenSource(); - // 2. Start the new, long-running update loop task. // We do not await this, as it's designed to run for the lifetime of the app. - _ = StartUpdateLoopAsync(_appShutdownCts.Token); + _updateLoopTask = StartUpdateLoopAsync(_appShutdownCts.Token); + Task _ = _updateLoopTask.ContinueWith(t => + { + if (t.IsFaulted) + { + _dnsServer.WriteLog($"FATAL: Update loop terminated unexpectedly: {t.Exception?.GetBaseException().Message}"); + _dnsServer.WriteLog(t.Exception); + } + }, TaskContinuationOptions.OnlyOnFaulted); } catch (Exception ex) { @@ -117,24 +137,28 @@ public Task IsAllowedAsync(DnsDatagram request, IPEndPoint remoteEP) public Task ProcessRequestAsync(DnsDatagram request, IPEndPoint remoteEP) { if (_config?.EnableBlocking != true) + { return Task.FromResult(null); + } DnsQuestionRecord question = request.Question[0]; - if (!IsDomainBlocked(question.Name, out string blockedDomain)) + bool domainBlocked = IsDomainBlocked(question.Name, out string blockedDomain); + if (!domainBlocked) { return Task.FromResult(null); } string blockingReport = $"source=misp-connector;domain={blockedDomain}"; + EDnsOption[] options = null; if (_config.AddExtendedDnsError && request.EDNS is not null) { - options = new EDnsOption[] { new EDnsOption(EDnsOptionCode.EXTENDED_DNS_ERROR, new EDnsExtendedDnsErrorOptionData(EDnsExtendedDnsErrorCode.Blocked, blockingReport)) }; + options = new EDnsOption[] { new EDnsOption(EDnsOptionCode.EXTENDED_DNS_ERROR, new EDnsExtendedDnsErrorOptionData(EDnsExtendedDnsErrorCode.Blocked, string.Empty)) }; } if (_config.AllowTxtBlockingReport && question.Type == DnsResourceRecordType.TXT) { - DnsResourceRecord[] answer = new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, 60, new DnsTXTRecordData(blockingReport)) }; + DnsResourceRecord[] answer = new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, 60, new DnsTXTRecordData(string.Empty)) }; return Task.FromResult(new DnsDatagram( ID: request.Identifier, isResponse: true, @@ -156,7 +180,7 @@ public Task ProcessRequestAsync(DnsDatagram request, IPEndPoint rem )); } - DnsResourceRecord[] authority = { new DnsResourceRecord(blockedDomain, DnsResourceRecordType.SOA, question.Class, 60, _soaRecord) }; + DnsResourceRecord[] authority = { new DnsResourceRecord(question.Name, DnsResourceRecordType.SOA, question.Class, 60, _soaRecord) }; return Task.FromResult(new DnsDatagram( ID: request.Identifier, isResponse: true, @@ -184,26 +208,27 @@ public Task ProcessRequestAsync(DnsDatagram request, IPEndPoint rem private async Task StartUpdateLoopAsync(CancellationToken cancellationToken) { await Task.Delay(TimeSpan.FromSeconds(Random.Shared.Next(5, 30)), cancellationToken); - using var timer = new PeriodicTimer(_updateInterval); - - while (!cancellationToken.IsCancellationRequested) + using (PeriodicTimer timer = new PeriodicTimer(_updateInterval)) { - try - { - await UpdateIocsAsync(cancellationToken); - } - catch (OperationCanceledException) + while (!cancellationToken.IsCancellationRequested) { - _dnsServer.WriteLog("Update loop is shutting down gracefully."); - break; - } - catch (Exception ex) - { - _dnsServer.WriteLog($"FATAL: The MispConnector update task failed unexpectedly. Error: {ex.Message}"); - _dnsServer.WriteLog(ex); - } + try + { + await UpdateIocsAsync(cancellationToken); + } + catch (OperationCanceledException) + { + _dnsServer.WriteLog("Update loop is shutting down gracefully."); + break; + } + catch (Exception ex) + { + _dnsServer.WriteLog($"FATAL: The MispConnector update task failed unexpectedly. Error: {ex.Message}"); + _dnsServer.WriteLog(ex); + } - await timer.WaitForNextTickAsync(cancellationToken); + await timer.WaitForNextTickAsync(cancellationToken); + } } } private static TimeSpan ParseUpdateInterval(string interval) @@ -247,10 +272,11 @@ private async Task CheckTcpPortAsync(Uri serverUri, CancellationToken canc try { - using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, new CancellationTokenSource(timeout).Token); - using TcpClient client = new TcpClient(); - - await client.ConnectAsync(host, port, cts.Token); + using (CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, new CancellationTokenSource(timeout).Token)) + using (TcpClient client = new TcpClient()) + { + await client.ConnectAsync(host, port, cts.Token); + } _dnsServer.WriteLog($"Pre-flight TCP check successful for {host}:{port}."); return true; @@ -284,19 +310,16 @@ private HttpClient CreateHttpClient(Uri serverUrl, bool disableTlsValidation) if (disableTlsValidation) { - handler.SslOptions.RemoteCertificateValidationCallback = (sender, certificate, chain, sslPolicyErrors) => - { - return true; - }; + handler.SslOptions.RemoteCertificateValidationCallback = (sender, certificate, chain, sslPolicyErrors) => true; _dnsServer.WriteLog($"WARNING: TLS certificate validation is DISABLED for MISP server: {serverUrl}"); } return new HttpClient(new HttpClientNetworkHandler(handler, _dnsServer.PreferIPv6 ? HttpClientNetworkType.PreferIPv6 : HttpClientNetworkType.Default, _dnsServer)); } - private async Task> FetchDomainsFromMispAsync(CancellationToken cancellationToken) + private async Task> FetchIocFromMispAsync(CancellationToken cancellationToken) { - HashSet domains = new HashSet(StringComparer.OrdinalIgnoreCase); + HashSet iocSet = new HashSet(StringComparer.OrdinalIgnoreCase); int page = 1; int limit = _config.PaginationLimit; bool hasMorePages = true; @@ -330,18 +353,18 @@ private async Task> FetchDomainsFromMispAsync(CancellationToke request.Headers.Add("Accept", "application/json"); _dnsServer.WriteLog($"Fetching page {page}, attempt {attempt}/{maxRetries}..."); - using HttpResponseMessage response = await _httpClient.SendAsync(request); + using HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); if (!response.IsSuccessStatusCode) { // This is a definitive failure from the server (e.g., 403, 500). // We should not retry this. Abort immediately. - string errorBody = await response.Content.ReadAsStringAsync(); + string errorBody = await response.Content.ReadAsStringAsync(cancellationToken); throw new HttpRequestException($"MISP API returned a non-success status code: {(int)response.StatusCode}. Body: {errorBody}", null, response.StatusCode); } - await using Stream responseStream = await response.Content.ReadAsStreamAsync(); - mispResponse = await JsonSerializer.DeserializeAsync(responseStream); + await using (Stream responseStream = await response.Content.ReadAsStreamAsync(cancellationToken)) + mispResponse = await JsonSerializer.DeserializeAsync(responseStream, cancellationToken: cancellationToken); break; } @@ -373,10 +396,13 @@ private async Task> FetchDomainsFromMispAsync(CancellationToke foreach (MispAttribute attribute in attributes) { - string domain = attribute.Value?.Trim().ToLowerInvariant(); - if (!string.IsNullOrEmpty(domain) && DnsClient.IsDomainNameValid(domain)) + string ioc = attribute.Value?.Trim().ToLowerInvariant(); + if (!string.IsNullOrEmpty(ioc)) { - domains.Add(domain); + if (DnsClient.IsDomainNameValid(ioc)) + { + iocSet.Add(ioc); + } } } @@ -391,13 +417,13 @@ private async Task> FetchDomainsFromMispAsync(CancellationToke } } - _dnsServer.WriteLog($"Finished paginated fetch. Freezing {domains.Count} domains for optimal read performance..."); - return domains.ToFrozenSet(StringComparer.OrdinalIgnoreCase); + _dnsServer.WriteLog($"Finished paginated fetch. Freezing {iocSet.Count} IOCs for optimal read performance..."); + return iocSet; } private bool IsDomainBlocked(string domain, out string foundZone) { - FrozenSet currentBlocklist = _globalBlocklist; + FrozenSet currentBlocklist = _domainBlocklist; ReadOnlySpan currentSpan = domain.AsSpan(); @@ -426,24 +452,21 @@ private bool IsDomainBlocked(string domain, out string foundZone) private async Task LoadBlocklistFromCacheAsync() { - if (!File.Exists(_cacheFilePath)) return; - try + if (File.Exists(_domainCacheFilePath)) { - FrozenSet domains = (await File.ReadAllLinesAsync(_cacheFilePath)).ToHashSet(StringComparer.OrdinalIgnoreCase).ToFrozenSet(StringComparer.OrdinalIgnoreCase); - ReloadBlocklist(domains); - _dnsServer.WriteLog($"MISP Connector: Loaded {domains.Count} domains from cache."); - } - catch (IOException ex) - { - _dnsServer.WriteLog($"ERROR: Failed to read cache file '{_cacheFilePath}'. Error: {ex.Message}"); + try + { + FrozenSet domains = (await File.ReadAllLinesAsync(_domainCacheFilePath)).ToHashSet(StringComparer.OrdinalIgnoreCase).ToFrozenSet(StringComparer.OrdinalIgnoreCase); + Interlocked.Exchange(ref _domainBlocklist, domains); + _dnsServer.WriteLog($"MISP Connector: Loaded {domains.Count} domains from cache."); + } + catch (IOException ex) + { + _dnsServer.WriteLog($"ERROR: Failed to read cache file '{_domainCacheFilePath}'. Error: {ex.Message}"); + } } } - private void ReloadBlocklist(FrozenSet newBlocklist) - { - Interlocked.Exchange(ref _globalBlocklist, newBlocklist); - } - private async Task UpdateIocsAsync(CancellationToken cancellationToken) { if (!await CheckTcpPortAsync(new Uri(_config.MispServerUrl), cancellationToken)) @@ -452,13 +475,15 @@ private async Task UpdateIocsAsync(CancellationToken cancellationToken) } _dnsServer.WriteLog("MISP Connector: Starting IOC update..."); - FrozenSet domains = await FetchDomainsFromMispAsync(cancellationToken); + + HashSet tmpDomains = await FetchIocFromMispAsync(cancellationToken); cancellationToken.ThrowIfCancellationRequested(); + FrozenSet domains = tmpDomains.ToFrozenSet(StringComparer.OrdinalIgnoreCase); - if (!domains.SetEquals(_globalBlocklist)) + if (!domains.SetEquals(_domainBlocklist)) { - await WriteDomainsToCacheAsync(domains, cancellationToken); - ReloadBlocklist(domains); + await WriteIocsToCacheAsync(domains, cancellationToken); + Interlocked.Exchange(ref _domainBlocklist, domains); _dnsServer.WriteLog($"MISP Connector: Successfully updated blocklist with {domains.Count} domains."); } else @@ -467,11 +492,11 @@ private async Task UpdateIocsAsync(CancellationToken cancellationToken) } } - private async Task WriteDomainsToCacheAsync(FrozenSet domains, CancellationToken cancellationToken) + private async Task WriteIocsToCacheAsync(FrozenSet iocs, CancellationToken cancellationToken) { - string tempPath = _cacheFilePath + ".tmp"; - await File.WriteAllLinesAsync(tempPath, domains, cancellationToken); - File.Move(tempPath, _cacheFilePath, true); + string tempPath = _domainCacheFilePath + ".tmp"; + await File.WriteAllLinesAsync(tempPath, iocs, cancellationToken); + File.Move(tempPath, _domainCacheFilePath, true); } #endregion private diff --git a/Apps/MispConnectorApp/README.md b/Apps/MispConnectorApp/README.md new file mode 100644 index 00000000..5b8a4a6c --- /dev/null +++ b/Apps/MispConnectorApp/README.md @@ -0,0 +1,43 @@ +# MISP Connector for Technitium DNS Server + +A plugin that pulls malicious domain names from MISP feeds and enforces blocking in Technitium DNS. + +It maintains in-memory blocklists with disk-backed caching and periodically refreshes from the source. + +## Features + +- Retrieves indicators of compromise (IOCs) aka. malicious domain names from a MISP server via its REST API. +- Handles paginated fetches with exponential backoff and retry on transient failures. +- Stores the latest blocklist in memory for fast lookup and persists it to disk for faster startup. +- Blocks matching DNS requests by returning NXDOMAIN or, for TXT queries when enabled, a human-readable blocking report. +- Optionally includes extended DNS error metadata. +- Configurable refresh interval and age window for which indicators are considered. +- Optional disabling of TLS certificate validation with explicit warning in logs. + +## Configuration + +Supply a JSON configuration like the following: + +```json +{ + "enableBlocking": true, + "mispServerUrl": "https://misp.example.com", + "mispApiKey": "YourMispApiKeyHere", + "disableTlsValidation": false, + "updateInterval": "2h", + "maxIocAge": "15d", + "allowTxtBlockingReport": true, + "paginationLimit": 5000, + "addExtendedDnsError": true +} +``` + +- You can disable the app without uninstalling. +- You can disable TLS validation for test instances and homelabs, but **it is not recommended use this option in production**. +- The `maxIocAge` option is used for filtering IOCs wih `lastSeen` attributes on MISP. So, you can dynamically filter for recent campaigns. +- The `allowTxtBlockingReport` rewrites the response with a blocking report. +- The `addExtendedDnsError` is useful when logs are exported to a SIEM. The blocking report gets added to EDNS payload of the package. + +# Acknowledgement + +Thanks to everyone who has been part of or contributed to [MISP Project](https://www.misp-project.org/) for being an amazing resource. \ No newline at end of file diff --git a/Apps/MispConnectorApp/dnsApp.config b/Apps/MispConnectorApp/dnsApp.config index 28bdc1fd..e5e76a05 100644 --- a/Apps/MispConnectorApp/dnsApp.config +++ b/Apps/MispConnectorApp/dnsApp.config @@ -4,7 +4,7 @@ "mispApiKey": "YourMispApiKeyHere", "disableTlsValidation": false, "updateInterval": "2h", - "maxIocAge": "90d", + "maxIocAge": "30d", "allowTxtBlockingReport": true, "paginationLimit": 5000, "addExtendedDnsError": true From 877a97ccb2d50836be2bc07e68474bb2b82c8736 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Mon, 4 Aug 2025 12:52:32 +0300 Subject: [PATCH 22/22] Removed editorconfig reference Signed-off-by: Zafer Balkan --- DnsServer.sln | 5 ----- 1 file changed, 5 deletions(-) diff --git a/DnsServer.sln b/DnsServer.sln index 05fc6280..676a1040 100644 --- a/DnsServer.sln +++ b/DnsServer.sln @@ -69,11 +69,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QueryLogsMySqlApp", "Apps\Q EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MispConnectorApp", "Apps\MispConnectorApp\MispConnectorApp.csproj", "{83C8180A-0F86-F9A0-8F41-6FD61FAC41CB}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{02EA681E-C7D8-13C7-8484-4AC65E1B71E8}" - ProjectSection(SolutionItems) = preProject - .editorconfig = .editorconfig - EndProjectSection -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU