From 246a39b9c24f18da07338b12e50ebfc6bc9da1f5 Mon Sep 17 00:00:00 2001 From: Gabe Stocco <98900+gfs@users.noreply.github.com> Date: Mon, 1 Apr 2024 14:48:11 -0700 Subject: [PATCH 01/10] WIP Conversion to ProtoBuf Object Storage --- .gitignore | 1 + Lib/Lib.csproj | 1 + Lib/Objects/CollectObject.cs | 22 ++++++++++-- Lib/Objects/FileSystemObject.cs | 41 +++++++++++++++------- Lib/Objects/MacSignature.cs | 10 +++++- Lib/Objects/SerializableCertificate.cs | 11 ++++++ Lib/Objects/Signature.cs | 6 ++++ Lib/Objects/WriteObject.cs | 4 +-- Lib/Utils/JsonUtils.cs | 48 ++++++++++++++++---------- Lib/Utils/SqliteDatabaseManager.cs | 20 +++++------ 10 files changed, 118 insertions(+), 46 deletions(-) diff --git a/.gitignore b/.gitignore index 367a20d44..ed2293b2e 100644 --- a/.gitignore +++ b/.gitignore @@ -271,3 +271,4 @@ asa.sqlite *asa.log.txt *.sqlite *.litedb +Cli/Properties/launchSettings.json diff --git a/Lib/Lib.csproj b/Lib/Lib.csproj index 757f1b9c9..92242d269 100644 --- a/Lib/Lib.csproj +++ b/Lib/Lib.csproj @@ -44,6 +44,7 @@ + diff --git a/Lib/Objects/CollectObject.cs b/Lib/Objects/CollectObject.cs index 33b92f228..3d477811f 100644 --- a/Lib/Objects/CollectObject.cs +++ b/Lib/Objects/CollectObject.cs @@ -2,6 +2,7 @@ using Microsoft.CST.AttackSurfaceAnalyzer.Types; using Microsoft.CST.AttackSurfaceAnalyzer.Utils; using Newtonsoft.Json; +using ProtoBuf; using System.Globalization; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects @@ -9,6 +10,23 @@ namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects /// /// Abstract parent class that all Collected data inherits from. /// + [ProtoContract] + [ProtoInclude(1, typeof(CertificateObject))] + [ProtoInclude(2, typeof(ComObject))] + [ProtoInclude(3, typeof(CryptographicKeyObject))] + [ProtoInclude(4, typeof(DriverObject))] + [ProtoInclude(5, typeof(EventLogObject))] + [ProtoInclude(6, typeof(FileMonitorObject))] + [ProtoInclude(7, typeof(FileSystemObject))] + [ProtoInclude(8, typeof(FirewallObject))] + [ProtoInclude(9, typeof(OpenPortObject))] + [ProtoInclude(10, typeof(ProcessObject))] + [ProtoInclude(11, typeof(RegistryObject))] + [ProtoInclude(12, typeof(ServiceObject))] + [ProtoInclude(13, typeof(TpmObject))] + [ProtoInclude(14, typeof(UserAccountObject))] + [ProtoInclude(15, typeof(GroupAccountObject))] + [ProtoInclude(16, typeof(WifiObject))] public abstract class CollectObject { public abstract string Identity { get; } @@ -26,7 +44,7 @@ public string RowKey [SkipCompare] [JsonIgnore] - public string Serialized + public byte[] Serialized { get { @@ -39,6 +57,6 @@ public string Serialized } } - private string? _serialized = null; + private byte[]? _serialized = null; } } \ No newline at end of file diff --git a/Lib/Objects/FileSystemObject.cs b/Lib/Objects/FileSystemObject.cs index 8434d093b..e7c424416 100644 --- a/Lib/Objects/FileSystemObject.cs +++ b/Lib/Objects/FileSystemObject.cs @@ -1,10 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. using Microsoft.CST.AttackSurfaceAnalyzer.Types; +using ProtoBuf; using System; using System.Collections.Generic; +using System.IO; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract] public class FileSystemObject : CollectObject { public FileSystemObject(string Path) @@ -16,120 +19,132 @@ public FileSystemObject() { Path = string.Empty; } - public override RESULT_TYPE ResultType => RESULT_TYPE.FILE; - + /// + /// The File's path + /// + public override string Identity => Path; /// /// If this is windows executable what DLL Characteristics are set /// + [ProtoMember(23)] public List? Characteristics { get; set; } /// /// A hash of the file (if collected) /// + [ProtoMember(4)] public string? ContentHash { get; set; } /// /// When was the file created in UTC /// + [ProtoMember(24)] public DateTime Created { get; set; } /// /// .ToString of Mono FileTypes result. Not available on Windows. /// + [ProtoMember(5)] public string? FileType { get; set; } /// /// The group of the file. /// + [ProtoMember(6)] public string? Group { get; set; } - /// - /// The File's path - /// - public override string Identity - { - get - { - return Path; - } - } - /// /// If the file is a directory /// + [ProtoMember(7)] public bool? IsDirectory { get; set; } /// /// If the file is an executable /// + [ProtoMember(8)] public bool? IsExecutable { get; set; } /// /// The type of the executable if it is one /// + [ProtoMember(9)] public EXECUTABLE_TYPE ExecutableType { get; set; } = EXECUTABLE_TYPE.UNKNOWN; /// /// If the file is a link /// + [ProtoMember(10)] public bool? IsLink { get; set; } /// /// When was the file last modified in UTC /// + [ProtoMember(11)] public DateTime LastModified { get; set; } /// /// Signature information for signed Mac binaries. /// + [ProtoMember(12)] public MacSignature? MacSignatureStatus { get; set; } /// /// The owner of the file. /// + [ProtoMember(13)] public string? Owner { get; set; } /// /// The location on disk of the file /// + [ProtoMember(14)] public string Path { get; set; } /// /// What are the permissions of this file. /// + [ProtoMember(15)] public Dictionary? Permissions { get; set; } /// /// A string representation of the permissions /// + [ProtoMember(16)] public string? PermissionsString { get; set; } /// /// If the SetGid bit is set /// + [ProtoMember(17)] public bool? SetGid { get; set; } /// /// If the SetUid bit is set /// + [ProtoMember(18)] public bool? SetUid { get; set; } /// /// Signature information for signed Windows binaries. /// + [ProtoMember(19)] public Signature? SignatureStatus { get; set; } /// /// File size in bytes /// + [ProtoMember(20)] public long? Size { get; set; } + [ProtoMember(21)] public long? SizeOnDisk { get; internal set; } /// /// If this is a link where does it point to. /// + [ProtoMember(22)] public string? Target { get; set; } public bool ShouldSerializeCharacteristics() diff --git a/Lib/Objects/MacSignature.cs b/Lib/Objects/MacSignature.cs index 8278d6faa..6fea8c2bc 100644 --- a/Lib/Objects/MacSignature.cs +++ b/Lib/Objects/MacSignature.cs @@ -1,9 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. +using ProtoBuf; using System; using System.Collections.Generic; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract] public class MacSignature { //Executable=/Applications/1Password 7.app/Contents/MacOS/1Password 7 @@ -31,13 +33,19 @@ public class MacSignature //Runtime Version=10.15.0 //Sealed Resources version=2 rules=13 files=2909 //Internal requirements count=1 size=220 - + [ProtoMember(1)] public List? Authorities { get; set; } + [ProtoMember(2)] public string? CandidateCDHashFull { get; set; } + [ProtoMember(3)] public string? CMSDigest { get; set; } + [ProtoMember(4)] public string? HashChoices { get; set; } + [ProtoMember(5)] public string? HashType { get; set; } + [ProtoMember(6)] public string? TeamIdentifier { get; set; } + [ProtoMember(7)] public DateTime Timestamp { get; set; } } } \ No newline at end of file diff --git a/Lib/Objects/SerializableCertificate.cs b/Lib/Objects/SerializableCertificate.cs index 455f46400..b3552064d 100644 --- a/Lib/Objects/SerializableCertificate.cs +++ b/Lib/Objects/SerializableCertificate.cs @@ -1,10 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. using Newtonsoft.Json; +using ProtoBuf; using System; using System.Security.Cryptography.X509Certificates; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract] public class SerializableCertificate { public SerializableCertificate(X509Certificate2 certificate) @@ -34,14 +36,23 @@ public SerializableCertificate(string Thumbprint, string Subject, string PublicK this.Pkcs7 = Pkcs7; } + [ProtoMember(1)] public string CertHashString { get; set; } + [ProtoMember(2)] public string Issuer { get; set; } + [ProtoMember(3)] public DateTime NotAfter { get; set; } + [ProtoMember(4)] public DateTime NotBefore { get; set; } + [ProtoMember(5)] public string Pkcs7 { get; set; } + [ProtoMember(6)] public string PublicKey { get; set; } + [ProtoMember(7)] public string SerialNumber { get; set; } + [ProtoMember(8)] public string Subject { get; set; } + [ProtoMember(9)] public string Thumbprint { get; set; } } } \ No newline at end of file diff --git a/Lib/Objects/Signature.cs b/Lib/Objects/Signature.cs index 0360db002..06358cff6 100644 --- a/Lib/Objects/Signature.cs +++ b/Lib/Objects/Signature.cs @@ -1,11 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. using Newtonsoft.Json; using PeNet.Header.Authenticode; +using ProtoBuf; using System; using System.Security.Cryptography.X509Certificates; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract] public class Signature { public Signature(AuthenticodeInfo authenticodeInfo) @@ -50,9 +52,13 @@ public bool IsTimeValid } } + [ProtoMember(1)] public bool IsAuthenticodeValid { get; set; } + [ProtoMember(2)] public string? SignedHash { get; set; } + [ProtoMember(3)] public string? SignerSerialNumber { get; set; } + [ProtoMember(4)] public SerializableCertificate? SigningCertificate { get; set; } } } \ No newline at end of file diff --git a/Lib/Objects/WriteObject.cs b/Lib/Objects/WriteObject.cs index f1735a218..fbcaeed67 100644 --- a/Lib/Objects/WriteObject.cs +++ b/Lib/Objects/WriteObject.cs @@ -28,9 +28,9 @@ public string Identity public string RowKey { get; } public string RunId { get; } - public string Serialized { get; } + public byte[] Serialized { get; } - public static WriteObject? FromString(string SerializedIn, RESULT_TYPE ResultTypeIn, string RunIdIn) + public static WriteObject? FromString(byte[] SerializedIn, RESULT_TYPE ResultTypeIn, string RunIdIn) { var deserialized = JsonUtils.Hydrate(SerializedIn, ResultTypeIn); diff --git a/Lib/Utils/JsonUtils.cs b/Lib/Utils/JsonUtils.cs index 9e66e6282..3d6e5915a 100644 --- a/Lib/Utils/JsonUtils.cs +++ b/Lib/Utils/JsonUtils.cs @@ -2,10 +2,14 @@ using Microsoft.CST.AttackSurfaceAnalyzer.Objects; using Microsoft.CST.AttackSurfaceAnalyzer.Types; using Newtonsoft.Json; +using ProtoBuf; using System; using System.ComponentModel; using System.Globalization; +using System.IO; using System.Linq; +using System.Text; +using System.Text.Unicode; using Tpm2Lib; namespace Microsoft.CST.AttackSurfaceAnalyzer.Utils @@ -22,9 +26,16 @@ static JsonUtils() /// /// The object to serialize /// The bytes of the serialized object - public static string Dehydrate(CollectObject colObj) + public static byte[] Dehydrate(CollectObject colObj) { - return JsonConvert.SerializeObject(colObj, jsonSettings); + using var ms = new MemoryStream(); + if (colObj is FileSystemObject fileObject) + { + Serializer.Serialize(ms, fileObject); + return ms.ToArray(); + } + return new byte[0]; +// return JsonConvert.SerializeObject(colObj, jsonSettings); } /// @@ -35,7 +46,7 @@ public static string Dehydrate(CollectObject colObj) /// An appropriately typed collect object based on the collect result passed in, or null if the /// RESULT_TYPE is unknown. /// - public static CollectObject? Hydrate(string serialized, RESULT_TYPE type) + public static CollectObject? Hydrate(byte[] serialized, RESULT_TYPE type) { if (serialized == null) { @@ -45,48 +56,49 @@ public static string Dehydrate(CollectObject colObj) switch (type) { case RESULT_TYPE.CERTIFICATE: - return JsonConvert.DeserializeObject(serialized, jsonSettings); + return Serializer.Deserialize(new MemoryStream(serialized)); case RESULT_TYPE.FILE: - return JsonConvert.DeserializeObject(serialized, jsonSettings); + return Serializer.Deserialize(new MemoryStream(serialized)); case RESULT_TYPE.PORT: - return JsonConvert.DeserializeObject(serialized, jsonSettings); + return Serializer.Deserialize(new MemoryStream(serialized)); case RESULT_TYPE.REGISTRY: - return JsonConvert.DeserializeObject(serialized, jsonSettings); + return Serializer.Deserialize(new MemoryStream(serialized)); case RESULT_TYPE.SERVICE: - return JsonConvert.DeserializeObject(serialized, jsonSettings); + return Serializer.Deserialize(new MemoryStream(serialized)); case RESULT_TYPE.USER: - return JsonConvert.DeserializeObject(serialized, jsonSettings); + return Serializer.Deserialize(new MemoryStream(serialized)); case RESULT_TYPE.GROUP: - return JsonConvert.DeserializeObject(serialized, jsonSettings); + return Serializer.Deserialize(new MemoryStream(serialized)); case RESULT_TYPE.FIREWALL: - return JsonConvert.DeserializeObject(serialized, jsonSettings); + return Serializer.Deserialize(new MemoryStream(serialized)); case RESULT_TYPE.COM: - return JsonConvert.DeserializeObject(serialized, jsonSettings); + return Serializer.Deserialize(new MemoryStream(serialized)); case RESULT_TYPE.LOG: - return JsonConvert.DeserializeObject(serialized, jsonSettings); + return Serializer.Deserialize(new MemoryStream(serialized)); case RESULT_TYPE.TPM: - return JsonConvert.DeserializeObject(serialized, jsonSettings); + return Serializer.Deserialize(new MemoryStream(serialized)); case RESULT_TYPE.KEY: - return JsonConvert.DeserializeObject(serialized, jsonSettings); + return Serializer.Deserialize(new MemoryStream(serialized)); case RESULT_TYPE.PROCESS: - return JsonConvert.DeserializeObject(serialized, jsonSettings); + return Serializer.Deserialize(new MemoryStream(serialized)); case RESULT_TYPE.DRIVER: - return JsonConvert.DeserializeObject(serialized, jsonSettings); + return Serializer.Deserialize(new MemoryStream(serialized)); + case RESULT_TYPE.FILEMONITOR: - return JsonConvert.DeserializeObject(serialized, jsonSettings); + return Serializer.Deserialize(new MemoryStream(serialized)); default: return null; } diff --git a/Lib/Utils/SqliteDatabaseManager.cs b/Lib/Utils/SqliteDatabaseManager.cs index a6ee5f426..037973a09 100644 --- a/Lib/Utils/SqliteDatabaseManager.cs +++ b/Lib/Utils/SqliteDatabaseManager.cs @@ -179,7 +179,7 @@ public override IEnumerable GetAllMissing(string? firstRunId, strin var resultTypeString = reader["result_type"].ToString(); if (runId != null && resultTypeString != null) { - var wo = WriteObject.FromString((string)reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); + var wo = WriteObject.FromString((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); if (wo is WriteObject WO) output.Enqueue(WO); } @@ -207,7 +207,7 @@ public IEnumerable GetAllMissing2(string firstRunId, string secondR var resultTypeString = reader["result_type"].ToString(); if (runId != null && resultTypeString != null) { - var wo = WriteObject.FromString((string)reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); + var wo = WriteObject.FromString((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); if (wo is WriteObject WO) output.Enqueue(WO); } @@ -233,7 +233,7 @@ public IEnumerable GetAllMissingExplicit(string firstRunId, string var resultTypeString = reader["result_type"].ToString(); if (runId != null && resultTypeString != null) { - var wo = WriteObject.FromString((string)reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); + var wo = WriteObject.FromString((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); if (wo is WriteObject WO) output.Enqueue(WO); } @@ -282,11 +282,11 @@ public override List GetComparisonResults(string? baseId, string { if (JsonConvert.DeserializeObject(meta_serialized) is CompareResult compareResult) { - if (reader["first_serialized"] is string first_serialized) + if (reader["first_serialized"] is byte[] first_serialized) { compareResult.Base = JsonUtils.Hydrate(first_serialized, exportType); } - if (reader["second_serialized"] is string second_serialized) + if (reader["second_serialized"] is byte[] second_serialized) { compareResult.Compare = JsonUtils.Hydrate(second_serialized, exportType); } @@ -322,11 +322,11 @@ public override List GetComparisonResults(string? baseId, string? { if (JsonConvert.DeserializeObject(meta_serialized) is CompareResult compareResult) { - if (reader["first_serialized"] is string first_serialized) + if (reader["first_serialized"] is byte[] first_serialized) { compareResult.Base = JsonUtils.Hydrate(first_serialized, resultType); } - if (reader["second_serialized"] is string second_serialized) + if (reader["second_serialized"] is byte[] second_serialized) { compareResult.Compare = JsonUtils.Hydrate(second_serialized, resultType); } @@ -415,7 +415,7 @@ public override IEnumerable GetMissingFromFirst(string firstRunId, var resultTypeString = reader["result_type"].ToString(); if (runId != null && resultTypeString != null) { - var wo = WriteObject.FromString((string)reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); + var wo = WriteObject.FromString((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); if (wo is WriteObject WO) output.Enqueue(WO); } @@ -444,7 +444,7 @@ public override IEnumerable GetMissingFromFirst(string firstRunId, if (aRunId != null && bRunId != null && aResultType != null && bResultType != null) { - if (reader["a_serialized"] is string a_serialized && reader["b_serialized"] is string b_serialized) + if (reader["a_serialized"] is byte[] a_serialized && reader["b_serialized"] is byte[] b_serialized) { var val1 = WriteObject.FromString(a_serialized, (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), aResultType), aRunId); var val2 = WriteObject.FromString(b_serialized, (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), bResultType), bRunId); @@ -585,7 +585,7 @@ public override IEnumerable GetResultsByRunid(string runid) var resultTypeString = reader["result_type"].ToString(); if (runId != null && resultTypeString != null) { - var wo = WriteObject.FromString((string)reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); + var wo = WriteObject.FromString((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); if (wo is WriteObject WO) { yield return WO; From f813f9e0de82cce3432dfecf184080cc13bec5a5 Mon Sep 17 00:00:00 2001 From: Gabe Stocco <98900+gfs@users.noreply.github.com> Date: Mon, 1 Apr 2024 16:52:08 -0700 Subject: [PATCH 02/10] Convert rest of objects Also change RowKey to be a hash and fix threading issues in cryptohelpers --- Benchmarks/Program.cs | 2 +- Benchmarks/SystemSqliteDatabaseManager.cs | 8 +-- Lib/Objects/AsaNvIndex.cs | 7 ++- Lib/Objects/CertificateObject.cs | 6 ++ Lib/Objects/CollectObject.cs | 2 +- Lib/Objects/ComObject.cs | 5 ++ Lib/Objects/CryptographicKeyObject.cs | 9 ++- Lib/Objects/DriverObject.cs | 29 ++++++++- Lib/Objects/EventLogObject.cs | 9 +++ Lib/Objects/FileMonitorObject.cs | 10 ++++ Lib/Objects/FileSystemObject.cs | 9 ++- Lib/Objects/FirewallObject.cs | 17 ++++++ Lib/Objects/MonitorObject.cs | 3 + Lib/Objects/OpenPortObject.cs | 9 ++- Lib/Objects/ProcessModuleObject.cs | 5 ++ Lib/Objects/ProcessObject.cs | 10 ++++ Lib/Objects/RegistryObject.cs | 8 +++ Lib/Objects/SerializableCertificate.cs | 2 +- Lib/Objects/SerializableFileVersionInfo.cs | 31 +++++++++- Lib/Objects/ServiceObject.cs | 28 +++++++++ Lib/Objects/TpmObject.cs | 12 ++++ Lib/Objects/UserAccountObject.cs | 26 +++++++++ Lib/Objects/UserGroupObject.cs | 13 +++++ Lib/Objects/WifiObject.cs | 6 ++ Lib/Utils/CryptoHelpers.cs | 19 +++++- Lib/Utils/JsonUtils.cs | 68 ++++++++++++++++------ Tests/HydrationTests.cs | 2 +- 27 files changed, 315 insertions(+), 40 deletions(-) diff --git a/Benchmarks/Program.cs b/Benchmarks/Program.cs index 5d15b5f1e..9f3493c8c 100644 --- a/Benchmarks/Program.cs +++ b/Benchmarks/Program.cs @@ -6,7 +6,7 @@ public class Program { public static void Main(string[] args) { - var summary = BenchmarkRunner.Run(); + var summary = BenchmarkRunner.Run(); } } } \ No newline at end of file diff --git a/Benchmarks/SystemSqliteDatabaseManager.cs b/Benchmarks/SystemSqliteDatabaseManager.cs index 57e79ac65..04cfd20fe 100644 --- a/Benchmarks/SystemSqliteDatabaseManager.cs +++ b/Benchmarks/SystemSqliteDatabaseManager.cs @@ -225,7 +225,7 @@ public static ConcurrentBag GetMissingFromFirst(string firstRunId, var resultTypeString = reader["result_type"].ToString(); if (runId != null && resultTypeString != null) { - var wo = WriteObject.FromString((string)reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); + var wo = WriteObject.FromString((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); if (wo is WriteObject WO) { output.Add(WO); @@ -256,8 +256,8 @@ public static ConcurrentBag GetMissingFromFirst(string firstRunId, if (aRunId != null && bRunId != null && aResultType != null && bResultType != null) { - var val1 = WriteObject.FromString((string)reader["a_serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), aResultType), aRunId); - var val2 = WriteObject.FromString((string)reader["b_serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), bResultType), bRunId); + var val1 = WriteObject.FromString((byte[])reader["a_serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), aResultType), aRunId); + var val2 = WriteObject.FromString((byte[])reader["b_serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), bResultType), bRunId); if (val1 is WriteObject V1 && val2 is WriteObject V2) { @@ -389,7 +389,7 @@ public static IEnumerable GetResultsByRunid(string runid) var resultTypeString = reader["result_type"].ToString(); if (runId != null && resultTypeString != null) { - var val = WriteObject.FromString((string)reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); + var val = WriteObject.FromString((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); if (val is WriteObject valid) yield return valid; } diff --git a/Lib/Objects/AsaNvIndex.cs b/Lib/Objects/AsaNvIndex.cs index b93263128..00ec392e5 100644 --- a/Lib/Objects/AsaNvIndex.cs +++ b/Lib/Objects/AsaNvIndex.cs @@ -1,10 +1,13 @@ -using System.Collections.Generic; +using ProtoBuf; +using System.Collections.Generic; using Tpm2Lib; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract] public class AsaNvIndex { + [ProtoMember(1)] public NvAttr Attributes { get; set; } // These are all derived properties of the NvAttr flags. Separating them like this allows analysis @@ -17,6 +20,7 @@ public class AsaNvIndex public bool Counter { get { return Attributes.HasFlag(NvAttr.Counter); } } public bool Extend { get { return Attributes.HasFlag(NvAttr.Extend); } } public bool GlobalLock { get { return Attributes.HasFlag(NvAttr.Globallock); } } + [ProtoMember(2)] public uint Index { get; set; } public bool NoDa { get { return Attributes.HasFlag(NvAttr.NoDa); } } public bool None { get { return Attributes.HasFlag(NvAttr.None); } } @@ -41,6 +45,7 @@ public class AsaNvIndex public bool TpmNtBitLength { get { return Attributes.HasFlag(NvAttr.TpmNtBitLength); } } public bool TpmNtBitMask { get { return Attributes.HasFlag(NvAttr.TpmNtBitMask); } } public bool TpmNtBitOffset { get { return Attributes.HasFlag(NvAttr.TpmNtBitOffset); } } + [ProtoMember(3)] public List? value { get; set; } public bool Writeall { get { return Attributes.HasFlag(NvAttr.Writeall); } } public bool Writedefine { get { return Attributes.HasFlag(NvAttr.Writedefine); } } diff --git a/Lib/Objects/CertificateObject.cs b/Lib/Objects/CertificateObject.cs index e9ba0710a..0273ac91b 100644 --- a/Lib/Objects/CertificateObject.cs +++ b/Lib/Objects/CertificateObject.cs @@ -1,8 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. using Microsoft.CST.AttackSurfaceAnalyzer.Types; +using ProtoBuf; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract] public class CertificateObject : CollectObject { public CertificateObject(string StoreLocation, string StoreName, SerializableCertificate Certificate) @@ -11,11 +13,13 @@ public CertificateObject(string StoreLocation, string StoreName, SerializableCer this.StoreName = StoreName; this.Certificate = Certificate; } + public CertificateObject() { } public override RESULT_TYPE ResultType => RESULT_TYPE.CERTIFICATE; /// /// A serializable representation of the Certificate. /// + [ProtoMember(1)] public SerializableCertificate Certificate { get; set; } /// @@ -43,11 +47,13 @@ public override string Identity /// /// The Store Location or Location on Disk where the Certificate was found /// + [ProtoMember(2)] public string StoreLocation { get; set; } /// /// The Name of an X509 Store or another source (like the filesystem) /// + [ProtoMember(3)] public string StoreName { get; set; } /// diff --git a/Lib/Objects/CollectObject.cs b/Lib/Objects/CollectObject.cs index 3d477811f..6c2241b1c 100644 --- a/Lib/Objects/CollectObject.cs +++ b/Lib/Objects/CollectObject.cs @@ -38,7 +38,7 @@ public string RowKey { get { - return Serialized.GetHashCode().ToString(CultureInfo.InvariantCulture); + return CryptoHelpers.CreateHash(Serialized); } } diff --git a/Lib/Objects/ComObject.cs b/Lib/Objects/ComObject.cs index a1119285d..0acb22926 100644 --- a/Lib/Objects/ComObject.cs +++ b/Lib/Objects/ComObject.cs @@ -1,8 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. using Microsoft.CST.AttackSurfaceAnalyzer.Types; +using ProtoBuf; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract(SkipConstructor = true)] public class ComObject : CollectObject { /// @@ -29,16 +31,19 @@ public override string Identity /// /// The Registry Key which specifies this COM object /// + [ProtoMember(1)] public RegistryObject Key { get; set; } /// /// The associated binary found (if any) in the x64 view of the registry /// + [ProtoMember(2)] public FileSystemObject? x64_Binary { get; set; } /// /// The associated binary found (if any) in the x86 view of the registry /// + [ProtoMember(3)] public FileSystemObject? x86_Binary { get; set; } } } \ No newline at end of file diff --git a/Lib/Objects/CryptographicKeyObject.cs b/Lib/Objects/CryptographicKeyObject.cs index 1a6395ece..a0c994d65 100644 --- a/Lib/Objects/CryptographicKeyObject.cs +++ b/Lib/Objects/CryptographicKeyObject.cs @@ -2,6 +2,7 @@ using Microsoft.CST.AttackSurfaceAnalyzer.Types; using Newtonsoft.Json; +using ProtoBuf; using Serilog; using System; using System.Security.Cryptography; @@ -9,6 +10,7 @@ namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract(SkipConstructor = true)] public class CryptographicKeyObject : CollectObject { public CryptographicKeyObject(string Source, TpmAlgId tpmAlgId) @@ -27,16 +29,20 @@ public override string Identity } } + [ProtoMember(1)] public RsaKeyDetails? RsaDetails { get; set; } + [ProtoMember(2)] public string Source { get; set; } - + [ProtoMember(3)] public TpmAlgId tpmAlgId { get; set; } = TpmAlgId.Null; } + [ProtoContract] public class KeyDetailObject { } + [ProtoContract(SkipConstructor = true)] public class RsaKeyDetails : KeyDetailObject { public RsaKeyDetails(byte[] modulus, byte[] d, byte[]? p = null, byte[]? q = null) @@ -82,7 +88,6 @@ public RsaKeyDetails(string? PublicString = null, string? FullString = null) Log.Debug(e, "Failed to import RSA key."); } } - public string? FullString { get diff --git a/Lib/Objects/DriverObject.cs b/Lib/Objects/DriverObject.cs index 5887eb080..b4939d94e 100644 --- a/Lib/Objects/DriverObject.cs +++ b/Lib/Objects/DriverObject.cs @@ -1,10 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. using Microsoft.CST.AttackSurfaceAnalyzer.Types; +using ProtoBuf; using System; using System.Collections.Generic; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract] public class DriverObject : CollectObject { public DriverObject(string Name) @@ -12,14 +14,23 @@ public DriverObject(string Name) this.Name = Name; } public override RESULT_TYPE ResultType => RESULT_TYPE.DRIVER; + [ProtoMember(1)] public bool? AcceptPause { get; set; } + [ProtoMember(2)] public bool? AcceptStop { get; set; } + [ProtoMember(3)] public string? Address { get; set; } + [ProtoMember(4)] public string? Architecture { get; set; } + [ProtoMember(5)] public long? BSS { get; set; } + [ProtoMember(6)] public long? Code { get; set; } + [ProtoMember(7)] public string? Description { get; set; } + [ProtoMember(8)] public string? DisplayName { get; set; } + [ProtoMember(9)] public string? DriverType { get; set; } public override string Identity @@ -30,23 +41,39 @@ public override string Identity } } + [ProtoMember(10)] public int? Index { get; set; } + [ProtoMember(11)] public long? Init { get; set; } + [ProtoMember(12)] public DateTime? LinkDate { get; set; } + [ProtoMember(13)] public List? LinkedAgainst { get; set; } + [ProtoMember(14)] public string Name { get; set; } + [ProtoMember(15)] public long? PagedPool { get; set; } + [ProtoMember(16)] public string? Path { get; set; } + [ProtoMember(17)] public Dictionary? Properties { get; set; } + [ProtoMember(18)] public int? Refs { get; set; } + [ProtoMember(19)] public Signature? Signature { get; set; } + [ProtoMember(20)] public string? Size { get; set; } - + [ProtoMember(21)] public string? StartMode { get; set; } + [ProtoMember(22)] public string? State { get; set; } + [ProtoMember(23)] public string? Status { get; set; } + [ProtoMember(24)] public string? UUID { get; set; } + [ProtoMember(25)] public string? Version { get; set; } + [ProtoMember(26)] public string? Wired { get; set; } } } \ No newline at end of file diff --git a/Lib/Objects/EventLogObject.cs b/Lib/Objects/EventLogObject.cs index a36e555a9..9196ee39a 100644 --- a/Lib/Objects/EventLogObject.cs +++ b/Lib/Objects/EventLogObject.cs @@ -1,10 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. using Microsoft.CST.AttackSurfaceAnalyzer.Types; +using ProtoBuf; using System; using System.Collections.Generic; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract(SkipConstructor = true)] public class EventLogObject : CollectObject { public EventLogObject(string Event) @@ -17,11 +19,13 @@ public EventLogObject(string Event) /// /// Additional associated data /// + [ProtoMember(1)] public List? Data { get; set; } /// /// The raw event text /// + [ProtoMember(2)] public string Event { get; set; } /// @@ -38,26 +42,31 @@ public override string Identity /// /// The severity level of the event message (availability platform dependent) /// + [ProtoMember(3)] public string? Level { get; set; } /// /// The process that the event log is from. /// + [ProtoMember(4)] public string? Process { get; set; } /// /// The Event Log source /// + [ProtoMember(5)] public string? Source { get; set; } /// /// A summary description of the event message (availability platform dependent) /// + [ProtoMember(6)] public string? Summary { get; set; } /// /// The recorded Timestamp in the log file /// + [ProtoMember(7)] public DateTime? Timestamp { get; set; } } } \ No newline at end of file diff --git a/Lib/Objects/FileMonitorObject.cs b/Lib/Objects/FileMonitorObject.cs index 182568674..522775a69 100644 --- a/Lib/Objects/FileMonitorObject.cs +++ b/Lib/Objects/FileMonitorObject.cs @@ -1,9 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. using Microsoft.CST.AttackSurfaceAnalyzer.Types; +using ProtoBuf; using System.IO; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract] public class FileMonitorObject : MonitorObject { public FileMonitorObject(string PathIn) @@ -12,7 +14,9 @@ public FileMonitorObject(string PathIn) } public override RESULT_TYPE ResultType => RESULT_TYPE.FILEMONITOR; + [ProtoMember(9)] public string? ExtendedResults { get; set; } + [ProtoMember(2)] public FileSystemObject? FileSystemObject { get; set; } public override string Identity @@ -23,11 +27,17 @@ public override string Identity } } + [ProtoMember(3)] public string? Name { get; set; } + [ProtoMember(4)] public NotifyFilters? NotifyFilters { get; set; } + [ProtoMember(5)] public string? OldName { get; set; } + [ProtoMember(6)] public string? OldPath { get; set; } + [ProtoMember(7)] public string Path { get; set; } + [ProtoMember(8)] public string? Timestamp { get; set; } } } \ No newline at end of file diff --git a/Lib/Objects/FileSystemObject.cs b/Lib/Objects/FileSystemObject.cs index e7c424416..a117c47ce 100644 --- a/Lib/Objects/FileSystemObject.cs +++ b/Lib/Objects/FileSystemObject.cs @@ -3,7 +3,6 @@ using ProtoBuf; using System; using System.Collections.Generic; -using System.IO; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { @@ -27,19 +26,19 @@ public FileSystemObject() /// /// If this is windows executable what DLL Characteristics are set /// - [ProtoMember(23)] + [ProtoMember(3)] public List? Characteristics { get; set; } /// /// A hash of the file (if collected) /// - [ProtoMember(4)] + [ProtoMember(1)] public string? ContentHash { get; set; } /// /// When was the file created in UTC /// - [ProtoMember(24)] + [ProtoMember(2)] public DateTime Created { get; set; } /// @@ -144,7 +143,7 @@ public FileSystemObject() /// /// If this is a link where does it point to. /// - [ProtoMember(22)] + [ProtoMember(4)] public string? Target { get; set; } public bool ShouldSerializeCharacteristics() diff --git a/Lib/Objects/FirewallObject.cs b/Lib/Objects/FirewallObject.cs index 0e1dd1fad..5fec13fe7 100644 --- a/Lib/Objects/FirewallObject.cs +++ b/Lib/Objects/FirewallObject.cs @@ -1,10 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. using Microsoft.CST.AttackSurfaceAnalyzer.Types; +using ProtoBuf; using System.Collections.Generic; using WindowsFirewallHelper; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract(SkipConstructor = true)] public class FirewallObject : CollectObject { public FirewallObject(string Name) @@ -18,21 +20,25 @@ public FirewallObject(string Name) /// /// Gets or sets the action that the rules defines /// + [ProtoMember(1)] public FirewallAction? Action { get; set; } /// /// Gets or sets the name of the application that this rule is about /// + [ProtoMember(2)] public string? ApplicationName { get; set; } /// /// Gets or sets the data direction that the rule applies to /// + [ProtoMember(3)] public FirewallDirection? Direction { get; set; } /// /// Gets or sets the resolved name of the rule /// + [ProtoMember(4)] public string? FriendlyName { get; set; } /// @@ -49,56 +55,67 @@ public override string Identity /// /// Gets or sets a Boolean value indicating if this rule is active /// + [ProtoMember(5)] public bool? IsEnable { get; set; } /// /// Gets or sets the local addresses that the rule applies to /// + [ProtoMember(6)] public List? LocalAddresses { get; set; } /// /// Gets or sets the local ports that the rule applies to /// + [ProtoMember(7)] public List? LocalPorts { get; set; } /// /// Gets or sets the type of local ports that the rules applies to /// + [ProtoMember(8)] public FirewallPortType? LocalPortType { get; set; } /// /// Gets or sets the name of the rule in native format w/o auto string resolving /// + [ProtoMember(9)] public string Name { get; set; } /// /// Gets the profiles that this rule belongs to /// + [ProtoMember(10)] public FirewallProfiles? Profiles { get; set; } /// /// Gets or sets the protocol that the rule applies to /// + [ProtoMember(11)] public string? Protocol { get; set; } /// /// Gets or sets the remote addresses that the rule applies to /// + [ProtoMember(12)] public List? RemoteAddresses { get; set; } /// /// Gets or sets the remote ports that the rule applies to /// + [ProtoMember(13)] public List? RemotePorts { get; set; } /// /// Gets or sets the scope that the rule applies to /// + [ProtoMember(14)] public FirewallScope? Scope { get; set; } /// /// Gets or sets the name of the service that this rule is about /// + [ProtoMember(15)] public string? ServiceName { get; set; } } } \ No newline at end of file diff --git a/Lib/Objects/MonitorObject.cs b/Lib/Objects/MonitorObject.cs index 9d47148a3..1a6e0ed13 100644 --- a/Lib/Objects/MonitorObject.cs +++ b/Lib/Objects/MonitorObject.cs @@ -1,13 +1,16 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. using Microsoft.CST.AttackSurfaceAnalyzer.Types; +using ProtoBuf; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { /// /// Abstract parent class that all Collected data inherits from. /// + [ProtoContract] public abstract class MonitorObject : CollectObject { + [ProtoMember(1)] public CHANGE_TYPE ChangeType { get; set; } } } \ No newline at end of file diff --git a/Lib/Objects/OpenPortObject.cs b/Lib/Objects/OpenPortObject.cs index a0eb0a02e..6375128ef 100644 --- a/Lib/Objects/OpenPortObject.cs +++ b/Lib/Objects/OpenPortObject.cs @@ -1,9 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. using Microsoft.CST.AttackSurfaceAnalyzer.Types; using Newtonsoft.Json; +using ProtoBuf; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract(SkipConstructor = true)] public class OpenPortObject : CollectObject { [JsonConstructor] @@ -17,12 +19,13 @@ public OpenPortObject(int Port, TRANSPORT Type, ADDRESS_FAMILY AddressFamily) } public override RESULT_TYPE ResultType => RESULT_TYPE.PORT; - + [ProtoMember(1)] public string? Address { get; set; } /// /// InterNetwork is IPv4 InterNetworkV6 is IPv6 /// + [ProtoMember(2)] public ADDRESS_FAMILY AddressFamily { get; set; } /// @@ -39,21 +42,25 @@ public override string Identity /// /// The port number /// + [ProtoMember(3)] public int Port { get; set; } /// /// The associated process if known /// + [ProtoMember(4)] public string? ProcessName { get; set; } /// /// The associated process ID if known /// + [ProtoMember(5)] public int? ProcessId { get; set; } /// /// TCP or UDP /// + [ProtoMember(6)] public TRANSPORT Type { get; set; } } } \ No newline at end of file diff --git a/Lib/Objects/ProcessModuleObject.cs b/Lib/Objects/ProcessModuleObject.cs index 5e1c180c4..58312bf5b 100644 --- a/Lib/Objects/ProcessModuleObject.cs +++ b/Lib/Objects/ProcessModuleObject.cs @@ -1,9 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. +using ProtoBuf; using System.Collections.Generic; using System.Diagnostics; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract] public class ProcessModuleObject { public ProcessModuleObject(string? FileName, string? ModuleName, SerializableFileVersionInfo? FileVersionInfo) @@ -15,8 +17,11 @@ public ProcessModuleObject(string? FileName, string? ModuleName, SerializableFil public ProcessModuleObject() { } + [ProtoMember(1)] public string? FileName { get; set; } + [ProtoMember(2)] public SerializableFileVersionInfo? FileVersionInfo { get; set; } + [ProtoMember(3)] public string? ModuleName { get; set; } internal static ProcessModuleObject FromProcessModule(ProcessModule mainModule) diff --git a/Lib/Objects/ProcessObject.cs b/Lib/Objects/ProcessObject.cs index 0a46336d0..7f0c9b727 100644 --- a/Lib/Objects/ProcessObject.cs +++ b/Lib/Objects/ProcessObject.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. using Microsoft.CST.AttackSurfaceAnalyzer.Types; +using ProtoBuf; using Serilog; using System; using System.Collections.Generic; @@ -7,6 +8,7 @@ namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract(SkipConstructor = true)] public class ProcessObject : CollectObject { public ProcessObject(int Id, string ProcessName) @@ -17,10 +19,13 @@ public ProcessObject(int Id, string ProcessName) public override RESULT_TYPE ResultType => RESULT_TYPE.PROCESS; + [ProtoMember(1)] public int BasePriority { get; set; } + [ProtoMember(2)] public bool HasExited { get; set; } + [ProtoMember(3)] public int Id { get; } /// @@ -34,14 +39,19 @@ public override string Identity } } + [ProtoMember(4)] public ProcessModuleObject? MainModule { get; set; } + [ProtoMember(5)] public List Modules { get; set; } = new List(); + [ProtoMember(6)] public ProcessPriorityClass PriorityClass { get; set; } + [ProtoMember(7)] public string ProcessName { get; set; } + [ProtoMember(8)] public DateTime StartTime { get; set; } public static ProcessObject? FromProcess(Process process) diff --git a/Lib/Objects/RegistryObject.cs b/Lib/Objects/RegistryObject.cs index f7f56a116..e6c5794fd 100644 --- a/Lib/Objects/RegistryObject.cs +++ b/Lib/Objects/RegistryObject.cs @@ -1,12 +1,14 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. using Microsoft.CST.AttackSurfaceAnalyzer.Types; using Microsoft.Win32; +using ProtoBuf; using System; using System.Collections.Generic; using System.Runtime.InteropServices; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract(SkipConstructor = true)] public class RegistryObject : CollectObject { public RegistryObject(string Key, RegistryView View) @@ -28,9 +30,12 @@ public override string Identity /// /// The Full Path to the Key in the Registry /// + [ProtoMember(1)] public string Key { get; set; } + [ProtoMember(2)] public Dictionary> Permissions { get; set; } = new Dictionary>(); + [ProtoMember(3)] public string? PermissionsString { get; set; } public int SubkeyCount @@ -38,6 +43,7 @@ public int SubkeyCount get { return Subkeys?.Count ?? 0; } } + [ProtoMember(4)] public List? Subkeys { get; set; } public int ValueCount @@ -45,7 +51,9 @@ public int ValueCount get { return Values?.Count ?? 0; } } + [ProtoMember(5)] public Dictionary? Values { get; set; } + [ProtoMember(6)] public RegistryView View { get; private set; } public static Dictionary GetValues(RegistryKey key) diff --git a/Lib/Objects/SerializableCertificate.cs b/Lib/Objects/SerializableCertificate.cs index b3552064d..4310f108d 100644 --- a/Lib/Objects/SerializableCertificate.cs +++ b/Lib/Objects/SerializableCertificate.cs @@ -6,7 +6,7 @@ namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { - [ProtoContract] + [ProtoContract(SkipConstructor = true)] public class SerializableCertificate { public SerializableCertificate(X509Certificate2 certificate) diff --git a/Lib/Objects/SerializableFileVersionInfo.cs b/Lib/Objects/SerializableFileVersionInfo.cs index 7fe8319b2..a8d74421a 100644 --- a/Lib/Objects/SerializableFileVersionInfo.cs +++ b/Lib/Objects/SerializableFileVersionInfo.cs @@ -1,7 +1,9 @@ -using System.Diagnostics; +using ProtoBuf; +using System.Diagnostics; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract] public class SerializableFileVersionInfo { public SerializableFileVersionInfo() @@ -11,56 +13,66 @@ public SerializableFileVersionInfo() // Summary: Gets the comments associated with the file. // // Returns: The comments associated with the file or null if the file did not contain version information. + [ProtoMember(1)] public string? Comments { get; set; } // Summary: Gets the name of the company that produced the file. // // Returns: The name of the company that produced the file or null if the file did not contain version information. + [ProtoMember(2)] public string? CompanyName { get; set; } // Summary: Gets the build number of the file. // // Returns: A value representing the build number of the file or null if the file did not contain // version information. + [ProtoMember(3)] public int FileBuildPart { get; set; } // Summary: Gets the description of the file. // // Returns: The description of the file or null if the file did not contain version information. + [ProtoMember(4)] public string? FileDescription { get; set; } // Summary: Gets the major part of the version number. // // Returns: A value representing the major part of the version number or 0 (zero) if the file did not // contain version information. + [ProtoMember(5)] public int FileMajorPart { get; set; } // Summary: Gets the minor part of the version number of the file. // // Returns: A value representing the minor part of the version number of the file or 0 (zero) if the // file did not contain version information. + [ProtoMember(6)] public int FileMinorPart { get; set; } // Summary: Gets the name of the file that this instance of System.Diagnostics.FileVersionInfo describes. // // Returns: The name of the file described by this instance of System.Diagnostics.FileVersionInfo. + [ProtoMember(7)] public string? FileName { get; set; } // Summary: Gets the file private part number. // // Returns: A value representing the file private part number or null if the file did not contain // version information. + [ProtoMember(8)] public int FilePrivatePart { get; set; } // Summary: Gets the file version number. // // Returns: The version number of the file or null if the file did not contain version information. + [ProtoMember(9)] public string? FileVersion { get; set; } // Summary: Gets the internal name of the file, if one exists. // // Returns: The internal name of the file. If none exists, this property will contain the original // name of the file without the extension. + [ProtoMember(10)] public string? InternalName { get; set; } // Summary: Gets a value that specifies whether the file contains debugging information or is compiled @@ -68,94 +80,111 @@ public SerializableFileVersionInfo() // // Returns: true if the file contains debugging information or is compiled with debugging features // enabled; otherwise, false. + [ProtoMember(11)] public bool IsDebug { get; set; } // Summary: Gets a value that specifies whether the file has been modified and is not identical to the // original shipping file of the same version number. // // Returns: true if the file is patched; otherwise, false. + [ProtoMember(12)] public bool IsPatched { get; set; } + [ProtoMember(13)] public bool IsPreRelease { get; set; } // Summary: Gets a value that specifies whether the file was built using standard release procedures. // // Returns: true if the file is a private build; false if the file was built using standard release // procedures or if the file did not contain version information. + [ProtoMember(14)] public bool IsPrivateBuild { get; set; } // Summary: Gets a value that specifies whether the file is a special build. // // Returns: true if the file is a special build; otherwise, false. + [ProtoMember(15)] public bool IsSpecialBuild { get; set; } // Summary: Gets the default language string for the version info block. // // Returns: The description string for the Microsoft Language Identifier in the version resource or // null if the file did not contain version information. + [ProtoMember(16)] public string? Language { get; set; } // Summary: Gets all copyright notices that apply to the specified file. // // Returns: The copyright notices that apply to the specified file. + [ProtoMember(17)] public string? LegalCopyright { get; set; } // Summary: Gets the trademarks and registered trademarks that apply to the file. // // Returns: The trademarks and registered trademarks that apply to the file or null if the file did // not contain version information. + [ProtoMember(18)] public string? LegalTrademarks { get; set; } // Summary: Gets the name the file was created with. // // Returns: The name the file was created with or null if the file did not contain version information. + [ProtoMember(19)] public string? OriginalFilename { get; set; } // Summary: Gets information about a private version of the file. // // Returns: Information about a private version of the file or null if the file did not contain // version information. + [ProtoMember(20)] public string? PrivateBuild { get; set; } // Summary: Gets the build number of the product this file is associated with. // // Returns: A value representing the build number of the product this file is associated with or null // if the file did not contain version information. + [ProtoMember(21)] public int ProductBuildPart { get; set; } // Summary: Gets the major part of the version number for the product this file is associated with. // // Returns: A value representing the major part of the product version number or null if the file did // not contain version information. + [ProtoMember(22)] public int ProductMajorPart { get; set; } // Summary: Gets the minor part of the version number for the product the file is associated with. // // Returns: A value representing the minor part of the product version number or null if the file did // not contain version information. + [ProtoMember(23)] public int ProductMinorPart { get; set; } // Summary: Gets the name of the product this file is distributed with. // // Returns: The name of the product this file is distributed with or null if the file did not contain // version information. + [ProtoMember(24)] public string? ProductName { get; set; } // Summary: Gets the private part number of the product this file is associated with. // // Returns: A value representing the private part number of the product this file is associated with // or null if the file did not contain version information. + [ProtoMember(25)] public int ProductPrivatePart { get; set; } // Summary: Gets the version of the product this file is distributed with. // // Returns: The version of the product this file is distributed with or null if the file did not // contain version information. + [ProtoMember(26)] public string? ProductVersion { get; set; } // Summary: Gets the special build information for the file. // // Returns: The special build information for the file or null if the file did not contain version information. + [ProtoMember(27)] public string? SpecialBuild { get; set; } public static SerializableFileVersionInfo? FromFileVersionInfo(FileVersionInfo fvi) diff --git a/Lib/Objects/ServiceObject.cs b/Lib/Objects/ServiceObject.cs index 72fce6787..1b7e4e780 100644 --- a/Lib/Objects/ServiceObject.cs +++ b/Lib/Objects/ServiceObject.cs @@ -1,10 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. using Microsoft.CST.AttackSurfaceAnalyzer.Types; using Newtonsoft.Json; +using ProtoBuf; using System; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract(SkipConstructor = true)] public class ServiceObject : CollectObject { [JsonConstructor] @@ -14,16 +16,27 @@ public ServiceObject(string Name) } public override RESULT_TYPE ResultType => RESULT_TYPE.SERVICE; + [ProtoMember(1)] public bool? AcceptPause { get; set; } + [ProtoMember(2)] public bool? AcceptStop { get; set; } + [ProtoMember(3)] public string? Caption { get; set; } + [ProtoMember(4)] public uint? CheckPoint { get; set; } + [ProtoMember(5)] public string? CreationClassName { get; set; } + [ProtoMember(6)] public bool? DelayedAutoStart { get; set; } + [ProtoMember(7)] public string? Description { get; set; } + [ProtoMember(8)] public bool? DesktopInteract { get; set; } + [ProtoMember(9)] public string? DisplayName { get; set; } + [ProtoMember(10)] public string? ErrorControl { get; set; } + [ProtoMember(11)] public uint? ExitCode { get; set; } public override string Identity @@ -34,20 +47,35 @@ public override string Identity } } + [ProtoMember(12)] public DateTime InstallDate { get; set; } + [ProtoMember(13)] public string Name { get; set; } + [ProtoMember(14)] public string? PathName { get; set; } + [ProtoMember(15)] public uint? ProcessId { get; set; } + [ProtoMember(16)] public uint? ServiceSpecificExitCode { get; set; } + [ProtoMember(17)] public string? ServiceType { get; set; } + [ProtoMember(18)] public bool? Started { get; set; } + [ProtoMember(19)] public string? StartMode { get; set; } + [ProtoMember(20)] public string? StartName { get; set; } + [ProtoMember(21)] public string? State { get; set; } + [ProtoMember(22)] public string? Status { get; set; } + [ProtoMember(23)] public string? SystemCreationClassName { get; set; } + [ProtoMember(24)] public string? SystemName { get; set; } + [ProtoMember(25)] public uint? TagId { get; set; } + [ProtoMember(26)] public uint? WaitHint { get; set; } } } \ No newline at end of file diff --git a/Lib/Objects/TpmObject.cs b/Lib/Objects/TpmObject.cs index 3862b520b..95b0cc92e 100644 --- a/Lib/Objects/TpmObject.cs +++ b/Lib/Objects/TpmObject.cs @@ -1,11 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. using Microsoft.CST.AttackSurfaceAnalyzer.Types; +using ProtoBuf; using System; using System.Collections.Generic; using Tpm2Lib; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract(SkipConstructor = true)] public class TpmObject : CollectObject { public TpmObject(string Location) @@ -14,7 +16,9 @@ public TpmObject(string Location) } public override RESULT_TYPE ResultType => RESULT_TYPE.TPM; + [ProtoMember(1)] public List Algorithms { get; set; } = new List(); + [ProtoMember(2)] public List Commands { get; set; } = new List(); public override string Identity @@ -25,13 +29,21 @@ public override string Identity } } + [ProtoMember(3)] public string Location { get; } + [ProtoMember(4)] public string? Manufacturer { get; set; } + [ProtoMember(5)] public List NV { get; set; } = new List(); + [ProtoMember(6)] public Dictionary<(TpmAlgId, uint), byte[]> PCRs { get; set; } = new Dictionary<(TpmAlgId, uint), byte[]>(); + [ProtoMember(7)] public List PersistentKeys { get; set; } = new List(); + [ProtoMember(8)] public List RandomKeys { get; set; } = new List(); + [ProtoMember(9)] public DateTime TpmSpecDate { get; set; } + [ProtoMember(10)] public string? Version { get; set; } } } \ No newline at end of file diff --git a/Lib/Objects/UserAccountObject.cs b/Lib/Objects/UserAccountObject.cs index 10254ee92..6f63776e9 100644 --- a/Lib/Objects/UserAccountObject.cs +++ b/Lib/Objects/UserAccountObject.cs @@ -1,10 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. using Microsoft.CST.AttackSurfaceAnalyzer.Types; +using ProtoBuf; using System.Collections.Generic; using System.Globalization; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract(SkipConstructor = true)] public class UserAccountObject : CollectObject { public UserAccountObject(string Name) @@ -13,15 +15,25 @@ public UserAccountObject(string Name) } public override RESULT_TYPE ResultType => RESULT_TYPE.USER; + [ProtoMember(1)] public string? AccountType { get; set; } + [ProtoMember(2)] public string? Caption { get; set; } + [ProtoMember(3)] public string? Description { get; set; } + [ProtoMember(4)] public string? Disabled { get; set; } + [ProtoMember(5)] public string? Domain { get; set; } + [ProtoMember(6)] public string? FullName { get; set; } + [ProtoMember(7)] public string? GID { get; set; } + [ProtoMember(8)] public List Groups { get; set; } = new List(); + [ProtoMember(9)] public bool? Hidden { get; set; } + [ProtoMember(10)] public string? HomeDirectory { get; set; } public override string Identity @@ -32,19 +44,33 @@ public override string Identity } } + [ProtoMember(11)] public string? Inactive { get; set; } + [ProtoMember(12)] public string? InstallDate { get; set; } + [ProtoMember(13)] public string? LocalAccount { get; set; } + [ProtoMember(14)] public string? Lockout { get; set; } + [ProtoMember(15)] public string Name { get; set; } + [ProtoMember(16)] public string? PasswordChangeable { get; set; } + [ProtoMember(17)] public string? PasswordExpires { get; set; } + [ProtoMember(18)] public string? PasswordRequired { get; set; } + [ProtoMember(19)] public string? PasswordStorageAlgorithm { get; set; } + [ProtoMember(20)] public bool? Privileged { get; set; } + [ProtoMember(21)] public Dictionary? Properties { get; set; } + [ProtoMember(22)] public string? Shell { get; set; } + [ProtoMember(23)] public string? SID { get; set; } + [ProtoMember(24)] public string? UID { get; set; } } } \ No newline at end of file diff --git a/Lib/Objects/UserGroupObject.cs b/Lib/Objects/UserGroupObject.cs index aa99fcc33..9459f160b 100644 --- a/Lib/Objects/UserGroupObject.cs +++ b/Lib/Objects/UserGroupObject.cs @@ -1,9 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. using Microsoft.CST.AttackSurfaceAnalyzer.Types; +using ProtoBuf; using System.Collections.Generic; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract(SkipConstructor = true)] public class GroupAccountObject : CollectObject { public GroupAccountObject(string Name) @@ -12,8 +14,11 @@ public GroupAccountObject(string Name) } public override RESULT_TYPE ResultType => RESULT_TYPE.GROUP; + [ProtoMember(1)] public string? Caption { get; set; } + [ProtoMember(2)] public string? Description { get; set; } + [ProtoMember(3)] public string? Domain { get; set; } public override string Identity @@ -24,13 +29,21 @@ public override string Identity } } + [ProtoMember(4)] public string? InstallDate { get; set; } + [ProtoMember(5)] public bool? LocalAccount { get; set; } + [ProtoMember(6)] public string Name { get; set; } + [ProtoMember(7)] public Dictionary? Properties { get; set; } + [ProtoMember(8)] public string? SID { get; set; } + [ProtoMember(9)] public int? SIDType { get; set; } + [ProtoMember(10)] public string? Status { get; set; } + [ProtoMember(11)] public List Users { get; set; } = new List(); } } \ No newline at end of file diff --git a/Lib/Objects/WifiObject.cs b/Lib/Objects/WifiObject.cs index afdb05b63..ae0b4d3bb 100644 --- a/Lib/Objects/WifiObject.cs +++ b/Lib/Objects/WifiObject.cs @@ -1,7 +1,9 @@ using Microsoft.CST.AttackSurfaceAnalyzer.Types; +using ProtoBuf; namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects { + [ProtoContract] public class WifiObject : CollectObject { public WifiObject(string SSID) @@ -11,8 +13,10 @@ public WifiObject(string SSID) public override RESULT_TYPE ResultType => RESULT_TYPE.WIFI; + [ProtoMember(1)] public string? Authentication { get; set; } + [ProtoMember(2)] public string? Encryption { get; set; } public override string Identity @@ -23,7 +27,9 @@ public override string Identity } } + [ProtoMember(3)] public string? Password { get; set; } + [ProtoMember(4)] public string SSID { get; set; } } } \ No newline at end of file diff --git a/Lib/Utils/CryptoHelpers.cs b/Lib/Utils/CryptoHelpers.cs index 953899ae4..a442c0bff 100644 --- a/Lib/Utils/CryptoHelpers.cs +++ b/Lib/Utils/CryptoHelpers.cs @@ -19,6 +19,7 @@ public static string CreateHash(string input) { try { + using HashAlgorithm sha512 = SHA512.Create(); byte[] hashOutput = sha512.ComputeHash(Encoding.UTF8.GetBytes(input)); return Convert.ToBase64String(hashOutput); } @@ -29,10 +30,26 @@ public static string CreateHash(string input) } } + public static string CreateHash(byte[] input) + { + try + { + using HashAlgorithm sha512 = SHA512.Create(); + byte[] hashOutput = sha512.ComputeHash(input); + return Convert.ToBase64String(hashOutput); + } + catch (CryptographicException e) + { + Log.Warning(e, Strings.Get("Err_CreateHash"), "[byte array]"); + return string.Empty; + } + } + public static string CreateHash(Stream stream) { try { + using HashAlgorithm sha512 = SHA512.Create(); return Convert.ToBase64String(sha512.ComputeHash(stream) ?? Array.Empty()); } catch (CryptographicException e) @@ -63,7 +80,5 @@ public static int GetRandomPositiveIndex(int max) public static string GetRandomString(int characters) => new(Enumerable.Range(1, characters).Select(_ => chars[GetRandomPositiveIndex(chars.Length)]).ToArray()); private const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - - private static readonly HashAlgorithm sha512 = SHA512.Create(); } } \ No newline at end of file diff --git a/Lib/Utils/JsonUtils.cs b/Lib/Utils/JsonUtils.cs index 3d6e5915a..60367860a 100644 --- a/Lib/Utils/JsonUtils.cs +++ b/Lib/Utils/JsonUtils.cs @@ -29,13 +29,57 @@ static JsonUtils() public static byte[] Dehydrate(CollectObject colObj) { using var ms = new MemoryStream(); - if (colObj is FileSystemObject fileObject) + switch (colObj) { - Serializer.Serialize(ms, fileObject); - return ms.ToArray(); + case CertificateObject certificateObject: + Serializer.Serialize(ms, certificateObject); + break; + case FileSystemObject fileSystemObject: + Serializer.Serialize(ms, fileSystemObject); + break; + case OpenPortObject openPortObject: + Serializer.Serialize(ms, openPortObject); + break; + case RegistryObject registryObject: + Serializer.Serialize(ms, registryObject); + break; + case ServiceObject serviceObject: + Serializer.Serialize(ms, serviceObject); + break; + case UserAccountObject userAccountObject: + Serializer.Serialize(ms, userAccountObject); + break; + case GroupAccountObject groupAccountObject: + Serializer.Serialize(ms, groupAccountObject); + break; + case FirewallObject firewallObject: + Serializer.Serialize(ms, firewallObject); + break; + case ComObject comObject: + Serializer.Serialize(ms, comObject); + break; + case EventLogObject eventLogObject: + Serializer.Serialize(ms, eventLogObject); + break; + case TpmObject tpmObject: + Serializer.Serialize(ms, tpmObject); + break; + case CryptographicKeyObject cryptographicKeyObject: + Serializer.Serialize(ms, cryptographicKeyObject); + break; + case ProcessObject processObject: + Serializer.Serialize(ms, processObject); + break; + case DriverObject driverObject: + Serializer.Serialize(ms, driverObject); + break; + case FileMonitorObject fileMonitorObject: + Serializer.Serialize(ms, fileMonitorObject); + break; + default: + throw new NotSupportedException(); } - return new byte[0]; -// return JsonConvert.SerializeObject(colObj, jsonSettings); + return ms.ToArray(); } /// @@ -57,46 +101,32 @@ public static byte[] Dehydrate(CollectObject colObj) { case RESULT_TYPE.CERTIFICATE: return Serializer.Deserialize(new MemoryStream(serialized)); - case RESULT_TYPE.FILE: return Serializer.Deserialize(new MemoryStream(serialized)); - case RESULT_TYPE.PORT: return Serializer.Deserialize(new MemoryStream(serialized)); - case RESULT_TYPE.REGISTRY: return Serializer.Deserialize(new MemoryStream(serialized)); - case RESULT_TYPE.SERVICE: return Serializer.Deserialize(new MemoryStream(serialized)); - case RESULT_TYPE.USER: return Serializer.Deserialize(new MemoryStream(serialized)); - case RESULT_TYPE.GROUP: return Serializer.Deserialize(new MemoryStream(serialized)); - case RESULT_TYPE.FIREWALL: return Serializer.Deserialize(new MemoryStream(serialized)); - case RESULT_TYPE.COM: return Serializer.Deserialize(new MemoryStream(serialized)); - case RESULT_TYPE.LOG: return Serializer.Deserialize(new MemoryStream(serialized)); - case RESULT_TYPE.TPM: return Serializer.Deserialize(new MemoryStream(serialized)); - case RESULT_TYPE.KEY: return Serializer.Deserialize(new MemoryStream(serialized)); - case RESULT_TYPE.PROCESS: return Serializer.Deserialize(new MemoryStream(serialized)); - case RESULT_TYPE.DRIVER: return Serializer.Deserialize(new MemoryStream(serialized)); - case RESULT_TYPE.FILEMONITOR: return Serializer.Deserialize(new MemoryStream(serialized)); default: diff --git a/Tests/HydrationTests.cs b/Tests/HydrationTests.cs index 52838772b..bb340164d 100644 --- a/Tests/HydrationTests.cs +++ b/Tests/HydrationTests.cs @@ -105,7 +105,7 @@ public void TestSerializeAndDeserializeProcessObject() { var po = ProcessObject.FromProcess(Process.GetCurrentProcess()); var serialized = JsonUtils.Dehydrate(po); - Assert.IsTrue(serialized == JsonUtils.Dehydrate(JsonUtils.Hydrate(serialized, RESULT_TYPE.PROCESS))); + Assert.IsTrue(po.RowKey.Equals(JsonUtils.Hydrate(serialized, RESULT_TYPE.PROCESS)?.RowKey)); } [TestMethod] From 350765d87a67e52bf3c1fc3687134a3e9105757e Mon Sep 17 00:00:00 2001 From: Gabe Stocco <98900+gfs@users.noreply.github.com> Date: Tue, 2 Apr 2024 09:01:37 -0700 Subject: [PATCH 03/10] Better workaround for hasher reuse --- Benchmarks/Program.cs | 2 +- Lib/Utils/CryptoHelpers.cs | 44 ++++++++++++++++++++++++++++++++------ 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/Benchmarks/Program.cs b/Benchmarks/Program.cs index 9f3493c8c..4b7e22d20 100644 --- a/Benchmarks/Program.cs +++ b/Benchmarks/Program.cs @@ -6,7 +6,7 @@ public class Program { public static void Main(string[] args) { - var summary = BenchmarkRunner.Run(); + var summary = BenchmarkRunner.Run(); } } } \ No newline at end of file diff --git a/Lib/Utils/CryptoHelpers.cs b/Lib/Utils/CryptoHelpers.cs index a442c0bff..3ad9f27c8 100644 --- a/Lib/Utils/CryptoHelpers.cs +++ b/Lib/Utils/CryptoHelpers.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. using Serilog; using System; +using System.Collections.Concurrent; using System.IO; using System.Linq; using System.Security.Cryptography; @@ -17,10 +18,10 @@ public static class CryptoHelpers /// public static string CreateHash(string input) { + HashAlgorithm hasher = GetHasher(); try { - using HashAlgorithm sha512 = SHA512.Create(); - byte[] hashOutput = sha512.ComputeHash(Encoding.UTF8.GetBytes(input)); + byte[] hashOutput = hasher.ComputeHash(Encoding.UTF8.GetBytes(input)); return Convert.ToBase64String(hashOutput); } catch (CryptographicException e) @@ -28,14 +29,18 @@ public static string CreateHash(string input) Log.Warning(e, Strings.Get("Err_CreateHash"), input is null ? "null string" : $"'{input}'"); return string.Empty; } + finally + { + ReleaseHasher(hasher); + } } public static string CreateHash(byte[] input) { + HashAlgorithm hasher = GetHasher(); try { - using HashAlgorithm sha512 = SHA512.Create(); - byte[] hashOutput = sha512.ComputeHash(input); + byte[] hashOutput = hasher.ComputeHash(input); return Convert.ToBase64String(hashOutput); } catch (CryptographicException e) @@ -43,20 +48,45 @@ public static string CreateHash(byte[] input) Log.Warning(e, Strings.Get("Err_CreateHash"), "[byte array]"); return string.Empty; } + finally + { + ReleaseHasher(hasher); + } } public static string CreateHash(Stream stream) { + HashAlgorithm hasher = GetHasher(); try { - using HashAlgorithm sha512 = SHA512.Create(); - return Convert.ToBase64String(sha512.ComputeHash(stream) ?? Array.Empty()); + return Convert.ToBase64String(hasher.ComputeHash(stream) ?? Array.Empty()); } catch (CryptographicException e) { Log.Warning(e, Strings.Get("Err_CreateHash"), "stream"); return string.Empty; } + finally + { + ReleaseHasher(hasher); + } + } + + private static HashAlgorithm GetHasher() + { + if (hashers.TryDequeue(out HashAlgorithm? hashAlgorithm) && hashAlgorithm is { }) + { + return hashAlgorithm; + } + else + { + return SHA512.Create(); + } + } + + private static void ReleaseHasher(HashAlgorithm hashAlgorithm) + { + hashers.Enqueue(hashAlgorithm); } public static double GetRandomPositiveDouble(double max) @@ -80,5 +110,7 @@ public static int GetRandomPositiveIndex(int max) public static string GetRandomString(int characters) => new(Enumerable.Range(1, characters).Select(_ => chars[GetRandomPositiveIndex(chars.Length)]).ToArray()); private const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + + private static ConcurrentQueue hashers = new ConcurrentQueue(); } } \ No newline at end of file From 0b921124df388d97b519d251c98d24c0c2301b6b Mon Sep 17 00:00:00 2001 From: Gabe Stocco <98900+gfs@users.noreply.github.com> Date: Tue, 2 Apr 2024 10:53:26 -0700 Subject: [PATCH 04/10] Rename JsonUtils ProtoBufUtils Since it now uses ProtoBuf for serialization. --- Lib/Objects/CollectObject.cs | 10 ++++-- Lib/Objects/WriteObject.cs | 2 +- Lib/Utils/{JsonUtils.cs => ProtoBufUtils.cs} | 6 ++-- Lib/Utils/SqliteDatabaseManager.cs | 10 +++--- Tests/HydrationTests.cs | 34 ++++++++++---------- 5 files changed, 33 insertions(+), 29 deletions(-) rename Lib/Utils/{JsonUtils.cs => ProtoBufUtils.cs} (98%) diff --git a/Lib/Objects/CollectObject.cs b/Lib/Objects/CollectObject.cs index 6c2241b1c..3e883b044 100644 --- a/Lib/Objects/CollectObject.cs +++ b/Lib/Objects/CollectObject.cs @@ -38,9 +38,15 @@ public string RowKey { get { - return CryptoHelpers.CreateHash(Serialized); + if (string.IsNullOrEmpty(_rowKey)) + { + _rowKey = CryptoHelpers.CreateHash(Serialized); + } + return _rowKey; } } + + private string _rowKey = string.Empty; [SkipCompare] [JsonIgnore] @@ -50,7 +56,7 @@ public byte[] Serialized { if (_serialized == null) { - _serialized = JsonUtils.Dehydrate(this); + _serialized = ProtoBufUtils.Dehydrate(this); } return _serialized; diff --git a/Lib/Objects/WriteObject.cs b/Lib/Objects/WriteObject.cs index fbcaeed67..64f020da1 100644 --- a/Lib/Objects/WriteObject.cs +++ b/Lib/Objects/WriteObject.cs @@ -32,7 +32,7 @@ public string Identity public static WriteObject? FromString(byte[] SerializedIn, RESULT_TYPE ResultTypeIn, string RunIdIn) { - var deserialized = JsonUtils.Hydrate(SerializedIn, ResultTypeIn); + var deserialized = ProtoBufUtils.Hydrate(SerializedIn, ResultTypeIn); if (deserialized is CollectObject) { diff --git a/Lib/Utils/JsonUtils.cs b/Lib/Utils/ProtoBufUtils.cs similarity index 98% rename from Lib/Utils/JsonUtils.cs rename to Lib/Utils/ProtoBufUtils.cs index 60367860a..fb207ce60 100644 --- a/Lib/Utils/JsonUtils.cs +++ b/Lib/Utils/ProtoBufUtils.cs @@ -8,15 +8,13 @@ using System.Globalization; using System.IO; using System.Linq; -using System.Text; -using System.Text.Unicode; using Tpm2Lib; namespace Microsoft.CST.AttackSurfaceAnalyzer.Utils { - public static class JsonUtils + public static class ProtoBufUtils { - static JsonUtils() + static ProtoBufUtils() { TypeDescriptor.AddAttributes(typeof((TpmAlgId, uint)), new TypeConverterAttribute(typeof(TpmPcrTupleConverter))); } diff --git a/Lib/Utils/SqliteDatabaseManager.cs b/Lib/Utils/SqliteDatabaseManager.cs index 037973a09..98181728d 100644 --- a/Lib/Utils/SqliteDatabaseManager.cs +++ b/Lib/Utils/SqliteDatabaseManager.cs @@ -284,11 +284,11 @@ public override List GetComparisonResults(string? baseId, string { if (reader["first_serialized"] is byte[] first_serialized) { - compareResult.Base = JsonUtils.Hydrate(first_serialized, exportType); + compareResult.Base = ProtoBufUtils.Hydrate(first_serialized, exportType); } if (reader["second_serialized"] is byte[] second_serialized) { - compareResult.Compare = JsonUtils.Hydrate(second_serialized, exportType); + compareResult.Compare = ProtoBufUtils.Hydrate(second_serialized, exportType); } if (compareResult.Base is not null && compareResult.Compare is not null) { @@ -324,11 +324,11 @@ public override List GetComparisonResults(string? baseId, string? { if (reader["first_serialized"] is byte[] first_serialized) { - compareResult.Base = JsonUtils.Hydrate(first_serialized, resultType); + compareResult.Base = ProtoBufUtils.Hydrate(first_serialized, resultType); } if (reader["second_serialized"] is byte[] second_serialized) { - compareResult.Compare = JsonUtils.Hydrate(second_serialized, resultType); + compareResult.Compare = ProtoBufUtils.Hydrate(second_serialized, resultType); } if (compareResult.Base is not null && compareResult.Compare is not null) { @@ -1087,7 +1087,7 @@ public override void Write(CollectObject? colObj, string? runId) " from collect a indexed by i_collect_collect_runid_row_type," + " collect b indexed by i_collect_collect_runid_row_type" + " where a.run_id=@first_run_id and b.run_id=@second_run_id and a.identity = b.identity and " + - "a.row_key != b.row_key and a.result_type = b.result_type and a.serialized != b.serialized;"; + "a.result_type = b.result_type and a.row_key != b.row_key and a.serialized != b.serialized;"; private const string SQL_GET_NUM_RESULTS = "select count(*) as the_count from collect where run_id = @run_id and result_type = @result_type"; private const string SQL_GET_PERSISTED_SETTINGS = "select serialized from persisted_settings where id=@id"; diff --git a/Tests/HydrationTests.cs b/Tests/HydrationTests.cs index bb340164d..8a416fb48 100644 --- a/Tests/HydrationTests.cs +++ b/Tests/HydrationTests.cs @@ -22,7 +22,7 @@ public void TestSerializeAndDeserializeCertificateObject() { var co = new CertificateObject("StoreLocation", "StoreName", new SerializableCertificate("Thumbprint", "Subject", "PublicKey", DateTime.Now.AddYears(1), DateTime.Now, "Issuer", "SerialNumber", "CertHashString", "Pkcs7")); - if (JsonUtils.Hydrate(JsonUtils.Dehydrate(co), RESULT_TYPE.CERTIFICATE) is CertificateObject co2) + if (ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(co), RESULT_TYPE.CERTIFICATE) is CertificateObject co2) { Assert.IsTrue(co.RowKey.Equals(co2.RowKey)); Assert.IsTrue(co.Certificate.Thumbprint.Equals(co2.Certificate.Thumbprint)); @@ -38,7 +38,7 @@ public void TestSerializeAndDeserializeComObject() { var com = new ComObject(new RegistryObject("Test Key", Microsoft.Win32.RegistryView.Default)); - Assert.IsTrue(com.RowKey.Equals(JsonUtils.Hydrate(JsonUtils.Dehydrate(com), RESULT_TYPE.COM)?.RowKey)); + Assert.IsTrue(com.RowKey.Equals(ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(com), RESULT_TYPE.COM)?.RowKey)); } [TestMethod] @@ -46,7 +46,7 @@ public void TestSerializeAndDeserializeCryptographicKeyObject() { var cko = new CryptographicKeyObject("Disk", Tpm2Lib.TpmAlgId.Rsa) { RsaDetails = new RsaKeyDetails() }; - var hydrated = JsonUtils.Hydrate(JsonUtils.Dehydrate(cko), RESULT_TYPE.KEY); + var hydrated = ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(cko), RESULT_TYPE.KEY); Assert.IsTrue(cko.RowKey.Equals(hydrated.RowKey)); } @@ -54,9 +54,9 @@ public void TestSerializeAndDeserializeDriverObject() { var DriverName = "MyName"; var driverObject = new DriverObject(DriverName); - var serialized = JsonUtils.Dehydrate(driverObject); - var rehydrated = JsonUtils.Hydrate(serialized, RESULT_TYPE.DRIVER); - Assert.IsTrue(serialized == JsonUtils.Dehydrate(rehydrated)); + var serialized = ProtoBufUtils.Dehydrate(driverObject); + var rehydrated = ProtoBufUtils.Hydrate(serialized, RESULT_TYPE.DRIVER); + Assert.IsTrue(serialized == ProtoBufUtils.Dehydrate(rehydrated)); Assert.IsTrue(rehydrated.Identity == DriverName); } @@ -65,7 +65,7 @@ public void TestSerializeAndDeserializeEventLogObject() { var elo = new EventLogObject("Disk"); - Assert.IsTrue(elo.RowKey.Equals(JsonUtils.Hydrate(JsonUtils.Dehydrate(elo), RESULT_TYPE.LOG)?.RowKey)); + Assert.IsTrue(elo.RowKey.Equals(ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(elo), RESULT_TYPE.LOG)?.RowKey)); } [TestMethod] @@ -73,7 +73,7 @@ public void TestSerializeAndDeserializeFileSystemObject() { var fso = new FileSystemObject("Test"); - Assert.IsTrue(fso.RowKey.Equals(JsonUtils.Hydrate(JsonUtils.Dehydrate(fso), RESULT_TYPE.FILE)?.RowKey)); + Assert.IsTrue(fso.RowKey.Equals(ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(fso), RESULT_TYPE.FILE)?.RowKey)); } [TestMethod] @@ -81,7 +81,7 @@ public void TestSerializeAndDeserializeFirewallObject() { var fwo = new FirewallObject("Test"); - Assert.IsTrue(fwo.RowKey.Equals(JsonUtils.Hydrate(JsonUtils.Dehydrate(fwo), RESULT_TYPE.FIREWALL)?.RowKey)); + Assert.IsTrue(fwo.RowKey.Equals(ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(fwo), RESULT_TYPE.FIREWALL)?.RowKey)); } [TestMethod] @@ -89,7 +89,7 @@ public void TestSerializeAndDeserializeGroupAccountObject() { var ugo = new GroupAccountObject("TestGroup"); - Assert.IsTrue(ugo.RowKey.Equals(JsonUtils.Hydrate(JsonUtils.Dehydrate(ugo), RESULT_TYPE.GROUP)?.RowKey)); + Assert.IsTrue(ugo.RowKey.Equals(ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(ugo), RESULT_TYPE.GROUP)?.RowKey)); } [TestMethod] @@ -97,15 +97,15 @@ public void TestSerializeAndDeserializeOpenPortObject() { var opo = new OpenPortObject(1024, TRANSPORT.TCP); - Assert.IsTrue(opo.RowKey.Equals(JsonUtils.Hydrate(JsonUtils.Dehydrate(opo), RESULT_TYPE.PORT)?.RowKey)); + Assert.IsTrue(opo.RowKey.Equals(ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(opo), RESULT_TYPE.PORT)?.RowKey)); } [TestMethod] public void TestSerializeAndDeserializeProcessObject() { var po = ProcessObject.FromProcess(Process.GetCurrentProcess()); - var serialized = JsonUtils.Dehydrate(po); - Assert.IsTrue(po.RowKey.Equals(JsonUtils.Hydrate(serialized, RESULT_TYPE.PROCESS)?.RowKey)); + var serialized = ProtoBufUtils.Dehydrate(po); + Assert.IsTrue(po.RowKey.Equals(ProtoBufUtils.Hydrate(serialized, RESULT_TYPE.PROCESS)?.RowKey)); } [TestMethod] @@ -113,7 +113,7 @@ public void TestSerializeAndDeserializeRegistryObject() { var ro = new RegistryObject("Test Key", Microsoft.Win32.RegistryView.Default); - Assert.IsTrue(ro.RowKey.Equals(JsonUtils.Hydrate(JsonUtils.Dehydrate(ro), RESULT_TYPE.REGISTRY)?.RowKey)); + Assert.IsTrue(ro.RowKey.Equals(ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(ro), RESULT_TYPE.REGISTRY)?.RowKey)); } [TestMethod] @@ -121,7 +121,7 @@ public void TestSerializeAndDeserializeServiceObject() { var so = new ServiceObject("TestService"); - Assert.IsTrue(so.RowKey.Equals(JsonUtils.Hydrate(JsonUtils.Dehydrate(so), RESULT_TYPE.SERVICE)?.RowKey)); + Assert.IsTrue(so.RowKey.Equals(ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(so), RESULT_TYPE.SERVICE)?.RowKey)); } [TestMethod] @@ -129,7 +129,7 @@ public void TestSerializeAndDeserializeTpmObject() { var tpmo = new TpmObject("TestLocation"); - Assert.IsTrue(tpmo.RowKey.Equals(JsonUtils.Hydrate(JsonUtils.Dehydrate(tpmo), RESULT_TYPE.TPM)?.RowKey)); + Assert.IsTrue(tpmo.RowKey.Equals(ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(tpmo), RESULT_TYPE.TPM)?.RowKey)); } [TestMethod] @@ -137,7 +137,7 @@ public void TestSerializeAndDeserializeUserAccountObject() { var uao = new UserAccountObject("TestUser"); - Assert.IsTrue(uao.RowKey.Equals(JsonUtils.Hydrate(JsonUtils.Dehydrate(uao), RESULT_TYPE.USER)?.RowKey)); + Assert.IsTrue(uao.RowKey.Equals(ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(uao), RESULT_TYPE.USER)?.RowKey)); } } } \ No newline at end of file From 959e8316cc50c5fe61f3d625a1f8677fef01bfb9 Mon Sep 17 00:00:00 2001 From: Gabe Stocco <98900+gfs@users.noreply.github.com> Date: Tue, 2 Apr 2024 15:37:06 -0700 Subject: [PATCH 05/10] Add Equals method to Serializable Certificate --- Lib/Objects/SerializableCertificate.cs | 48 ++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/Lib/Objects/SerializableCertificate.cs b/Lib/Objects/SerializableCertificate.cs index 4310f108d..a76dae746 100644 --- a/Lib/Objects/SerializableCertificate.cs +++ b/Lib/Objects/SerializableCertificate.cs @@ -54,5 +54,53 @@ public SerializableCertificate(string Thumbprint, string Subject, string PublicK public string Subject { get; set; } [ProtoMember(9)] public string Thumbprint { get; set; } + + public override bool Equals(object? obj) + { + if(ReferenceEquals(this, obj)) return true; + if (obj is SerializableCertificate) return Equals((SerializableCertificate)obj); + return false; + } + + public bool Equals(SerializableCertificate other) + { + if (!CertHashString.Equals(other.CertHashString)) + { + return false; + } + if (!Issuer.Equals(other.Issuer)) + { + return false; + } + if (!NotAfter.Equals(other.NotAfter)) + { + return false; + } + if (!NotBefore.Equals(other.NotBefore)) + { + return false; + } + if (!Pkcs7.Equals(other.Pkcs7)) + { + return false; + } + if (!PublicKey.Equals(other.PublicKey)) + { + return false; + } + if (!SerialNumber.Equals(other.SerialNumber)) + { + return false; + } + if (!Subject.Equals(other.Subject)) + { + return false; + } + if (!Thumbprint.Equals(other.Thumbprint)) + { + return false; + } + return true; + } } } \ No newline at end of file From 991d408f902c75a629bbdb6cf911227e62dab4a7 Mon Sep 17 00:00:00 2001 From: Gabe Stocco <98900+gfs@users.noreply.github.com> Date: Tue, 2 Apr 2024 15:37:19 -0700 Subject: [PATCH 06/10] Update HydrationTests to use a shared compare logic --- Tests/HydrationTests.cs | 105 ++++++++++++++++++++++++++-------------- 1 file changed, 69 insertions(+), 36 deletions(-) diff --git a/Tests/HydrationTests.cs b/Tests/HydrationTests.cs index 8a416fb48..9183744fc 100644 --- a/Tests/HydrationTests.cs +++ b/Tests/HydrationTests.cs @@ -1,9 +1,14 @@ -using Microsoft.CST.AttackSurfaceAnalyzer.Objects; +using KellermanSoftware.CompareNetObjects; +using Microsoft.CST.AttackSurfaceAnalyzer.Objects; using Microsoft.CST.AttackSurfaceAnalyzer.Types; using Microsoft.CST.AttackSurfaceAnalyzer.Utils; using Microsoft.VisualStudio.TestTools.UnitTesting; +using ProtoBuf; +using ProtoBuf.Meta; using System; using System.Diagnostics; +using Tpm2Lib; +using System.Linq; namespace Microsoft.CST.AttackSurfaceAnalyzer.Tests { @@ -17,127 +22,155 @@ public static void ClassSetup(TestContext _) Strings.Setup(); } - [TestMethod] - public void TestSerializeAndDeserializeCertificateObject() + /// + /// Check that all Public Properties of the objects are Equal + /// + /// + /// + /// + /// + private bool AreAllPropsEqual(Type objectType, object obj1, object obj2) { - var co = new CertificateObject("StoreLocation", "StoreName", new SerializableCertificate("Thumbprint", "Subject", "PublicKey", DateTime.Now.AddYears(1), DateTime.Now, "Issuer", "SerialNumber", "CertHashString", "Pkcs7")); + foreach (var prop in objectType.GetProperties(System.Reflection.BindingFlags.Public)) + { + var val1 = prop.GetValue(obj1); + var val2 = prop.GetValue(obj2); + if (prop.GetType().IsSubclassOf(typeof(CollectObject))) + { + AreAllPropsEqual(prop.GetType(), val1, val2); + } + else if (prop.GetType().Equals(typeof(byte[]))) + { + Assert.IsTrue(((byte[])val1).SequenceEqual((byte[])val2)); + } + else + { + Assert.AreEqual(prop.GetValue(obj1), prop.GetValue(obj2)); + } + } + return true; + } - if (ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(co), RESULT_TYPE.CERTIFICATE) is CertificateObject co2) + /// + /// Validate that an object survives serialization and deserialization + /// + /// + /// + /// + private void ValidateHydrateDehydrateEquality(T one, RESULT_TYPE resultType) where T : CollectObject + { + // Check by Serializing and Deserializing using our own util + if (ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(one), resultType) is T two) { - Assert.IsTrue(co.RowKey.Equals(co2.RowKey)); - Assert.IsTrue(co.Certificate.Thumbprint.Equals(co2.Certificate.Thumbprint)); + Assert.IsTrue(AreAllPropsEqual(one.GetType(), one, two)); } else { Assert.Fail(); } + // Check with cloning + var copy = Serializer.DeepClone(one); + Assert.IsTrue(AreAllPropsEqual(one.GetType(), one, copy)); + } + + [TestMethod] + public void TestSerializeAndDeserializeCertificateObject() + { + var comparator = new CompareLogic(); + + var co = new CertificateObject("StoreLocation", "StoreName", new SerializableCertificate("Thumbprint", "Subject", "PublicKey", DateTime.Now.AddYears(1), DateTime.Now, "Issuer", "SerialNumber", "CertHashString", "Pkcs7")); + ValidateHydrateDehydrateEquality(co, RESULT_TYPE.CERTIFICATE); } [TestMethod] public void TestSerializeAndDeserializeComObject() { var com = new ComObject(new RegistryObject("Test Key", Microsoft.Win32.RegistryView.Default)); - - Assert.IsTrue(com.RowKey.Equals(ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(com), RESULT_TYPE.COM)?.RowKey)); + ValidateHydrateDehydrateEquality(com, RESULT_TYPE.COM); } [TestMethod] public void TestSerializeAndDeserializeCryptographicKeyObject() { var cko = new CryptographicKeyObject("Disk", Tpm2Lib.TpmAlgId.Rsa) { RsaDetails = new RsaKeyDetails() }; - - var hydrated = ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(cko), RESULT_TYPE.KEY); - Assert.IsTrue(cko.RowKey.Equals(hydrated.RowKey)); + ValidateHydrateDehydrateEquality(cko, RESULT_TYPE.KEY); } public void TestSerializeAndDeserializeDriverObject() { var DriverName = "MyName"; var driverObject = new DriverObject(DriverName); - var serialized = ProtoBufUtils.Dehydrate(driverObject); - var rehydrated = ProtoBufUtils.Hydrate(serialized, RESULT_TYPE.DRIVER); - Assert.IsTrue(serialized == ProtoBufUtils.Dehydrate(rehydrated)); - Assert.IsTrue(rehydrated.Identity == DriverName); + ValidateHydrateDehydrateEquality(driverObject, RESULT_TYPE.DRIVER); } [TestMethod] public void TestSerializeAndDeserializeEventLogObject() { var elo = new EventLogObject("Disk"); - - Assert.IsTrue(elo.RowKey.Equals(ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(elo), RESULT_TYPE.LOG)?.RowKey)); + ValidateHydrateDehydrateEquality(elo, RESULT_TYPE.LOG); } [TestMethod] public void TestSerializeAndDeserializeFileSystemObject() { var fso = new FileSystemObject("Test"); - - Assert.IsTrue(fso.RowKey.Equals(ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(fso), RESULT_TYPE.FILE)?.RowKey)); + ValidateHydrateDehydrateEquality(fso, RESULT_TYPE.FILE); } [TestMethod] public void TestSerializeAndDeserializeFirewallObject() { var fwo = new FirewallObject("Test"); - - Assert.IsTrue(fwo.RowKey.Equals(ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(fwo), RESULT_TYPE.FIREWALL)?.RowKey)); + ValidateHydrateDehydrateEquality(fwo, RESULT_TYPE.FIREWALL); } [TestMethod] public void TestSerializeAndDeserializeGroupAccountObject() { var ugo = new GroupAccountObject("TestGroup"); - - Assert.IsTrue(ugo.RowKey.Equals(ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(ugo), RESULT_TYPE.GROUP)?.RowKey)); + ValidateHydrateDehydrateEquality(ugo, RESULT_TYPE.GROUP); } [TestMethod] public void TestSerializeAndDeserializeOpenPortObject() { var opo = new OpenPortObject(1024, TRANSPORT.TCP); - - Assert.IsTrue(opo.RowKey.Equals(ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(opo), RESULT_TYPE.PORT)?.RowKey)); + ValidateHydrateDehydrateEquality(opo, RESULT_TYPE.PORT); } [TestMethod] public void TestSerializeAndDeserializeProcessObject() { var po = ProcessObject.FromProcess(Process.GetCurrentProcess()); - var serialized = ProtoBufUtils.Dehydrate(po); - Assert.IsTrue(po.RowKey.Equals(ProtoBufUtils.Hydrate(serialized, RESULT_TYPE.PROCESS)?.RowKey)); + ValidateHydrateDehydrateEquality(po, RESULT_TYPE.PROCESS); } [TestMethod] public void TestSerializeAndDeserializeRegistryObject() { var ro = new RegistryObject("Test Key", Microsoft.Win32.RegistryView.Default); - - Assert.IsTrue(ro.RowKey.Equals(ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(ro), RESULT_TYPE.REGISTRY)?.RowKey)); + ValidateHydrateDehydrateEquality(ro, RESULT_TYPE.REGISTRY); } [TestMethod] public void TestSerializeAndDeserializeServiceObject() { var so = new ServiceObject("TestService"); - - Assert.IsTrue(so.RowKey.Equals(ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(so), RESULT_TYPE.SERVICE)?.RowKey)); + ValidateHydrateDehydrateEquality(so, RESULT_TYPE.SERVICE); } [TestMethod] public void TestSerializeAndDeserializeTpmObject() { var tpmo = new TpmObject("TestLocation"); - - Assert.IsTrue(tpmo.RowKey.Equals(ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(tpmo), RESULT_TYPE.TPM)?.RowKey)); + ValidateHydrateDehydrateEquality(tpmo, RESULT_TYPE.TPM); } [TestMethod] public void TestSerializeAndDeserializeUserAccountObject() { var uao = new UserAccountObject("TestUser"); - - Assert.IsTrue(uao.RowKey.Equals(ProtoBufUtils.Hydrate(ProtoBufUtils.Dehydrate(uao), RESULT_TYPE.USER)?.RowKey)); + ValidateHydrateDehydrateEquality(uao, RESULT_TYPE.USER); } } } \ No newline at end of file From e6f0c19f9729d20d553a9e23de688148b846db9c Mon Sep 17 00:00:00 2001 From: Gabe Stocco <98900+gfs@users.noreply.github.com> Date: Tue, 2 Apr 2024 15:46:08 -0700 Subject: [PATCH 07/10] Comment Unneeded Hash Benchmarks --- Benchmarks/CryptoTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Benchmarks/CryptoTests.cs b/Benchmarks/CryptoTests.cs index 4e46a4c9d..707d6d3e1 100644 --- a/Benchmarks/CryptoTests.cs +++ b/Benchmarks/CryptoTests.cs @@ -29,7 +29,7 @@ public CryptoTests() [Params(1000)] public int ObjectPadding { get; set; } - [Benchmark] + //[Benchmark] public void Generate_N_Murmur_Hashes() { for (int i = 0; i < N; i++) @@ -65,7 +65,7 @@ public void Generate_N_SHA256_Hashes() } } - [Benchmark] + //[Benchmark] public void Generate_N_SHA256Managed_Hashes() { for (int i = 0; i < N; i++) @@ -101,7 +101,7 @@ public void Generate_N_SHA512_Hashes() } } - [Benchmark] + //[Benchmark] public void Generate_N_SHA512_Managed_Hashes() { for (int i = 0; i < N; i++) From 2617e1e795385e7bb47395250643b1f62c044863 Mon Sep 17 00:00:00 2001 From: Gabe Stocco <98900+gfs@users.noreply.github.com> Date: Tue, 2 Apr 2024 15:46:33 -0700 Subject: [PATCH 08/10] Switch back to SHA256 hashing --- Lib/Utils/CryptoHelpers.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/Utils/CryptoHelpers.cs b/Lib/Utils/CryptoHelpers.cs index 3ad9f27c8..a67244dd6 100644 --- a/Lib/Utils/CryptoHelpers.cs +++ b/Lib/Utils/CryptoHelpers.cs @@ -80,7 +80,7 @@ private static HashAlgorithm GetHasher() } else { - return SHA512.Create(); + return SHA256.Create(); } } From f27bb19862f062f793f639f7b1ba7431792969a6 Mon Sep 17 00:00:00 2001 From: Gabe Stocco <98900+gfs@users.noreply.github.com> Date: Tue, 2 Apr 2024 15:48:56 -0700 Subject: [PATCH 09/10] Delete launchSettings.json --- Cli/Properties/launchSettings.json | 37 ------------------------------ 1 file changed, 37 deletions(-) delete mode 100644 Cli/Properties/launchSettings.json diff --git a/Cli/Properties/launchSettings.json b/Cli/Properties/launchSettings.json deleted file mode 100644 index e892bfa79..000000000 --- a/Cli/Properties/launchSettings.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "iisSettings": { - "windowsAuthentication": false, - "anonymousAuthentication": true, - "iisExpress": { - "applicationUrl": "http://localhost:61357", - "sslPort": 44392 - } - }, - "profiles": { - "IIS Express": { - "commandName": "IISExpress", - "launchBrowser": true, - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - }, - "ASA GUI": { - "commandName": "Project", - "commandLineArgs": "gui --nolaunch", - "launchBrowser": true, - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - }, - "applicationUrl": "https://localhost:7280;http://localhost:5280", - "dotnetRunMessages": true - }, - "Export-collect last 2": { - "commandName": "Project", - "commandLineArgs": "export-collect" - }, - "Export-monitor": { - "commandName": "Project", - "commandLineArgs": "export-monitor" - } - } -} \ No newline at end of file From 8a251dd4e5abe494f61893ddcf31ee6108d71dac Mon Sep 17 00:00:00 2001 From: Gabe Stocco <98900+gfs@users.noreply.github.com> Date: Tue, 2 Apr 2024 16:20:15 -0700 Subject: [PATCH 10/10] Rename WriteObject.FromString to WriteObject.FromSerialized --- Benchmarks/SystemSqliteDatabaseManager.cs | 8 ++++---- Lib/Objects/WriteObject.cs | 2 +- Lib/Utils/SqliteDatabaseManager.cs | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Benchmarks/SystemSqliteDatabaseManager.cs b/Benchmarks/SystemSqliteDatabaseManager.cs index 04cfd20fe..473331ca0 100644 --- a/Benchmarks/SystemSqliteDatabaseManager.cs +++ b/Benchmarks/SystemSqliteDatabaseManager.cs @@ -225,7 +225,7 @@ public static ConcurrentBag GetMissingFromFirst(string firstRunId, var resultTypeString = reader["result_type"].ToString(); if (runId != null && resultTypeString != null) { - var wo = WriteObject.FromString((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); + var wo = WriteObject.FromSerialized((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); if (wo is WriteObject WO) { output.Add(WO); @@ -256,8 +256,8 @@ public static ConcurrentBag GetMissingFromFirst(string firstRunId, if (aRunId != null && bRunId != null && aResultType != null && bResultType != null) { - var val1 = WriteObject.FromString((byte[])reader["a_serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), aResultType), aRunId); - var val2 = WriteObject.FromString((byte[])reader["b_serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), bResultType), bRunId); + var val1 = WriteObject.FromSerialized((byte[])reader["a_serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), aResultType), aRunId); + var val2 = WriteObject.FromSerialized((byte[])reader["b_serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), bResultType), bRunId); if (val1 is WriteObject V1 && val2 is WriteObject V2) { @@ -389,7 +389,7 @@ public static IEnumerable GetResultsByRunid(string runid) var resultTypeString = reader["result_type"].ToString(); if (runId != null && resultTypeString != null) { - var val = WriteObject.FromString((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); + var val = WriteObject.FromSerialized((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); if (val is WriteObject valid) yield return valid; } diff --git a/Lib/Objects/WriteObject.cs b/Lib/Objects/WriteObject.cs index 64f020da1..03f50219f 100644 --- a/Lib/Objects/WriteObject.cs +++ b/Lib/Objects/WriteObject.cs @@ -30,7 +30,7 @@ public string Identity public string RunId { get; } public byte[] Serialized { get; } - public static WriteObject? FromString(byte[] SerializedIn, RESULT_TYPE ResultTypeIn, string RunIdIn) + public static WriteObject? FromSerialized(byte[] SerializedIn, RESULT_TYPE ResultTypeIn, string RunIdIn) { var deserialized = ProtoBufUtils.Hydrate(SerializedIn, ResultTypeIn); diff --git a/Lib/Utils/SqliteDatabaseManager.cs b/Lib/Utils/SqliteDatabaseManager.cs index 98181728d..ebf69751d 100644 --- a/Lib/Utils/SqliteDatabaseManager.cs +++ b/Lib/Utils/SqliteDatabaseManager.cs @@ -179,7 +179,7 @@ public override IEnumerable GetAllMissing(string? firstRunId, strin var resultTypeString = reader["result_type"].ToString(); if (runId != null && resultTypeString != null) { - var wo = WriteObject.FromString((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); + var wo = WriteObject.FromSerialized((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); if (wo is WriteObject WO) output.Enqueue(WO); } @@ -207,7 +207,7 @@ public IEnumerable GetAllMissing2(string firstRunId, string secondR var resultTypeString = reader["result_type"].ToString(); if (runId != null && resultTypeString != null) { - var wo = WriteObject.FromString((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); + var wo = WriteObject.FromSerialized((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); if (wo is WriteObject WO) output.Enqueue(WO); } @@ -233,7 +233,7 @@ public IEnumerable GetAllMissingExplicit(string firstRunId, string var resultTypeString = reader["result_type"].ToString(); if (runId != null && resultTypeString != null) { - var wo = WriteObject.FromString((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); + var wo = WriteObject.FromSerialized((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); if (wo is WriteObject WO) output.Enqueue(WO); } @@ -415,7 +415,7 @@ public override IEnumerable GetMissingFromFirst(string firstRunId, var resultTypeString = reader["result_type"].ToString(); if (runId != null && resultTypeString != null) { - var wo = WriteObject.FromString((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); + var wo = WriteObject.FromSerialized((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); if (wo is WriteObject WO) output.Enqueue(WO); } @@ -446,8 +446,8 @@ public override IEnumerable GetMissingFromFirst(string firstRunId, { if (reader["a_serialized"] is byte[] a_serialized && reader["b_serialized"] is byte[] b_serialized) { - var val1 = WriteObject.FromString(a_serialized, (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), aResultType), aRunId); - var val2 = WriteObject.FromString(b_serialized, (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), bResultType), bRunId); + var val1 = WriteObject.FromSerialized(a_serialized, (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), aResultType), aRunId); + var val2 = WriteObject.FromSerialized(b_serialized, (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), bResultType), bRunId); if (val1 is WriteObject V1 && val2 is WriteObject V2) { @@ -585,7 +585,7 @@ public override IEnumerable GetResultsByRunid(string runid) var resultTypeString = reader["result_type"].ToString(); if (runId != null && resultTypeString != null) { - var wo = WriteObject.FromString((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); + var wo = WriteObject.FromSerialized((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId); if (wo is WriteObject WO) { yield return WO;