Serialize Objects as Protobuf Instead of JSON

.gitignore

@@ -271,3 +271,4 @@ asa.sqlite
 *asa.log.txt
 *.sqlite
 *.litedb
+Cli/Properties/launchSettings.json

Benchmarks/CryptoTests.cs

@@ -29,7 +29,7 @@ public CryptoTests()
         [Params(1000)]
         public int ObjectPadding { get; set; }
 
-        [Benchmark]
+        //[Benchmark]
         public void Generate_N_Murmur_Hashes()
         {
             for (int i = 0; i < N; i++)
@@ -65,7 +65,7 @@ public void Generate_N_SHA256_Hashes()
             }
         }
 
-        [Benchmark]
+        //[Benchmark]
         public void Generate_N_SHA256Managed_Hashes()
         {
             for (int i = 0; i < N; i++)
@@ -101,7 +101,7 @@ public void Generate_N_SHA512_Hashes()
             }
         }
 
-        [Benchmark]
+        //[Benchmark]
         public void Generate_N_SHA512_Managed_Hashes()
         {
             for (int i = 0; i < N; i++)

Benchmarks/Program.cs

@@ -6,7 +6,7 @@ public class Program
     {
         public static void Main(string[] args)
         {
-            var summary = BenchmarkRunner.Run<CryptoTests>();
+            var summary = BenchmarkRunner.Run<QueryTests>();
         }
     }
 }

Benchmarks/SystemSqliteDatabaseManager.cs

@@ -225,7 +225,7 @@ public static ConcurrentBag<WriteObject> GetMissingFromFirst(string firstRunId,
                     var resultTypeString = reader["result_type"].ToString();
                     if (runId != null && resultTypeString != null)
                     {
-                        var wo = WriteObject.FromString((string)reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId);
+                        var wo = WriteObject.FromSerialized((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId);
                         if (wo is WriteObject WO)
                         {
                             output.Add(WO);
@@ -256,8 +256,8 @@ public static ConcurrentBag<WriteObject> GetMissingFromFirst(string firstRunId,
 
                     if (aRunId != null && bRunId != null && aResultType != null && bResultType != null)
                     {
-                        var val1 = WriteObject.FromString((string)reader["a_serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), aResultType), aRunId);
-                        var val2 = WriteObject.FromString((string)reader["b_serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), bResultType), bRunId);
+                        var val1 = WriteObject.FromSerialized((byte[])reader["a_serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), aResultType), aRunId);
+                        var val2 = WriteObject.FromSerialized((byte[])reader["b_serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), bResultType), bRunId);
 
                         if (val1 is WriteObject V1 && val2 is WriteObject V2)
                         {
@@ -389,7 +389,7 @@ public static IEnumerable<WriteObject> GetResultsByRunid(string runid)
                     var resultTypeString = reader["result_type"].ToString();
                     if (runId != null && resultTypeString != null)
                     {
-                        var val = WriteObject.FromString((string)reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId);
+                        var val = WriteObject.FromSerialized((byte[])reader["serialized"], (RESULT_TYPE)Enum.Parse(typeof(RESULT_TYPE), resultTypeString), runId);
                         if (val is WriteObject valid)
                             yield return valid;
                     }

Lib/Lib.csproj

@@ -44,6 +44,7 @@
     <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
     <PackageReference Include="Microsoft.Windows.Compatibility" Version="8.0.1" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="protobuf-net" Version="3.2.30" />
     <PackageReference Include="Serilog" Version="3.1.1" />
     <PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
     <PackageReference Include="Serilog.Sinks.File" Version="5.0.0" />

Lib/Objects/AsaNvIndex.cs

@@ -1,10 +1,13 @@
-using System.Collections.Generic;
+using ProtoBuf;
+using System.Collections.Generic;
 using Tpm2Lib;
 
 namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects
 {
+    [ProtoContract]
     public class AsaNvIndex
     {
+        [ProtoMember(1)]
         public NvAttr Attributes { get; set; }
 
         // These are all derived properties of the NvAttr flags. Separating them like this allows analysis
@@ -17,6 +20,7 @@ public class AsaNvIndex
         public bool Counter { get { return Attributes.HasFlag(NvAttr.Counter); } }
         public bool Extend { get { return Attributes.HasFlag(NvAttr.Extend); } }
         public bool GlobalLock { get { return Attributes.HasFlag(NvAttr.Globallock); } }
+        [ProtoMember(2)]
         public uint Index { get; set; }
         public bool NoDa { get { return Attributes.HasFlag(NvAttr.NoDa); } }
         public bool None { get { return Attributes.HasFlag(NvAttr.None); } }
@@ -41,6 +45,7 @@ public class AsaNvIndex
         public bool TpmNtBitLength { get { return Attributes.HasFlag(NvAttr.TpmNtBitLength); } }
         public bool TpmNtBitMask { get { return Attributes.HasFlag(NvAttr.TpmNtBitMask); } }
         public bool TpmNtBitOffset { get { return Attributes.HasFlag(NvAttr.TpmNtBitOffset); } }
+        [ProtoMember(3)]
         public List<byte>? value { get; set; }
         public bool Writeall { get { return Attributes.HasFlag(NvAttr.Writeall); } }
         public bool Writedefine { get { return Attributes.HasFlag(NvAttr.Writedefine); } }

Lib/Objects/CertificateObject.cs

@@ -1,8 +1,10 @@
 // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License.
 using Microsoft.CST.AttackSurfaceAnalyzer.Types;
+using ProtoBuf;
 
 namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects
 {
+    [ProtoContract]
     public class CertificateObject : CollectObject
     {
         public CertificateObject(string StoreLocation, string StoreName, SerializableCertificate Certificate)
@@ -11,11 +13,13 @@ public CertificateObject(string StoreLocation, string StoreName, SerializableCer
             this.StoreName = StoreName;
             this.Certificate = Certificate;
         }
+        public CertificateObject() { }
         public override RESULT_TYPE ResultType => RESULT_TYPE.CERTIFICATE;
 
         /// <summary>
         ///     A serializable representation of the Certificate.
         /// </summary>
+        [ProtoMember(1)]
         public SerializableCertificate Certificate { get; set; }
 
         /// <summary>
@@ -43,11 +47,13 @@ public override string Identity
         /// <summary>
         ///     The Store Location or Location on Disk where the Certificate was found
         /// </summary>
+        [ProtoMember(2)]
         public string StoreLocation { get; set; }
 
         /// <summary>
         ///     The Name of an X509 Store or another source (like the filesystem)
         /// </summary>
+        [ProtoMember(3)]
         public string StoreName { get; set; }
 
         /// <summary>

Lib/Objects/CollectObject.cs

@@ -2,13 +2,31 @@
 using Microsoft.CST.AttackSurfaceAnalyzer.Types;
 using Microsoft.CST.AttackSurfaceAnalyzer.Utils;
 using Newtonsoft.Json;
+using ProtoBuf;
 using System.Globalization;
 
 namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects
 {
     /// <summary>
     ///     Abstract parent class that all Collected data inherits from.
     /// </summary>
+    [ProtoContract]
+    [ProtoInclude(1, typeof(CertificateObject))]
+    [ProtoInclude(2, typeof(ComObject))]
+    [ProtoInclude(3, typeof(CryptographicKeyObject))]
+    [ProtoInclude(4, typeof(DriverObject))]
+    [ProtoInclude(5, typeof(EventLogObject))]
+    [ProtoInclude(6, typeof(FileMonitorObject))]
+    [ProtoInclude(7, typeof(FileSystemObject))]
+    [ProtoInclude(8, typeof(FirewallObject))]
+    [ProtoInclude(9, typeof(OpenPortObject))]
+    [ProtoInclude(10, typeof(ProcessObject))]
+    [ProtoInclude(11, typeof(RegistryObject))]
+    [ProtoInclude(12, typeof(ServiceObject))]
+    [ProtoInclude(13, typeof(TpmObject))]
+    [ProtoInclude(14, typeof(UserAccountObject))]
+    [ProtoInclude(15, typeof(GroupAccountObject))]
+    [ProtoInclude(16, typeof(WifiObject))]
     public abstract class CollectObject
     {
         public abstract string Identity { get; }
@@ -20,25 +38,31 @@ public string RowKey
         {
             get
             {
-                return Serialized.GetHashCode().ToString(CultureInfo.InvariantCulture);
+                if (string.IsNullOrEmpty(_rowKey))
+                {
+                    _rowKey = CryptoHelpers.CreateHash(Serialized);
+                }
+                return _rowKey;
             }
         }
+
+        private string _rowKey = string.Empty;
 
         [SkipCompare]
         [JsonIgnore]
-        public string Serialized
+        public byte[] Serialized
         {
             get
             {
                 if (_serialized == null)
                 {
-                    _serialized = JsonUtils.Dehydrate(this);
+                    _serialized = ProtoBufUtils.Dehydrate(this);
                 }
 
                 return _serialized;
             }
         }
 
-        private string? _serialized = null;
+        private byte[]? _serialized = null;
     }
 }

Lib/Objects/ComObject.cs

@@ -1,8 +1,10 @@
 // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License.
 using Microsoft.CST.AttackSurfaceAnalyzer.Types;
+using ProtoBuf;
 
 namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects
 {
+    [ProtoContract(SkipConstructor = true)]
     public class ComObject : CollectObject
     {
         /// <summary>
@@ -29,16 +31,19 @@ public override string Identity
         /// <summary>
         ///     The Registry Key which specifies this COM object
         /// </summary>
+        [ProtoMember(1)]
         public RegistryObject Key { get; set; }
 
         /// <summary>
         ///     The associated binary found (if any) in the x64 view of the registry
         /// </summary>
+        [ProtoMember(2)]
         public FileSystemObject? x64_Binary { get; set; }
 
         /// <summary>
         ///     The associated binary found (if any) in the x86 view of the registry
         /// </summary>
+        [ProtoMember(3)]
         public FileSystemObject? x86_Binary { get; set; }
     }
 }

Lib/Objects/CryptographicKeyObject.cs

@@ -2,13 +2,15 @@
 
 using Microsoft.CST.AttackSurfaceAnalyzer.Types;
 using Newtonsoft.Json;
+using ProtoBuf;
 using Serilog;
 using System;
 using System.Security.Cryptography;
 using Tpm2Lib;
 
 namespace Microsoft.CST.AttackSurfaceAnalyzer.Objects
 {
+    [ProtoContract(SkipConstructor = true)]
     public class CryptographicKeyObject : CollectObject
     {
         public CryptographicKeyObject(string Source, TpmAlgId tpmAlgId)
@@ -27,16 +29,20 @@ public override string Identity
             }
         }
 
+        [ProtoMember(1)]
         public RsaKeyDetails? RsaDetails { get; set; }
+        [ProtoMember(2)]
         public string Source { get; set; }
-
+        [ProtoMember(3)]
         public TpmAlgId tpmAlgId { get; set; } = TpmAlgId.Null;
     }
 
+    [ProtoContract]
     public class KeyDetailObject
     {
     }
 
+    [ProtoContract(SkipConstructor = true)]
     public class RsaKeyDetails : KeyDetailObject
     {
         public RsaKeyDetails(byte[] modulus, byte[] d, byte[]? p = null, byte[]? q = null)
@@ -82,7 +88,6 @@ public RsaKeyDetails(string? PublicString = null, string? FullString = null)
                 Log.Debug(e, "Failed to import RSA key.");
             }
         }
-
         public string? FullString
         {
             get