Reconcole Release 1.0 to main (#12)

* Update CreateTemplate.ps1

* Fix Meta Data Sync Issue

* Fixed Sync Issue that Impacts all clients

* Sync Troubleshooting

* Simplified Syncing process

* New Templates Added

* Update SampleConfig.json

* Jsonupdates (#11)

* Template Install Updates for New Templates

Author: fiddlermikey

.github/workflows/keyfactor-starter-workflow.yml

@@ -5,13 +5,26 @@ jobs:
   call-create-github-release-workflow:
     uses: Keyfactor/actions/.github/workflows/github-release.yml@main
 
+  get-manifest-properties:
+    runs-on: windows-latest
+    outputs:
+      update_catalog: ${{ steps.read-json.outputs.prop }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Read json
+        id: read-json
+        shell: pwsh
+        run: |
+          $json = Get-Content integration-manifest.json | ConvertFrom-Json
+          echo "::set-output name=prop::$(echo $json.update_catalog)"
+
   call-dotnet-build-and-release-workflow:
     needs: [call-create-github-release-workflow]
     uses: Keyfactor/actions/.github/workflows/dotnet-build-and-release.yml@main
     with:
       release_version: ${{ needs.call-create-github-release-workflow.outputs.release_version }}
       release_url: ${{ needs.call-create-github-release-workflow.outputs.release_url }}
-      release_dir: CscGlobalCaProxy/bin/Release # TODO: set build output directory to upload as a release, relative to checkout workspace
+      release_dir: CscGlobalCaProxy/bin/Release
     secrets: 
       token: ${{ secrets.PRIVATE_PACKAGE_ACCESS }}
 
@@ -22,7 +35,8 @@ jobs:
       token: ${{ secrets.APPROVE_README_PUSH }}
 
   call-update-catalog-workflow:
-    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
+    needs: get-manifest-properties
+    if: needs.get-manifest-properties.outputs.update_catalog == 'True' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
     uses: Keyfactor/actions/.github/workflows/update-catalog.yml@main
     secrets: 
       token: ${{ secrets.SDK_SYNC_PAT }}

CHANGELOG.md

@@ -1,2 +1,14 @@
+v1.0.10
+- Sync Issue where Sync only works after service restart fixed
+- Sync Fix when errors occur in the CSC Api so next sync works
+- Support for CSC TrustedSecure Domain Validated SSL (new CSC Template)
+- Support for CSC TrustedSecure Domain Validated Wildcard SSL (new CSC Template)
+- Support for CSC TrustedSecure Domain Validated UC Certificate (new CSC Template)
+- Install support for new templates.  Setup only no code changed in Gateway.
+
+v1.0.9
 - Updated Readme to go from Powershell based install to Manual Install With More Detailed Instructions
 - Removed unneeded config items
+- Fixed Meta Data Sync Issue
+- For Patch Installation for V1.0.9, see bottom of Readme File
+

CscGlobalCaProxy/Client/CscGlobalClient.cs

@@ -1,15 +1,12 @@
 using System;
-using System.Collections.Concurrent;
 using System.Net;
 using System.Net.Http;
 using System.Net.Http.Headers;
 using System.Text;
-using System.Threading;
 using System.Threading.Tasks;
 using CAProxy.AnyGateway.Interfaces;
 using CSS.Common.Logging;
 using Keyfactor.AnyGateway.CscGlobal.Client.Models;
-using Keyfactor.AnyGateway.CscGlobal.Exceptions;
 using Keyfactor.AnyGateway.CscGlobal.Interfaces;
 using Newtonsoft.Json;
 
@@ -30,7 +27,6 @@ public CscGlobalClient(ICAConnectorConfigProvider config)
 
         private Uri BaseUrl { get; }
         private HttpClient RestClient { get; }
-        private int PageSize { get; } = 100;
         private string ApiKey { get; }
         private string Authorization { get; }
 
@@ -41,7 +37,7 @@ public async Task<RegistrationResponse> SubmitRegistrationAsync(
                 JsonConvert.SerializeObject(registerRequest), Encoding.ASCII, "application/json")))
             {
                 Logger.Trace(JsonConvert.SerializeObject(registerRequest));
-                var settings = new JsonSerializerSettings {NullValueHandling = NullValueHandling.Ignore};
+                var settings = new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore };
                 if (resp.StatusCode == HttpStatusCode.BadRequest) //Csc Sends Errors back in 400 Json Response
                 {
                     var errorResponse =
@@ -68,7 +64,7 @@ public async Task<RenewalResponse> SubmitRenewalAsync(
             {
                 Logger.Trace(JsonConvert.SerializeObject(renewalRequest));
 
-                var settings = new JsonSerializerSettings {NullValueHandling = NullValueHandling.Ignore};
+                var settings = new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore };
                 if (resp.StatusCode == HttpStatusCode.BadRequest) //Csc Sends Errors back in 400 Json Response
                 {
                     var errorResponse =
@@ -94,7 +90,7 @@ public async Task<ReissueResponse> SubmitReissueAsync(
             {
                 Logger.Trace(JsonConvert.SerializeObject(reissueRequest));
 
-                var settings = new JsonSerializerSettings {NullValueHandling = NullValueHandling.Ignore};
+                var settings = new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore };
                 if (resp.StatusCode == HttpStatusCode.BadRequest) //Csc Sends Errors back in 400 Json Response
                 {
                     var errorResponse =
@@ -127,7 +123,7 @@ public async Task<RevokeResponse> SubmitRevokeCertificateAsync(string uuId)
         {
             using (var resp = await RestClient.PutAsync($"/dbs/api/v2/tls/revoke/{uuId}", new StringContent("")))
             {
-                var settings = new JsonSerializerSettings {NullValueHandling = NullValueHandling.Ignore};
+                var settings = new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore };
                 if (resp.StatusCode == HttpStatusCode.BadRequest) //Csc Sends Errors back in 400 Json Response
                 {
                     var errorResponse =
@@ -145,92 +141,27 @@ public async Task<RevokeResponse> SubmitRevokeCertificateAsync(string uuId)
             }
         }
 
-        public async Task SubmitCertificateListRequestAsync(BlockingCollection<ICertificateResponse> bc,
-            CancellationToken ct)
+        public async Task<CertificateListResponse> SubmitCertificateListRequestAsync()
         {
             Logger.MethodEntry(ILogExtensions.MethodLogLevel.Debug);
-            try
-            {
-                var itemsProcessed = 0;
-                var isComplete = false;
-                var retryCount = 0;
-                do
-                {
-                    var batchItemsProcessed = 0;
-                    using (var resp = await RestClient.GetAsync("/dbs/api/v2/tls/certificate?filter=status=in=(ACTIVE,REVOKED)", ct))
-                    {
-                        if (!resp.IsSuccessStatusCode)
-                        {
-                            var responseMessage = resp.Content.ReadAsStringAsync().Result;
-                            Logger.Error(
-                                $"Failed Request to Keyfactor. Retrying request. Status Code {resp.StatusCode} | Message: {responseMessage}");
-                            retryCount++;
-                            if (retryCount > 5)
-                                throw new RetryCountExceededException(
-                                    $"5 consecutive failures to {resp.RequestMessage.RequestUri}");
-
-                            continue;
-                        }
-
-                        var stringResponse = await resp.Content.ReadAsStringAsync();
-
-                        var batchResponse =
-                            JsonConvert.DeserializeObject<CertificateListResponse>(stringResponse);
-
-                        var batchCount = batchResponse.Results.Count;
-
-                        Logger.Trace($"Processing {batchCount} items in batch");
-                        do
-                        {
-                            var r = batchResponse.Results[batchItemsProcessed];
-                            if (bc.TryAdd(r, 10, ct))
-                            {
-                                Logger.Trace($"Added Template ID {r.Uuid} to Queue for processing");
-                                batchItemsProcessed++;
-                                itemsProcessed++;
-                                Logger.Trace($"Processed {batchItemsProcessed} of {batchCount}");
-                                Logger.Trace($"Total Items Processed: {itemsProcessed}");
-                            }
-                            else
-                            {
-                                Logger.Trace($"Adding {r} blocked. Retry");
-                            }
-                        } while (batchItemsProcessed < batchCount); //batch loop
-                    }
-
-                    //assume that if we process less records than requested that we have reached the end of the certificate list
-                    if (batchItemsProcessed < PageSize)
-                        isComplete = true;
-                } while (!isComplete); //page loop
-
-                bc.CompleteAdding();
-            }
-            catch (OperationCanceledException cancelEx)
-            {
-                Logger.Warn($"Synchronize method was cancelled. Message: {cancelEx.Message}");
-                bc.CompleteAdding();
-                Logger.MethodExit(ILogExtensions.MethodLogLevel.Debug);
-                // ReSharper disable once PossibleIntendedRethrow
-                throw cancelEx;
-            }
-            catch (RetryCountExceededException retryEx)
-            {
-                Logger.Error($"Retries Failed: {retryEx.Message}");
-                Logger.MethodExit(ILogExtensions.MethodLogLevel.Debug);
-            }
-            catch (HttpRequestException ex)
+            var resp = RestClient.GetAsync("/dbs/api/v2/tls/certificate?filter=status=in=(ACTIVE,REVOKED)").Result;
+
+            if (!resp.IsSuccessStatusCode)
             {
-                Logger.Error($"HttpRequest Failed: {ex.Message}");
-                Logger.MethodExit(ILogExtensions.MethodLogLevel.Debug);
+                var responseMessage = resp.Content.ReadAsStringAsync().Result;
+                Logger.Error(
+                    $"Failed Request to Keyfactor. Retrying request. Status Code {resp.StatusCode} | Message: {responseMessage}");
             }
 
-            Logger.MethodExit(ILogExtensions.MethodLogLevel.Debug);
+            var certificateListResponse =
+                JsonConvert.DeserializeObject<CertificateListResponse>(await resp.Content.ReadAsStringAsync());
+            return certificateListResponse;
         }
 
         private HttpClient ConfigureRestClient()
         {
             var clientHandler = new WebRequestHandler();
-            var returnClient = new HttpClient(clientHandler, true) {BaseAddress = BaseUrl};
+            var returnClient = new HttpClient(clientHandler, true) { BaseAddress = BaseUrl };
             returnClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
             returnClient.DefaultRequestHeaders.Add("Authorization", "Bearer " + Authorization);
             returnClient.DefaultRequestHeaders.Add("apikey", ApiKey);

CscGlobalCaProxy/CscGlobalCaProxy.cs

@@ -34,25 +34,23 @@ public CscGlobalCaProxy()
         public override int Revoke(string caRequestId, string hexSerialNumber, uint revocationReason)
         {
 
-                Logger.Trace($"Staring Revoke Method");
-                var revokeResponse =
-                    Task.Run(async () =>
-                            await CscGlobalClient.SubmitRevokeCertificateAsync(caRequestId.Substring(0,36)))
-                        .Result; //todo fix to use pipe delimiter
+            Logger.Trace($"Staring Revoke Method");
+            var revokeResponse =
+                Task.Run(async () =>
+                        await CscGlobalClient.SubmitRevokeCertificateAsync(caRequestId.Substring(0, 36)))
+                    .Result; //todo fix to use pipe delimiter
 
-                Logger.Trace($"Revoke Response JSON: {JsonConvert.SerializeObject(revokeResponse)}");
-                Logger.MethodExit(ILogExtensions.MethodLogLevel.Debug);
-                
-                var revokeResult=_requestManager.GetRevokeResult(revokeResponse);
+            Logger.Trace($"Revoke Response JSON: {JsonConvert.SerializeObject(revokeResponse)}");
+            Logger.MethodExit(ILogExtensions.MethodLogLevel.Debug);
 
-                if(revokeResult== Convert.ToInt32(PKIConstants.Microsoft.RequestDisposition.FAILED))
-                {
-                    return -1;
-                }
-                else
-                {
-                    return revokeResult;
-                }
+            var revokeResult = _requestManager.GetRevokeResult(revokeResponse);
+
+            if (revokeResult == Convert.ToInt32(PKIConstants.Microsoft.RequestDisposition.FAILED))
+            {
+                return -1;
+            }
+
+            return revokeResult;
 
         }
 
@@ -73,19 +71,14 @@ public override void Synchronize(ICertificateDataReader certificateDataReader,
             Logger.MethodEntry(ILogExtensions.MethodLogLevel.Debug);
             try
             {
-                var certs = new BlockingCollection<ICertificateResponse>(100);
-                CscGlobalClient.SubmitCertificateListRequestAsync(certs, cancelToken);
-
-                foreach (var currentResponseItem in certs.GetConsumingEnumerable(cancelToken))
+                if (certificateAuthoritySyncInfo.DoFullSync)
                 {
-                    if (cancelToken.IsCancellationRequested)
-                    {
-                        Logger.Error("Synchronize was canceled.");
-                        break;
-                    }
+                    var certs = Task.Run(async () => await CscGlobalClient.SubmitCertificateListRequestAsync()).Result;
 
-                    try
+                    foreach (var currentResponseItem in certs.Results)
                     {
+
+                        cancelToken.ThrowIfCancellationRequested();
                         Logger.Trace($"Took Certificate ID {currentResponseItem?.Uuid} from Queue");
                         var certStatus = _requestManager.MapReturnStatus(currentResponseItem?.Status);
 
@@ -105,46 +98,37 @@ public override void Synchronize(ICertificateDataReader certificateDataReader,
                             {
                                 var certData = fileContent.Replace("\r\n", string.Empty);
                                 var splitCerts =
-                                    certData.Split(new[] {"-----END CERTIFICATE-----", "-----BEGIN CERTIFICATE-----"},
+                                    certData.Split(new[] { "-----END CERTIFICATE-----", "-----BEGIN CERTIFICATE-----" },
                                         StringSplitOptions.RemoveEmptyEntries);
                                 foreach (var cert in splitCerts)
                                     if (!cert.Contains(".crt"))
                                     {
                                         Logger.Trace($"Split Cert Value: {cert}");
 
                                         var currentCert = new X509Certificate2(Encoding.ASCII.GetBytes(cert));
-                                        if (!currentCert.Subject.Contains("AAA Certificate Services") &&
-                                            !currentCert.Subject.Contains("USERTrust RSA Certification Authority") &&
-                                            !currentCert.Subject.Contains("Trusted Secure Certificate Authority 5") && 
-                                            !currentCert.Subject.Contains("AddTrust External CA Root") &&
-                                            !currentCert.Subject.Contains("Trusted Secure Certificate Authority DV"))
-                                            blockingBuffer.Add(new CAConnectorCertificate
-                                            {
-                                                CARequestID =$"{currentResponseItem?.Uuid}-{currentCert.SerialNumber}",
-                                                Certificate = cert,
-                                                SubmissionDate = currentResponseItem?.OrderDate == null
-                                                    ? Convert.ToDateTime(currentCert.NotBefore)
-                                                    : Convert.ToDateTime(currentResponseItem.OrderDate),
-                                                Status = certStatus,
-                                                ProductID = productId
-                                            }, cancelToken);
+                                        blockingBuffer.Add(new CAConnectorCertificate
+                                        {
+                                            CARequestID = $"{currentResponseItem?.Uuid}",
+                                            Certificate = cert,
+                                            SubmissionDate = currentResponseItem?.OrderDate == null
+                                                ? Convert.ToDateTime(currentCert.NotBefore)
+                                                : Convert.ToDateTime(currentResponseItem.OrderDate),
+                                            Status = certStatus,
+                                            ProductID = productId
+                                        }, cancelToken);
                                     }
                             }
                         }
                     }
-                    catch (OperationCanceledException)
-                    {
-                        Logger.Error("Synchronize was canceled.");
-                        break;
-                    }
+                    blockingBuffer.CompleteAdding();
                 }
             }
-            catch (AggregateException aggEx)
+            catch (Exception e)
             {
-                Logger.Error("Csc Global Synchronize Task failed!");
+                Logger.Error($"Csc Global Synchronize Task failed! {LogHandler.FlattenException(e)}");
                 Logger.MethodExit(ILogExtensions.MethodLogLevel.Debug);
-                // ReSharper disable once PossibleIntendedRethrow
-                throw aggEx;
+                blockingBuffer.CompleteAdding();
+                throw;
             }
 
             Logger.MethodExit(ILogExtensions.MethodLogLevel.Debug);
@@ -198,7 +182,7 @@ public override EnrollmentResult Enroll(ICertificateDataReader certificateDataRe
                 case RequestUtilities.EnrollmentType.Renew:
                     Logger.Trace($"Entering Renew Enrollment");
                     //One click won't work for this implementation b/c we are missing enrollment params
-                    if (productInfo.ProductParameters.ContainsKey("Applicant Last Name")) 
+                    if (productInfo.ProductParameters.ContainsKey("Applicant Last Name"))
                     {
                         priorCert = certificateDataReader.GetCertificateRecord(
                             DataConversion.HexToBytes(productInfo.ProductParameters["PriorCertSN"]));
@@ -251,7 +235,7 @@ public override EnrollmentResult Enroll(ICertificateDataReader certificateDataRe
             Logger.MethodExit(ILogExtensions.MethodLogLevel.Debug);
             return null;
         }
-        
+
 
         public override CAConnectorCertificate GetSingleRecord(string caRequestId)
         {

CscGlobalCaProxy/CscGlobalCaProxy.csproj

@@ -83,9 +83,9 @@
   <ItemGroup>
     <Compile Include="Client\CscGlobalClient.cs" />
     <Compile Include="Client\Models\RevokeSuccessResponse.cs" />
+    <Compile Include="Interfaces\ICscGlobalClient.cs" />
     <Compile Include="Interfaces\IRegistrationError.cs" />
     <Compile Include="Client\Models\RegistrationError.cs" />
-    <Compile Include="Interfaces\ICscGlobalClient.cs" />
     <Compile Include="Constants.cs" />
     <Compile Include="CscGlobalCaProxy.cs" />
     <Compile Include="Client\Models\CertificateListResponse.cs" />

CscGlobalCaProxy/Interfaces/ICscGlobalClient.cs

@@ -18,7 +18,7 @@ Task<ReissueResponse> SubmitReissueAsync(
 
         Task<CertificateResponse> SubmitGetCertificateAsync(string certificateId);
 
-        Task SubmitCertificateListRequestAsync(BlockingCollection<ICertificateResponse> bc, CancellationToken ct);
+        Task<CertificateListResponse> SubmitCertificateListRequestAsync();
 
         Task<RevokeResponse> SubmitRevokeCertificateAsync(string uuId);
     }