From 6eaad6a394fd59050711359cc88d6ebac9d83526 Mon Sep 17 00:00:00 2001 From: Olli Janatuinen Date: Fri, 29 Jul 2022 01:49:33 -0700 Subject: [PATCH 1/4] Convert to .NET 6.0, support running as Linux container --- Dockerfile | 28 ++++++++ ExportPipelineDefinitions/App.config | 24 ------- .../ExportPipelineDefinitions.csproj | 70 ++----------------- ExportPipelineDefinitions/Program.cs | 21 +++--- .../Properties/AssemblyInfo.cs | 4 +- .../Properties/Settings.Designer.cs | 53 -------------- .../Properties/Settings.settings | 17 ----- ExportPipelineDefinitions/packages.config | 5 -- README.md | 35 ++++++++-- 9 files changed, 77 insertions(+), 180 deletions(-) create mode 100644 Dockerfile delete mode 100644 ExportPipelineDefinitions/App.config delete mode 100644 ExportPipelineDefinitions/Properties/Settings.Designer.cs delete mode 100644 ExportPipelineDefinitions/Properties/Settings.settings delete mode 100644 ExportPipelineDefinitions/packages.config diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..617aec0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +# Stage 1: https://hub.docker.com/_/microsoft-dotnet +FROM mcr.microsoft.com/dotnet/aspnet:6.0 AS base +ENV outputPath=/output/data/ +WORKDIR /app +ENTRYPOINT ["dotnet", "ExportPipelineDefinitions.dll"] + +# Disable .NET diagnostics so app does no need write permissions to /tmp +ENV COMPlus_EnableDiagnostics=0 + + +# Stage 2: Build application with SDK +FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build +WORKDIR /source + +# copy csproj and restore as distinct layers +COPY *.sln . +COPY ExportPipelineDefinitions/*.csproj ./ExportPipelineDefinitions/ +RUN dotnet restore + +# copy everything else and build app +COPY ExportPipelineDefinitions/. ./ExportPipelineDefinitions/ +WORKDIR /source/ExportPipelineDefinitions +RUN dotnet publish -o /app --no-restore + + +# Stage 3: Copy binaries to hardened runtime image +FROM base AS final +COPY --from=build /app ./ diff --git a/ExportPipelineDefinitions/App.config b/ExportPipelineDefinitions/App.config deleted file mode 100644 index f2e508c..0000000 --- a/ExportPipelineDefinitions/App.config +++ /dev/null @@ -1,24 +0,0 @@ - - - - -
- - - - - - - - - C:\temp\PipelineDefinitions\ - - - - - - - - - - \ No newline at end of file diff --git a/ExportPipelineDefinitions/ExportPipelineDefinitions.csproj b/ExportPipelineDefinitions/ExportPipelineDefinitions.csproj index f50a05e..7cda654 100644 --- a/ExportPipelineDefinitions/ExportPipelineDefinitions.csproj +++ b/ExportPipelineDefinitions/ExportPipelineDefinitions.csproj @@ -1,70 +1,12 @@ - - - + - Debug - AnyCPU - {AB417819-7751-4B6B-95C5-0A127625DC6E} + net6.0 Exe - ExportPipelineDefinitions - ExportPipelineDefinitions - v4.6.1 - 512 - true - true + false - - AnyCPU - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - AnyCPU - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - - ..\packages\JSON.1.0.1\lib\net40\Json.dll - - - ..\packages\Newtonsoft.Json.12.0.2\lib\net45\Newtonsoft.Json.dll - - - - - - - - - - - - - - - True - True - Settings.settings - - - - - - SettingsSingleFileGenerator - Settings.Designer.cs - Designer - + + + - \ No newline at end of file diff --git a/ExportPipelineDefinitions/Program.cs b/ExportPipelineDefinitions/Program.cs index 95883ba..6d2b957 100644 --- a/ExportPipelineDefinitions/Program.cs +++ b/ExportPipelineDefinitions/Program.cs @@ -122,18 +122,17 @@ static void Main(string[] args) if (Directory.Exists(outputPath)) { BuildCsvString(1, "Writing to column 1 flushes the csvColumns buffer to csvString"); - File.WriteAllText(outputPath + @"\BuildDefinitions.csv", csvString); + File.WriteAllText(outputPath + Path.DirectorySeparatorChar + @"BuildDefinitions.csv", csvString); } - Console.WriteLine("Done. Press any key"); - Console.ReadKey(); + Console.WriteLine("Done."); } public static void GetSettings() { - personalAccessToken = Properties.Settings.Default.personalAccessToken; - organization = Properties.Settings.Default.organization; - outputPath = Properties.Settings.Default.outputPath; + personalAccessToken = Environment.GetEnvironmentVariable("personalAccessToken"); + organization = Environment.GetEnvironmentVariable("organization"); + outputPath = Environment.GetEnvironmentVariable("outputPath"); Validate(nameof(personalAccessToken), personalAccessToken); Validate(nameof(organization), organization); @@ -146,7 +145,7 @@ public static void Validate(string nameOfVar, string value) { throw new InvalidDataException( string.Format( - "Invalid {0} value in file {1}.config.", + "Invalid {0} value in environment variable", nameOfVar, System.Diagnostics.Process.GetCurrentProcess().MainModule.FileName)); } @@ -313,17 +312,17 @@ public static async Task WriteDefinitionToFile(string project, string definition // Write json to a file string fileContent = json.ToString(); - string directory = outputPath + project + "\\" + definitionType + "s\\" + buildDef.path; + string directory = outputPath + project + Path.DirectorySeparatorChar + definitionType + "s" + Path.DirectorySeparatorChar + buildDef.path; + directory = directory.Replace("/\\", "/"); System.IO.Directory.CreateDirectory(directory); string buildName = buildDef.name.Replace("?", "").Replace(":", ""); if (isYamlPipeline) { - directory += "\\" + buildName; + directory += Path.DirectorySeparatorChar + buildName; System.IO.Directory.CreateDirectory(directory); } - directory = directory.Replace("\\\\", "\\").Replace("\\\\", "\\"); - string fullFilePath = directory + "\\" + buildDef.name.Replace("?", "").Replace(":", "") + ".json"; + string fullFilePath = directory + Path.DirectorySeparatorChar + buildDef.name.Replace("?", "").Replace(":", "") + ".json"; System.IO.File.WriteAllText(fullFilePath, fileContent); if (isYamlPipeline) { diff --git a/ExportPipelineDefinitions/Properties/AssemblyInfo.cs b/ExportPipelineDefinitions/Properties/AssemblyInfo.cs index 043e4b2..2532aef 100644 --- a/ExportPipelineDefinitions/Properties/AssemblyInfo.cs +++ b/ExportPipelineDefinitions/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.1.0.0")] -[assembly: AssemblyFileVersion("1.1.0.0")] +[assembly: AssemblyVersion("1.2.0.0")] +[assembly: AssemblyFileVersion("1.2.0.0")] diff --git a/ExportPipelineDefinitions/Properties/Settings.Designer.cs b/ExportPipelineDefinitions/Properties/Settings.Designer.cs deleted file mode 100644 index f6543ea..0000000 --- a/ExportPipelineDefinitions/Properties/Settings.Designer.cs +++ /dev/null @@ -1,53 +0,0 @@ -//------------------------------------------------------------------------------ -// -// This code was generated by a tool. -// Runtime Version:4.0.30319.42000 -// -// Changes to this file may cause incorrect behavior and will be lost if -// the code is regenerated. -// -//------------------------------------------------------------------------------ - -namespace ExportPipelineDefinitions.Properties { - - - [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.Editors.SettingsDesigner.SettingsSingleFileGenerator", "16.2.0.0")] - internal sealed partial class Settings : global::System.Configuration.ApplicationSettingsBase { - - private static Settings defaultInstance = ((Settings)(global::System.Configuration.ApplicationSettingsBase.Synchronized(new Settings()))); - - public static Settings Default { - get { - return defaultInstance; - } - } - - [global::System.Configuration.ApplicationScopedSettingAttribute()] - [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] - [global::System.Configuration.DefaultSettingValueAttribute("C:\\temp\\PipelineDefinitions\\")] - public string outputPath { - get { - return ((string)(this["outputPath"])); - } - } - - [global::System.Configuration.ApplicationScopedSettingAttribute()] - [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] - [global::System.Configuration.DefaultSettingValueAttribute("")] - public string personalAccessToken { - get { - return ((string)(this["personalAccessToken"])); - } - } - - [global::System.Configuration.ApplicationScopedSettingAttribute()] - [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] - [global::System.Configuration.DefaultSettingValueAttribute("")] - public string organization { - get { - return ((string)(this["organization"])); - } - } - } -} diff --git a/ExportPipelineDefinitions/Properties/Settings.settings b/ExportPipelineDefinitions/Properties/Settings.settings deleted file mode 100644 index 96e367e..0000000 --- a/ExportPipelineDefinitions/Properties/Settings.settings +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - C:\temp\PipelineDefinitions\ - - - - - - - - - - - \ No newline at end of file diff --git a/ExportPipelineDefinitions/packages.config b/ExportPipelineDefinitions/packages.config deleted file mode 100644 index 062a3c8..0000000 --- a/ExportPipelineDefinitions/packages.config +++ /dev/null @@ -1,5 +0,0 @@ - - - - - \ No newline at end of file diff --git a/README.md b/README.md index 19df1e9..3934262 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,34 @@ If you manage a large number of ADO pipelines, **you need this app.** Note: The exported .json files are not in a format that can be imported using the Azure DevOps UI. ## Quickstart -1. Go to [releases](../../releases). Click-to-download the .exe, .config, and .dll files. Put them together in a folder. -1. Edit the .config file and set the three values: personalAccessToken, organization, and outputPath. - See details in [ExportPipelineDefinitions/Program.cs](https://github.com/BruceHaley/ExportPipelineDefinitions/blob/51792ed245a4c62cadb4707ed62960c6d959102f/ExportPipelineDefinitions/Program.cs#L30), lines 30 - 32. -1. Run ExportPipelineDefinitions.exe. +Prerequirements: +1. Create PAT token with following scopes: +* Build: Read +* Project and Team: Read +* Release: Read + +### On Windows +1. Go to [releases](../../releases). Click-to-download the .zip file and extract it to folder. +2. Start PowerShell and run commands: +```powershell +$env:personalAccessToken = "" +$env:organization = "" +$env:outputPath = "C:\temp\PipelineDefinitions\" +.\ExportPipelineDefinitions.exe +``` + +### On Linux container +1. Clone this GIT repository +2. Build container image `docker build . -t export-pipeline-definitions` +3. Run tool as container: +```bash +mkdir output +docker run -it --rm \ + --user $(id -u):$(id -g) \ + --cap-drop ALL \ + --read-only \ + -e personalAccessToken="" \ + -e organization="" \ + -v $(pwd)/output:/output \ + export-pipeline-definitions +``` From 475a583c5d68b5b9d9bdb2dc58b644824edaf64b Mon Sep 17 00:00:00 2001 From: Olli Janatuinen Date: Fri, 29 Jul 2022 05:38:30 -0700 Subject: [PATCH 2/4] Support Azure DevOps YAML pipelines download --- ExportPipelineDefinitions/Program.cs | 66 ++++++++++++++++++++++++---- README.md | 1 + 2 files changed, 58 insertions(+), 9 deletions(-) diff --git a/ExportPipelineDefinitions/Program.cs b/ExportPipelineDefinitions/Program.cs index 6d2b957..f67fa8e 100644 --- a/ExportPipelineDefinitions/Program.cs +++ b/ExportPipelineDefinitions/Program.cs @@ -46,6 +46,7 @@ public class BuildDef public int id; public string name; public string path; + public string type; } static List definitionList = new List(); @@ -224,6 +225,7 @@ public static async Task PopulateBuildDefinitionList(string project = "SDK_v4") bd.id = int.Parse(o["id"].ToString()); bd.name = o["name"].ToString(); bd.path = o["path"].ToString(); + bd.type = o.SelectToken("repository.type").ToString(); definitionList.Add(bd); } } @@ -328,7 +330,7 @@ public static async Task WriteDefinitionToFile(string project, string definition { string saveDirectory = Directory.GetCurrentDirectory(); //Directory.SetCurrentDirectory(directory); - DownloadYamlFilesToDirectory(json, directory); + DownloadYamlFilesToDirectory(json, directory, project, buildDef.type); //Directory.SetCurrentDirectory(saveDirectory); } } @@ -354,7 +356,7 @@ public static bool CheckForYamlPipeline(JObject json) return false; } - public static void DownloadYamlFilesToDirectory(JObject json, string directory) + public static void DownloadYamlFilesToDirectory(JObject json, string directory, string project, string type) { // Download the .yml files from the GitHub repo. // Get GitHub repo URL for this Azure project. @@ -418,15 +420,61 @@ public static void DownloadYamlFilesToDirectory(JObject json, string directory) logIndent = " "; // 8 spaces } string fileName = githubFileUrl.Substring(githubFileUrl.LastIndexOf('/') + 1); - string targetFullFilePath = $"{directory}\\{fileName}"; - bool succeeded = DownloadFileFromGithub(githubFileUrl, targetFullFilePath, logIndent); - yamlFileNames.RemoveAt(0); - if (succeeded) + string targetFullFilePath = directory + Path.DirectorySeparatorChar + fileName; + + if (type == "TfsGit") + { + string restUrl = ""; + try { + // targetFullFilePath = $"{directory}" + Path.DirectorySeparatorChar + json["process"]["yamlFilename"]; + restUrl = String.Format("https://{0}/{1}/{2}/_apis/git/repositories/{3}/items?path={4}&download=true&api-version=5.0", domain, organization, project, json["repository"]["properties"]["fullName"], filePath); + + using (HttpClient client = new HttpClient()) + { + client.DefaultRequestHeaders.Accept.Add( + new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue("application/octet-stream")); + + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", + Convert.ToBase64String( + System.Text.ASCIIEncoding.ASCII.GetBytes( + string.Format("{0}:{1}", "", personalAccessToken)))); + + var webRequest = new HttpRequestMessage(HttpMethod.Get, restUrl); + using (HttpResponseMessage response = client.Send(webRequest)) + { + response.EnsureSuccessStatusCode(); + using var reader = new StreamReader(response.Content.ReadAsStream()); + string responseBody = reader.ReadToEnd(); + + using (StreamWriter outputFile = new StreamWriter(targetFullFilePath)) + { + outputFile.Write(responseBody); + } + } + } + int count = yamlFileNames.Count; + yamlFileNames.AddRange(GetYamlTemplateReferencesFromFile(targetFullFilePath)); + } + catch (Exception ex) + { + Console.WriteLine(ex.ToString()); + Console.WriteLine(restUrl); + } + + // There can be only one Azure DevOps YAML file per pipeline so we can break here. + break; + } + else { - int count = yamlFileNames.Count; - yamlFileNames.AddRange(GetYamlTemplateReferencesFromFile(targetFullFilePath)); + bool succeeded = DownloadFileFromGithub(githubFileUrl, targetFullFilePath, logIndent); + yamlFileNames.RemoveAt(0); + if (succeeded) + { + int count = yamlFileNames.Count; + yamlFileNames.AddRange(GetYamlTemplateReferencesFromFile(targetFullFilePath)); + } + githubFileUrl = string.Empty; } - githubFileUrl = string.Empty; } // Read .yml file from repo. // Look for template references to other .yml files. Add any found to yamlFileNames list. diff --git a/README.md b/README.md index 3934262..dc3eb13 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ Note: The exported .json files are not in a format that can be imported using th Prerequirements: 1. Create PAT token with following scopes: * Build: Read +* Code: Read * Project and Team: Read * Release: Read From e1a73fadb822211eefc0b3f621c73187d289f983 Mon Sep 17 00:00:00 2001 From: Olli Janatuinen Date: Mon, 1 Aug 2022 00:29:21 -0700 Subject: [PATCH 3/4] Correctly handle renamed Git repos and build pipelines in subfolders --- ExportPipelineDefinitions/Program.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ExportPipelineDefinitions/Program.cs b/ExportPipelineDefinitions/Program.cs index f67fa8e..1bb208e 100644 --- a/ExportPipelineDefinitions/Program.cs +++ b/ExportPipelineDefinitions/Program.cs @@ -10,6 +10,7 @@ using System.Text; using System.Threading.Tasks; using System.Net; +using System.Runtime.InteropServices; namespace ExportPipelineDefinitions { @@ -316,6 +317,10 @@ public static async Task WriteDefinitionToFile(string project, string definition string directory = outputPath + project + Path.DirectorySeparatorChar + definitionType + "s" + Path.DirectorySeparatorChar + buildDef.path; directory = directory.Replace("/\\", "/"); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + directory = directory.Replace("\\", "/"); + } System.IO.Directory.CreateDirectory(directory); string buildName = buildDef.name.Replace("?", "").Replace(":", ""); if (isYamlPipeline) @@ -426,8 +431,7 @@ public static void DownloadYamlFilesToDirectory(JObject json, string directory, { string restUrl = ""; try { - // targetFullFilePath = $"{directory}" + Path.DirectorySeparatorChar + json["process"]["yamlFilename"]; - restUrl = String.Format("https://{0}/{1}/{2}/_apis/git/repositories/{3}/items?path={4}&download=true&api-version=5.0", domain, organization, project, json["repository"]["properties"]["fullName"], filePath); + restUrl = String.Format("https://{0}/{1}/{2}/_apis/git/repositories/{3}/items?path={4}&download=true&api-version=5.0", domain, organization, project, json["repository"]["name"], filePath); using (HttpClient client = new HttpClient()) { From 32a012fe32b5b7c62ad8c4a751ea541845451a83 Mon Sep 17 00:00:00 2001 From: Olli Janatuinen Date: Mon, 1 Aug 2022 04:49:19 -0700 Subject: [PATCH 4/4] Download fully rendered version of Azure DevOps Yaml pipelines --- ExportPipelineDefinitions/Program.cs | 37 +++++++++++++++++----------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/ExportPipelineDefinitions/Program.cs b/ExportPipelineDefinitions/Program.cs index 1bb208e..790bca1 100644 --- a/ExportPipelineDefinitions/Program.cs +++ b/ExportPipelineDefinitions/Program.cs @@ -431,33 +431,42 @@ public static void DownloadYamlFilesToDirectory(JObject json, string directory, { string restUrl = ""; try { - restUrl = String.Format("https://{0}/{1}/{2}/_apis/git/repositories/{3}/items?path={4}&download=true&api-version=5.0", domain, organization, project, json["repository"]["name"], filePath); + restUrl = String.Format("https://{0}/{1}/{2}/_apis/pipelines/{3}/preview?api-version=7.1-preview.1", domain, organization, project, json["id"]); using (HttpClient client = new HttpClient()) { client.DefaultRequestHeaders.Accept.Add( - new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue("application/octet-stream")); + new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue("application/json")); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String( System.Text.ASCIIEncoding.ASCII.GetBytes( string.Format("{0}:{1}", "", personalAccessToken)))); - var webRequest = new HttpRequestMessage(HttpMethod.Get, restUrl); - using (HttpResponseMessage response = client.Send(webRequest)) + var requestContent = new StringContent("{\"previewRun\":true}", Encoding.UTF8, "application/json"); + var webRequest = new HttpRequestMessage(HttpMethod.Post, restUrl) + { + Content = requestContent + }; + using (HttpResponseMessage response = client.Send(webRequest)) + { + response.EnsureSuccessStatusCode(); + using var reader = new StreamReader(response.Content.ReadAsStream()); + string responseBody = reader.ReadToEnd(); + if (responseBody.Contains("!DOCTYPE")) { - response.EnsureSuccessStatusCode(); - using var reader = new StreamReader(response.Content.ReadAsStream()); - string responseBody = reader.ReadToEnd(); - - using (StreamWriter outputFile = new StreamWriter(targetFullFilePath)) - { - outputFile.Write(responseBody); - } + throw new AccessViolationException( + string.Format( + "Cannot access Azure. Please ensure personalAccessToken and organization values are correct in file {0}.config.", + System.Diagnostics.Process.GetCurrentProcess().MainModule.FileName)); } + JObject jsonNew = JObject.Parse(responseBody); + using (StreamWriter outputFile = new StreamWriter(targetFullFilePath)) + { + outputFile.Write(jsonNew["finalYaml"].ToString()); + } + } } - int count = yamlFileNames.Count; - yamlFileNames.AddRange(GetYamlTemplateReferencesFromFile(targetFullFilePath)); } catch (Exception ex) {