From cf0dff6927848127f9a0392a69558e8c90a956de Mon Sep 17 00:00:00 2001 From: Marcello de Sales Date: Wed, 1 May 2019 14:55:15 -0700 Subject: [PATCH 01/12] Issue #8: Python3: converting using 2to3 validator Just ran the following command: 2to3 -n -W --add-suffix=3 validate_config_files.py A new file was created validate_config_files.py3 and I just replaced the existing with this version. https://docs.python.org/2/library/2to3.html Signed-off-by: Marcello de Sales --- validate_config_files.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/validate_config_files.py b/validate_config_files.py index 18c5f8f..a8dd741 100755 --- a/validate_config_files.py +++ b/validate_config_files.py @@ -32,7 +32,7 @@ def isOnTestCases(): * python -m unittest discover -v tests """ # http://stackoverflow.com/questions/4858100/how-to-list-imported-modules/4858123#4858123 - return 'unittest' in sys.modules.keys() or 'tests.sys' in sys.modules.keys() + return 'unittest' in list(sys.modules.keys()) or 'tests.sys' in list(sys.modules.keys()) @staticmethod def getCurrentDirPath(): @@ -56,7 +56,7 @@ def getCurrentDirPath(): # If the execution is on github if not ExecutionContext.isOnGithub(): - print "=> Validating repo " + currentDirPath + print("=> Validating repo " + currentDirPath) else: # https://help.github.com/enterprise/2.6/admin/guides/developer-workflow/creating-a-pre-receive-hook-script/#writing-a-pre-receive-hook-script @@ -72,20 +72,20 @@ def getCurrentDirPath(): # Deleting a branch should NOT validate anything... skipping... if "0000000" in commit: - print "Deleting branch... Skip validation" + print("Deleting branch... Skip validation") exit(0) if "0000000" in base: - print "Validating new branch..." + print("Validating new branch...") - print "Processing commit=" + commit + " ref=" + ref + print("Processing commit=" + commit + " ref=" + ref) currentDirPath = Validator.processPreReceivehookFilesInGithub(base, commit) if "0000000" not in base: - print "=> Validating " + base + ".." + commit + print("=> Validating " + base + ".." + commit) else: - print "=> Validating SHA " + commit + print("=> Validating SHA " + commit) return currentDirPath @@ -269,7 +269,7 @@ def listAllConfigFiles(dirPath): # Valid configuration files configMatches = ["**/*.json", "**/*.yaml", "**/*.yml", "**/*.properties"] - print "Filtering Spring Cloud Config Server's files: ", configMatches + print("Filtering Spring Cloud Config Server's files: ", configMatches) # Get all the types config files based on the matches. allConfigs = [] @@ -313,9 +313,9 @@ def run(dirPath = None): """Runs the validation on a given directory, printing the report about each file verified""" # Starting the process - print "#####################################################" - print "#### Intuit Spring Cloud Config Validator " + VERSION + " #####" - print "#####################################################" + print("#####################################################") + print("#### Intuit Spring Cloud Config Validator " + VERSION + " #####") + print("#####################################################") #for key in os.environ.keys(): # print "%30s %s \n" % (key,os.environ[key]) @@ -332,15 +332,15 @@ def explain(currentDirPath, validationIndex): noErrors = True # Iterate over the index of the verifications - for filePath, isValid in validationIndex.iteritems(): + for filePath, isValid in validationIndex.items(): filePath = filePath if not ExecutionContext.isOnGithub() else str.replace(filePath, currentDirPath + "/", "") if isValid == True: - print "(v) File " + filePath + " is valid!" + print("(v) File " + filePath + " is valid!") else: isValid = isValid if not ExecutionContext.isOnGithub() else str.replace(str(isValid), currentDirPath + "/", "") # Only when we are running in github - print "(x) File " + filePath + " is invalid: " + str(isValid) + print("(x) File " + filePath + " is invalid: " + str(isValid)) noErrors = False # Exist with the value for errors From 062b2ce41b3117a4a01f7a66a49eb4a22b8e0382 Mon Sep 17 00:00:00 2001 From: Marcello de Sales Date: Wed, 1 May 2019 14:57:43 -0700 Subject: [PATCH 02/12] Issue #8: Dockerfile: Using pip instead of pip2: use python3 Signed-off-by: Marcello de Sales --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3a730c1..95f721f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,13 +4,13 @@ FROM marcellodesales/github-enterprise-prereceive-hook-base as tests # Install dependencies RUN apk add --no-cache py-pip && \ - pip2 install coverage + pip install coverage # Make a cache-eligible dependencies COPY requirements.txt /build/requirements.txt # Install dependencies -RUN pip2 install -r /build/requirements.txt +RUN pip install -r /build/requirements.txt # Copy resources COPY ./tests /build/tests @@ -25,6 +25,6 @@ FROM marcellodesales/github-enterprise-prereceive-hook-base as runtime RUN apk add --no-cache py-pip COPY --from=tests /build/requirements.txt requirements.txt -RUN pip2 install -r requirements.txt +RUN pip install -r requirements.txt COPY --from=tests /build/validate_config_files.py /home/git/test.git/hooks/pre-receive From 85fd3638f306eee2c03d54499e97444d03724602 Mon Sep 17 00:00:00 2001 From: Marcello de Sales Date: Wed, 1 May 2019 23:50:00 -0700 Subject: [PATCH 03/12] Issue #8: Tests: using 2to3 migration of test cases Just minor changes $ 2to3 --output-dir=tests3 -W -n tests WARNING: --write-unchanged-files/-W implies -w. lib2to3.main: Output in 'tests3' will mirror the input directory 'tests' layout. RefactoringTool: Skipping optional fixer: buffer RefactoringTool: Skipping optional fixer: idioms RefactoringTool: Skipping optional fixer: set_literal RefactoringTool: Skipping optional fixer: ws_comma RefactoringTool: Refactored tests/context.py --- tests/context.py (original) +++ tests/context.py (refactored) @@ -43,12 +43,12 @@ message = str(isValid) if isValid else str(isValid) + " ERROR: " + str(validationObject) # The V of successful in green - v = ShellColor.OKGREEN + str(u'\u2714'.encode('UTF-8')) + v = ShellColor.OKGREEN + str('\u2714'.encode('UTF-8')) # The X of failure in red - x = ShellColor.FAIL + str(u'\u2718'.encode('UTF-8')) + x = ShellColor.FAIL + str('\u2718'.encode('UTF-8')) shellStatus = v if isValid else x - print shellStatus + " is " + getRelativeFixturePath(filePath) + " valid? " + message + ShellColor.ENDC + print(shellStatus + " is " + getRelativeFixturePath(filePath) + " valid? " + message + ShellColor.ENDC) class ValidationAssertions: """Validation Assertions for the test cases to validate on the values.""" RefactoringTool: Writing converted tests/context.py to tests3/context.py. RefactoringTool: Refactored tests/test_all_valid_config_validation.py --- tests/test_all_valid_config_validation.py (original) +++ tests/test_all_valid_config_validation.py (refactored) @@ -26,8 +26,8 @@ self.assertTrue(len(self.validationIndex) > 0) def test_all_properties_are_valid(self): - print ShellColor.WARNING + "All config files are valid" + ShellColor.ENDC - for filePath, validationObject in self.validationIndex.iteritems(): + print(ShellColor.WARNING + "All config files are valid" + ShellColor.ENDC) + for filePath, validationObject in self.validationIndex.items(): printFileValidationStatus(filePath, validationObject) # Verify if the directory is in the file path RefactoringTool: Writing converted tests/test_all_valid_config_validation.py to tests3/test_all_valid_config_validation.py. RefactoringTool: Refactored tests/test_invalid_matrix_json_validation.py --- tests/test_invalid_matrix_json_validation.py (original) +++ tests/test_invalid_matrix_json_validation.py (refactored) @@ -26,8 +26,8 @@ self.assertTrue(len(self.validationIndex) > 0) def test_all_matrix_json_files_are_invalid(self): - print ShellColor.WARNING + "The android matrix file is invalid" + ShellColor.ENDC - for filePath, validationObject in self.validationIndex.iteritems(): + print(ShellColor.WARNING + "The android matrix file is invalid" + ShellColor.ENDC) + for filePath, validationObject in self.validationIndex.items(): isValid = isConfigValid(validationObject) printFileValidationStatus(filePath, validationObject) RefactoringTool: Writing converted tests/test_invalid_matrix_json_validation.py to tests3/test_invalid_matrix_json_validation.py. RefactoringTool: Refactored tests/test_invalid_properties_validation.py --- tests/test_invalid_properties_validation.py (original) +++ tests/test_invalid_properties_validation.py (refactored) @@ -26,8 +26,8 @@ self.assertTrue(len(self.validationIndex) > 0) def test_some_yaml_yml_files_are_invalid(self): - print ShellColor.WARNING + "Properties files are invalid without associated values" + ShellColor.ENDC - for filePath, validationObject in self.validationIndex.iteritems(): + print(ShellColor.WARNING + "Properties files are invalid without associated values" + ShellColor.ENDC) + for filePath, validationObject in self.validationIndex.items(): isValid = isConfigValid(validationObject) printFileValidationStatus(filePath, validationObject) RefactoringTool: Writing converted tests/test_invalid_properties_validation.py to tests3/test_invalid_properties_validation.py. RefactoringTool: Refactored tests/test_invalid_yaml_yml_duplicate_keys.py --- tests/test_invalid_yaml_yml_duplicate_keys.py (original) +++ tests/test_invalid_yaml_yml_duplicate_keys.py (refactored) @@ -26,8 +26,8 @@ self.assertTrue(len(self.validationIndex) > 0) def test_some_yaml_yml_files_are_invalid(self): - print ShellColor.WARNING + "Some Yaml Single documents are invalid" + ShellColor.ENDC - for filePath, validationObject in self.validationIndex.iteritems(): + print(ShellColor.WARNING + "Some Yaml Single documents are invalid" + ShellColor.ENDC) + for filePath, validationObject in self.validationIndex.items(): isValid = isConfigValid(validationObject) printFileValidationStatus(filePath, validationObject) RefactoringTool: Writing converted tests/test_invalid_yaml_yml_duplicate_keys.py to tests3/test_invalid_yaml_yml_duplicate_keys.py. RefactoringTool: Refactored tests/test_invalid_yaml_yml_multi_document_validation.py --- tests/test_invalid_yaml_yml_multi_document_validation.py (original) +++ tests/test_invalid_yaml_yml_multi_document_validation.py (refactored) @@ -26,8 +26,8 @@ self.assertTrue(len(self.validationIndex) > 0) def test_some_yaml_yml_files_are_invalid(self): - print ShellColor.WARNING + "Some Yaml Multi documents are invalid" + ShellColor.ENDC - for filePath, validationObject in self.validationIndex.iteritems(): + print(ShellColor.WARNING + "Some Yaml Multi documents are invalid" + ShellColor.ENDC) + for filePath, validationObject in self.validationIndex.items(): isValid = isConfigValid(validationObject) printFileValidationStatus(filePath, validationObject) RefactoringTool: Writing converted tests/test_invalid_yaml_yml_multi_document_validation.py to tests3/test_invalid_yaml_yml_multi_document_validation.py. RefactoringTool: Refactored tests/test_invalid_yaml_yml_single_document_validation.py --- tests/test_invalid_yaml_yml_single_document_validation.py (original) +++ tests/test_invalid_yaml_yml_single_document_validation.py (refactored) @@ -26,8 +26,8 @@ self.assertTrue(len(self.validationIndex) > 0) def test_some_yaml_yml_files_are_invalid(self): - print ShellColor.WARNING + "Some Yaml Single documents are invalid" + ShellColor.ENDC - for filePath, validationObject in self.validationIndex.iteritems(): + print(ShellColor.WARNING + "Some Yaml Single documents are invalid" + ShellColor.ENDC) + for filePath, validationObject in self.validationIndex.items(): isValid = isConfigValid(validationObject) printFileValidationStatus(filePath, validationObject) RefactoringTool: Writing converted tests/test_invalid_yaml_yml_single_document_validation.py to tests3/test_invalid_yaml_yml_single_document_validation.py. RefactoringTool: Files that were modified: RefactoringTool: tests/context.py RefactoringTool: tests/test_all_valid_config_validation.py RefactoringTool: tests/test_invalid_matrix_json_validation.py RefactoringTool: tests/test_invalid_properties_validation.py RefactoringTool: tests/test_invalid_yaml_yml_duplicate_keys.py RefactoringTool: tests/test_invalid_yaml_yml_multi_document_validation.py RefactoringTool: tests/test_invalid_yaml_yml_single_document_validation.py Signed-off-by: Marcello de Sales --- tests/context.py | 6 +++--- tests/test_all_valid_config_validation.py | 4 ++-- tests/test_invalid_matrix_json_validation.py | 4 ++-- tests/test_invalid_properties_validation.py | 4 ++-- tests/test_invalid_yaml_yml_duplicate_keys.py | 4 ++-- tests/test_invalid_yaml_yml_multi_document_validation.py | 4 ++-- tests/test_invalid_yaml_yml_single_document_validation.py | 4 ++-- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/context.py b/tests/context.py index 8354c5d..3a65e08 100644 --- a/tests/context.py +++ b/tests/context.py @@ -43,12 +43,12 @@ def printFileValidationStatus(filePath, validationObject): message = str(isValid) if isValid else str(isValid) + " ERROR: " + str(validationObject) # The V of successful in green - v = ShellColor.OKGREEN + str(u'\u2714'.encode('UTF-8')) + v = ShellColor.OKGREEN + str('\u2714'.encode('UTF-8')) # The X of failure in red - x = ShellColor.FAIL + str(u'\u2718'.encode('UTF-8')) + x = ShellColor.FAIL + str('\u2718'.encode('UTF-8')) shellStatus = v if isValid else x - print shellStatus + " is " + getRelativeFixturePath(filePath) + " valid? " + message + ShellColor.ENDC + print(shellStatus + " is " + getRelativeFixturePath(filePath) + " valid? " + message + ShellColor.ENDC) class ValidationAssertions: """Validation Assertions for the test cases to validate on the values.""" diff --git a/tests/test_all_valid_config_validation.py b/tests/test_all_valid_config_validation.py index 61f03e1..50e6bfb 100644 --- a/tests/test_all_valid_config_validation.py +++ b/tests/test_all_valid_config_validation.py @@ -26,8 +26,8 @@ def test_that_validation_index_is_dictionary(self): self.assertTrue(len(self.validationIndex) > 0) def test_all_properties_are_valid(self): - print ShellColor.WARNING + "All config files are valid" + ShellColor.ENDC - for filePath, validationObject in self.validationIndex.iteritems(): + print(ShellColor.WARNING + "All config files are valid" + ShellColor.ENDC) + for filePath, validationObject in self.validationIndex.items(): printFileValidationStatus(filePath, validationObject) # Verify if the directory is in the file path diff --git a/tests/test_invalid_matrix_json_validation.py b/tests/test_invalid_matrix_json_validation.py index 8eb2434..de9d30f 100644 --- a/tests/test_invalid_matrix_json_validation.py +++ b/tests/test_invalid_matrix_json_validation.py @@ -26,8 +26,8 @@ def test_that_validation_index_is_dictionary(self): self.assertTrue(len(self.validationIndex) > 0) def test_all_matrix_json_files_are_invalid(self): - print ShellColor.WARNING + "The android matrix file is invalid" + ShellColor.ENDC - for filePath, validationObject in self.validationIndex.iteritems(): + print(ShellColor.WARNING + "The android matrix file is invalid" + ShellColor.ENDC) + for filePath, validationObject in self.validationIndex.items(): isValid = isConfigValid(validationObject) printFileValidationStatus(filePath, validationObject) diff --git a/tests/test_invalid_properties_validation.py b/tests/test_invalid_properties_validation.py index effa1d1..0031432 100644 --- a/tests/test_invalid_properties_validation.py +++ b/tests/test_invalid_properties_validation.py @@ -26,8 +26,8 @@ def test_that_validation_index_is_dictionary(self): self.assertTrue(len(self.validationIndex) > 0) def test_some_yaml_yml_files_are_invalid(self): - print ShellColor.WARNING + "Properties files are invalid without associated values" + ShellColor.ENDC - for filePath, validationObject in self.validationIndex.iteritems(): + print(ShellColor.WARNING + "Properties files are invalid without associated values" + ShellColor.ENDC) + for filePath, validationObject in self.validationIndex.items(): isValid = isConfigValid(validationObject) printFileValidationStatus(filePath, validationObject) diff --git a/tests/test_invalid_yaml_yml_duplicate_keys.py b/tests/test_invalid_yaml_yml_duplicate_keys.py index 50967ce..4f0fcb3 100644 --- a/tests/test_invalid_yaml_yml_duplicate_keys.py +++ b/tests/test_invalid_yaml_yml_duplicate_keys.py @@ -26,8 +26,8 @@ def test_that_validation_index_is_dictionary(self): self.assertTrue(len(self.validationIndex) > 0) def test_some_yaml_yml_files_are_invalid(self): - print ShellColor.WARNING + "Some Yaml Single documents are invalid" + ShellColor.ENDC - for filePath, validationObject in self.validationIndex.iteritems(): + print(ShellColor.WARNING + "Some Yaml Single documents are invalid" + ShellColor.ENDC) + for filePath, validationObject in self.validationIndex.items(): isValid = isConfigValid(validationObject) printFileValidationStatus(filePath, validationObject) diff --git a/tests/test_invalid_yaml_yml_multi_document_validation.py b/tests/test_invalid_yaml_yml_multi_document_validation.py index 5898bc5..52dd845 100644 --- a/tests/test_invalid_yaml_yml_multi_document_validation.py +++ b/tests/test_invalid_yaml_yml_multi_document_validation.py @@ -26,8 +26,8 @@ def test_that_validation_index_is_dictionary(self): self.assertTrue(len(self.validationIndex) > 0) def test_some_yaml_yml_files_are_invalid(self): - print ShellColor.WARNING + "Some Yaml Multi documents are invalid" + ShellColor.ENDC - for filePath, validationObject in self.validationIndex.iteritems(): + print(ShellColor.WARNING + "Some Yaml Multi documents are invalid" + ShellColor.ENDC) + for filePath, validationObject in self.validationIndex.items(): isValid = isConfigValid(validationObject) printFileValidationStatus(filePath, validationObject) diff --git a/tests/test_invalid_yaml_yml_single_document_validation.py b/tests/test_invalid_yaml_yml_single_document_validation.py index bc5e345..58c3367 100644 --- a/tests/test_invalid_yaml_yml_single_document_validation.py +++ b/tests/test_invalid_yaml_yml_single_document_validation.py @@ -26,8 +26,8 @@ def test_that_validation_index_is_dictionary(self): self.assertTrue(len(self.validationIndex) > 0) def test_some_yaml_yml_files_are_invalid(self): - print ShellColor.WARNING + "Some Yaml Single documents are invalid" + ShellColor.ENDC - for filePath, validationObject in self.validationIndex.iteritems(): + print(ShellColor.WARNING + "Some Yaml Single documents are invalid" + ShellColor.ENDC) + for filePath, validationObject in self.validationIndex.items(): isValid = isConfigValid(validationObject) printFileValidationStatus(filePath, validationObject) From a6a40ca1a0b57dbc548aabb0da48ad411d17e51d Mon Sep 17 00:00:00 2001 From: Marcello de Sales Date: Thu, 2 May 2019 00:14:40 -0700 Subject: [PATCH 04/12] Issue #8: Tests: python 3 must add the UTF-8 param to str for decoding Signed-off-by: Marcello de Sales --- tests/context.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/context.py b/tests/context.py index 3a65e08..081ea92 100644 --- a/tests/context.py +++ b/tests/context.py @@ -42,10 +42,11 @@ def printFileValidationStatus(filePath, validationObject): isValid = isConfigValid(validationObject) message = str(isValid) if isValid else str(isValid) + " ERROR: " + str(validationObject) + # https://stackoverflow.com/questions/6812031/how-to-make-unicode-string-with-python3/6812069#6812069 (added , UTF-8 param to str) # The V of successful in green - v = ShellColor.OKGREEN + str('\u2714'.encode('UTF-8')) + v = ShellColor.OKGREEN + str('\u2714'.encode('UTF-8'), 'UTF-8') # The X of failure in red - x = ShellColor.FAIL + str('\u2718'.encode('UTF-8')) + x = ShellColor.FAIL + str('\u2718'.encode('UTF-8'), 'UTF-8') shellStatus = v if isValid else x print(shellStatus + " is " + getRelativeFixturePath(filePath) + " valid? " + message + ShellColor.ENDC) From 50ea725e715e0537cc29d0d1d859e52273071c7f Mon Sep 17 00:00:00 2001 From: Marcello de Sales Date: Thu, 2 May 2019 00:50:41 -0700 Subject: [PATCH 05/12] Issue #8: Validate: Python3: Closing file resource: reading yaml during lint Getting the following error: /build/validate_config_files.py:181: ResourceWarning: unclosed file <_io.TextIOWrapper name='/build/tests/fixtures/all-valid-config/publisher-onboard_preprod.yml' mode='r' encoding='UTF-8'> lintError = list(linter.run(open(filePath), yamlLintConfig)); Signed-off-by: Marcello de Sales --- validate_config_files.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/validate_config_files.py b/validate_config_files.py index a8dd741..b2f642b 100755 --- a/validate_config_files.py +++ b/validate_config_files.py @@ -169,7 +169,10 @@ def isYamlFileValid(filePath): yamlLintConfig = YamlLintConfig(yaml.safe_dump(conf)) # ymlDocs = yaml.load(open(filePath), Loader = yaml.Loader) - lintError = list(linter.run(open(filePath), yamlLintConfig)); + lintError = None + with open(filePath) as yamlFile: + lintError = list(linter.run(yamlFile, yamlLintConfig)) + yamlFile.close() if lintError: return lintError From 1be42553b136e139562ec2cb0ca755ac27f1d2fa Mon Sep 17 00:00:00 2001 From: Marcello de Sales Date: Thu, 2 May 2019 01:19:25 -0700 Subject: [PATCH 06/12] Issue #8: Dockerfile: Python3 hide deprecation warnings: glob2: system-wide https://stackoverflow.com/questions/879173/how-to-ignore-deprecation-warnings-in-python/55947748#55947748 test_all_properties_are_valid (test_all_valid_config_validation.AllSuccessfulTests) ... /usr/lib/python3.6/site-packages/glob2/fnmatch.py:80: DeprecationWarning: Flags not at the start of the expression '(.*)\\Z(?ms)' return re.compile(res, flags).match /usr/lib/python3.6/site-packages/glob2/fnmatch.py:80: DeprecationWarning: Flags not at the start of the expression '(.*)\\.json\\Z(?ms)' return re.compile(res, flags).match /usr/lib/python3.6/site-packages/glob2/fnmatch.py:80: DeprecationWarning: Flags not at the start of the expression '(.*)\\.yaml\\Z(?ms)' return re.compile(res, flags).match /usr/lib/python3.6/site-packages/glob2/fnmatch.py:80: DeprecationWarning: Flags not at the start of the expression '(.*)\\.yml\\Z(?ms)' return re.compile(res, flags).match /usr/lib/python3.6/site-packages/glob2/fnmatch.py:80: DeprecationWarning: Flags not at the start of the expression '(.*)\\.properties\\Z(?' (truncated) return re.compile(res, flags).match ok Signed-off-by: Marcello de Sales --- Dockerfile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 95f721f..aa216af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,9 @@ ####### Build Image -FROM marcellodesales/github-enterprise-prereceive-hook-base as tests - -# Install dependencies -RUN apk add --no-cache py-pip && \ - pip install coverage +FROM marcellodesales/git-pre-receive-hook:python3 as tests # Make a cache-eligible dependencies -COPY requirements.txt /build/requirements.txt +COPY requirements-tests.txt /build/requirements.txt # Install dependencies RUN pip install -r /build/requirements.txt @@ -16,15 +12,19 @@ RUN pip install -r /build/requirements.txt COPY ./tests /build/tests COPY ./validate_config_files.py /build +# Glob2 warnings https://github.com/miracle2k/python-glob2/issues/24 +ENV PYTHONWARNINGS="ignore::DeprecationWarning:glob2" + RUN coverage run -m unittest discover -v /build/tests ###### Runtime Image -FROM marcellodesales/github-enterprise-prereceive-hook-base as runtime +FROM marcellodesales/git-pre-receive-hook:python3 as runtime -RUN apk add --no-cache py-pip - -COPY --from=tests /build/requirements.txt requirements.txt +COPY requirements.txt requirements.txt RUN pip install -r requirements.txt COPY --from=tests /build/validate_config_files.py /home/git/test.git/hooks/pre-receive + +# Glob2 warnings https://github.com/miracle2k/python-glob2/issues/24 +ENV PYTHONWARNINGS="ignore::DeprecationWarning:glob2" From bd64de823e8e26a993c218e842003fe12510ce27 Mon Sep 17 00:00:00 2001 From: Marcello de Sales Date: Thu, 2 May 2019 01:22:06 -0700 Subject: [PATCH 07/12] Issue #8: Validate: initial setup failing: object str byte Fixed by using the decode UTF-8 method https://stackoverflow.com/questions/21689365/python-3-typeerror-must-be-str-not-bytes-with-sys-stdout-write/21689447#21689447 remote: ##################################################### remote: #### Intuit Spring Cloud Config Validator 1.2.0 ##### remote: ##################################################### remote: Validating new branch... remote: Processing commit=5f2acca1293550dc4ecc7aab26e77c52d74f15aa ref=refs/heads/master remote: Traceback (most recent call last): remote: File "hooks/pre-receive", line 354, in remote: (executationDirPath, validationIndex) = ShellExecution.run() remote: File "hooks/pre-receive", line 326, in run remote: currentDirPath = dirPath if dirPath else ExecutionContext.getCurrentDirPath() remote: File "hooks/pre-receive", line 83, in getCurrentDirPath remote: currentDirPath = Validator.processPreReceivehookFilesInGithub(base, commit) remote: File "hooks/pre-receive", line 243, in processPreReceivehookFilesInGithub remote: content = GitRepo.openCommitFileContent(fileName, head) remote: File "hooks/pre-receive", line 129, in openCommitFileContent remote: (results, code) = GitRepo.git(('git', 'show', commit + ":" + fileName)) remote: TypeError: must be str, not bytes To 192.168.86.249:test.git Signed-off-by: Marcello de Sales --- validate_config_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validate_config_files.py b/validate_config_files.py index b2f642b..a7e927c 100755 --- a/validate_config_files.py +++ b/validate_config_files.py @@ -118,7 +118,7 @@ def listConfigFilesInGitCommits(base, commit): # Filter the non-empty, non-repeated elements as the command returns a\nb\n\c # http://stackoverflow.com/questions/33944647/what-is-the-most-pythonic-way-to-filter-a-set/33944663#33944663 - return [x for x in set(results.strip().split('\n')) if x != ''] + return [x for x in set(results.decode('utf-8').strip().split('\n')) if x != ''] @staticmethod def openCommitFileContent(fileName, commit = "HEAD"): From b8643fd4c5c08d1ee2b3c626fd170eb178365a31 Mon Sep 17 00:00:00 2001 From: Marcello de Sales Date: Thu, 2 May 2019 01:27:00 -0700 Subject: [PATCH 08/12] Issue #8: Validate: python3: bytes to string on github stream Fixed with the decode method described at https://stackoverflow.com/questions/21689365/python-3-typeerror-must-be-str-not-bytes-with-sys-stdout-write/21689447#21689447 * Broken issue: GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 52311 -i /Users/mdesales/dev/github/public/intuit/intuit-spring-cloud-config-validator/.id_rsa_from_github_simulator_server" git push test new-branch -f Warning: Permanently added '[192.168.86.249]:52311' (ECDSA) to the list of known hosts. Enumerating objects: 37, done. Counting objects: 100% (37/37), done. Delta compression using up to 8 threads Compressing objects: 100% (12/12), done. Writing objects: 100% (35/35), 1.93 KiB | 659.00 KiB/s, done. Total 35 (delta 8), reused 0 (delta 0) remote: ##################################################### remote: #### Intuit Spring Cloud Config Validator 1.3.0 ##### remote: ##################################################### remote: Validating new branch... remote: Processing commit=a34a90b2bfcd2018b746e8ee78e51bf393e6dbc1 ref=refs/heads/new-branch remote: Traceback (most recent call last): remote: File "hooks/pre-receive", line 354, in remote: (executationDirPath, validationIndex) = ShellExecution.run() remote: File "hooks/pre-receive", line 326, in run remote: currentDirPath = dirPath if dirPath else ExecutionContext.getCurrentDirPath() remote: File "hooks/pre-receive", line 83, in getCurrentDirPath remote: currentDirPath = Validator.processPreReceivehookFilesInGithub(base, commit) remote: File "hooks/pre-receive", line 263, in processPreReceivehookFilesInGithub remote: filePath = Validator.saveFileContent(fileName, content, contextDir) remote: File "hooks/pre-receive", line 202, in saveFileContent remote: text_file.write(content) remote: TypeError: write() argument must be str, not bytes Signed-off-by: Marcello de Sales --- validate_config_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validate_config_files.py b/validate_config_files.py index a7e927c..33701f8 100755 --- a/validate_config_files.py +++ b/validate_config_files.py @@ -199,7 +199,7 @@ def saveFileContent(fileName, content, contextDir): # Save the file in the context with open(filePath, "w") as text_file: - text_file.write(content) + text_file.write(content.decode('UTF-8')) return filePath From c261e3ef2ce29c2badd99a262e1f6e3c975a6133 Mon Sep 17 00:00:00 2001 From: Marcello de Sales Date: Tue, 26 Nov 2019 02:21:09 -0800 Subject: [PATCH 09/12] Issue #8: python3: Env: update version to 1.3.0 Signed-off-by: Marcello de Sales --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 6aa1ebd..0743293 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -VERSION=1.2.0 +VERSION=1.3.0 From 324bbc48211a540a2b8454844f47bc4d88a00dba Mon Sep 17 00:00:00 2001 From: Marcello de Sales Date: Tue, 26 Nov 2019 02:32:28 -0800 Subject: [PATCH 10/12] Issue #8: python3: README: add link and asciirec to package.sh Signed-off-by: Marcello de Sales --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d614901..a95339e 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Go to the [Wiki](https://github.com/intuit/intuit-spring-cloud-config-validator/ [![resolution](http://dockeri.co/image/intuit/intuit-spring-cloud-config-validator "Github Enterprise Pre-Receive Hook Base Image")](https://hub.docker.com/r/intuit/intuit-spring-cloud-config-validator/) -* Python Base Image +* Golang, Python2 and Python3 images at the following: [![resolution](http://dockeri.co/image/marcellodesales/github-enterprise-prereceive-hook-base "Github Enterprise Pre-Receive Hook Base Image")](https://hub.docker.com/r/marcellodesales/github-enterprise-prereceive-hook-base/) @@ -32,6 +32,16 @@ Run the following to setup a local development environment: 1. `setup-github-simulator.sh`: Create a Git server with the pre-receive hook script `validate_config_files.py` 2. `test.sh`: test a given github config repo locally by attempting to push to the test git server +# Package for Github Enterprise + +* When you finish developing your hook, just run: + +```console +./package.sh +``` + +[![asciicast](https://asciinema.org/a/mzKWawCw5lZlhxIGOtHv1rJij.svg)](https://asciinema.org/a/mzKWawCw5lZlhxIGOtHv1rJij) + ## Create Git Server * Run the command `./setup-github-simulator.sh` From b14054a6cf9476b932481270b2a942224af59daf Mon Sep 17 00:00:00 2001 From: Marcello de Sales Date: Tue, 26 Nov 2019 02:58:08 -0800 Subject: [PATCH 11/12] Issue #8: Details about package.sh for Hook packaging For packaging for Github Enterprise, users can now package from the current source or package from the given docker image version specified in the .env file. Signed-off-by: Marcello de Sales --- .env | 1 + README.md | 15 ++++++++++++++- package.sh | 32 +++++++++++++++++++++++--------- 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/.env b/.env index 0743293..fb5fdee 100644 --- a/.env +++ b/.env @@ -1 +1,2 @@ +ACTION=package VERSION=1.3.0 diff --git a/README.md b/README.md index a95339e..0e95d01 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,18 @@ Run the following to setup a local development environment: # Package for Github Enterprise -* When you finish developing your hook, just run: +* You can package the latest version of this branch by making sure `.env` file has the following: +* VERSION: The version to apply the action +* ACTION: the action to perform + * `pacakge` makes a tar.gz by building q docker image of the current files + * `pull` downloads the given version from the Docker Registry + +```sh +ACTION=package +VERSION=1.3.0 +``` + +* Just run ```console ./package.sh @@ -42,6 +53,8 @@ Run the following to setup a local development environment: [![asciicast](https://asciinema.org/a/mzKWawCw5lZlhxIGOtHv1rJij.svg)](https://asciinema.org/a/mzKWawCw5lZlhxIGOtHv1rJij) + + ## Create Git Server * Run the command `./setup-github-simulator.sh` diff --git a/package.sh b/package.sh index 3b98f9e..0fb540e 100755 --- a/package.sh +++ b/package.sh @@ -15,8 +15,17 @@ echo "${red}##### INTUIT SPRING CLOUD CONFIG VALIDATOR ${VERSION} #####${yellow} echo "" date echo "" -echo "${green}* Packaging the Github Enterprise artifact for upload" -echo "${green}* File will be: ${yellow}intuit-spring-cloud-config-validator-latest.tar.gz" + +if [ "${ACTION}" == "package" ]; then + echo "${green}* Packaging the Github Enterprise artifact for upload" + +elif [ "${ACTION}" == "pull" ]; then + echo "${green}* Pulling the Github Enterprise artifact from Docker Hub" + +else + echo "${red}ERROR: set ACTION=package | pull in file '.env'" +fi +echo "${green}* File will be: ${yellow}intuit-spring-cloud-config-validator-${VERSION}.tar.gz" echo "" echo "${red}==========--------- Building the new docker image -----------==========" @@ -24,7 +33,12 @@ echo "" echo "${green}* docker-compose build${yellow}" echo "" -docker-compose build +if [ "${ACTION}" == "package" ]; then + docker-compose build + +elif [ "${ACTION}" == "pull" ]; then + docker-compose pull +fi echo "" DOCKER_IMAGE=intuit/intuit-spring-cloud-config-validator:${VERSION} @@ -47,17 +61,17 @@ docker create --name config-validator ${DOCKER_IMAGE} /bin/true echo "" echo "${red}==========--------- Export .tar.gz -----------==========" echo "" -echo "${green}* docker export config-validator | gzip > intuit-spring-cloud-config-validator-latest.tar.gz${yellow}" +echo "${green}* docker export config-validator | gzip > intuit-spring-cloud-config-validator-${VERSION}.tar.gz${yellow}" echo "" -docker export config-validator | gzip > intuit-spring-cloud-config-validator-latest.tar.gz +docker export config-validator | gzip > intuit-spring-cloud-config-validator-${VERSION}.tar.gz echo "" echo "${red}==========--------- Clean up -----------==========" echo "" -echo "${green}* docker stop config-validator && docker rm config-validator && ls -la intuit-spring-cloud-config-validator-latest.tar.gz${yellow}" +echo "${green}* docker stop config-validator && docker rm config-validator && ls -la intuit-spring-cloud-config-validator-${VERSION}.tar.gz${yellow}" echo "" -docker stop config-validator && docker rm config-validator && ls -la intuit-spring-cloud-config-validator-latest.tar.gz +docker stop config-validator && docker rm config-validator && ls -la intuit-spring-cloud-config-validator-${VERSION}.tar.gz echo "" echo "${red}==========--------- Upload to Github Enterprise -----------==========" @@ -66,8 +80,8 @@ echo "${yellow}" date echo "" -echo "${green}You can now install the new version on your Github Enterprise installation" -echo "${green}* File to upload: ${yellow}$(pwd)/intuit-spring-cloud-config-validator-latest.tar.gz" +echo "${green}You can now install version '${VERSION}' on your Github Enterprise installation" +echo "${green}* File to upload: ${yellow}$(pwd)/intuit-spring-cloud-config-validator-${VERSION}.tar.gz" echo "" echo "${green}* Your users can now enable this ${yellow}Pre-receive hook${green} on a github repo to verify it" echo "" From 2eaa2df76a441d5a786b29235f4d7f487fd4753b Mon Sep 17 00:00:00 2001 From: Marcello de Sales Date: Tue, 26 Nov 2019 03:03:47 -0800 Subject: [PATCH 12/12] Issue #8: README: details on the pull vs pacakge.sh: Signed-off-by: Marcello de Sales --- .env | 4 ++-- README.md | 11 +++-------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.env b/.env index fb5fdee..dbf777c 100644 --- a/.env +++ b/.env @@ -1,2 +1,2 @@ -ACTION=package -VERSION=1.3.0 +ACTION=pull +VERSION=1.2.0 diff --git a/README.md b/README.md index 0e95d01..03be268 100644 --- a/README.md +++ b/README.md @@ -45,15 +45,10 @@ ACTION=package VERSION=1.3.0 ``` -* Just run - -```console -./package.sh -``` - -[![asciicast](https://asciinema.org/a/mzKWawCw5lZlhxIGOtHv1rJij.svg)](https://asciinema.org/a/mzKWawCw5lZlhxIGOtHv1rJij) - +* Just run `./package.sh` after setting up the proper values on `.env` as specified above. +* See building from sources or pulling from an existing version below +[![asciicast](https://asciinema.org/a/iXe3m2rYyyszG2aJ053WjYmK2.svg)](https://asciinema.org/a/iXe3m2rYyyszG2aJ053WjYmK2) ## Create Git Server