diff --git a/.env b/.env index 6aa1ebd..dbf777c 100644 --- a/.env +++ b/.env @@ -1 +1,2 @@ +ACTION=pull VERSION=1.2.0 diff --git a/Dockerfile b/Dockerfile index 3a730c1..aa216af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,30 +1,30 @@ ####### Build Image -FROM marcellodesales/github-enterprise-prereceive-hook-base as tests - -# Install dependencies -RUN apk add --no-cache py-pip && \ - pip2 install coverage +FROM marcellodesales/git-pre-receive-hook:python3 as tests # Make a cache-eligible dependencies -COPY requirements.txt /build/requirements.txt +COPY requirements-tests.txt /build/requirements.txt # Install dependencies -RUN pip2 install -r /build/requirements.txt +RUN pip install -r /build/requirements.txt # Copy resources COPY ./tests /build/tests COPY ./validate_config_files.py /build +# Glob2 warnings https://github.com/miracle2k/python-glob2/issues/24 +ENV PYTHONWARNINGS="ignore::DeprecationWarning:glob2" + RUN coverage run -m unittest discover -v /build/tests ###### Runtime Image -FROM marcellodesales/github-enterprise-prereceive-hook-base as runtime +FROM marcellodesales/git-pre-receive-hook:python3 as runtime -RUN apk add --no-cache py-pip - -COPY --from=tests /build/requirements.txt requirements.txt -RUN pip2 install -r requirements.txt +COPY requirements.txt requirements.txt +RUN pip install -r requirements.txt COPY --from=tests /build/validate_config_files.py /home/git/test.git/hooks/pre-receive + +# Glob2 warnings https://github.com/miracle2k/python-glob2/issues/24 +ENV PYTHONWARNINGS="ignore::DeprecationWarning:glob2" diff --git a/README.md b/README.md index d614901..03be268 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Go to the [Wiki](https://github.com/intuit/intuit-spring-cloud-config-validator/ [![resolution](http://dockeri.co/image/intuit/intuit-spring-cloud-config-validator "Github Enterprise Pre-Receive Hook Base Image")](https://hub.docker.com/r/intuit/intuit-spring-cloud-config-validator/) -* Python Base Image +* Golang, Python2 and Python3 images at the following: [![resolution](http://dockeri.co/image/marcellodesales/github-enterprise-prereceive-hook-base "Github Enterprise Pre-Receive Hook Base Image")](https://hub.docker.com/r/marcellodesales/github-enterprise-prereceive-hook-base/) @@ -32,6 +32,24 @@ Run the following to setup a local development environment: 1. `setup-github-simulator.sh`: Create a Git server with the pre-receive hook script `validate_config_files.py` 2. `test.sh`: test a given github config repo locally by attempting to push to the test git server +# Package for Github Enterprise + +* You can package the latest version of this branch by making sure `.env` file has the following: +* VERSION: The version to apply the action +* ACTION: the action to perform + * `pacakge` makes a tar.gz by building q docker image of the current files + * `pull` downloads the given version from the Docker Registry + +```sh +ACTION=package +VERSION=1.3.0 +``` + +* Just run `./package.sh` after setting up the proper values on `.env` as specified above. +* See building from sources or pulling from an existing version below + +[![asciicast](https://asciinema.org/a/iXe3m2rYyyszG2aJ053WjYmK2.svg)](https://asciinema.org/a/iXe3m2rYyyszG2aJ053WjYmK2) + ## Create Git Server * Run the command `./setup-github-simulator.sh` diff --git a/package.sh b/package.sh index 3b98f9e..0fb540e 100755 --- a/package.sh +++ b/package.sh @@ -15,8 +15,17 @@ echo "${red}##### INTUIT SPRING CLOUD CONFIG VALIDATOR ${VERSION} #####${yellow} echo "" date echo "" -echo "${green}* Packaging the Github Enterprise artifact for upload" -echo "${green}* File will be: ${yellow}intuit-spring-cloud-config-validator-latest.tar.gz" + +if [ "${ACTION}" == "package" ]; then + echo "${green}* Packaging the Github Enterprise artifact for upload" + +elif [ "${ACTION}" == "pull" ]; then + echo "${green}* Pulling the Github Enterprise artifact from Docker Hub" + +else + echo "${red}ERROR: set ACTION=package | pull in file '.env'" +fi +echo "${green}* File will be: ${yellow}intuit-spring-cloud-config-validator-${VERSION}.tar.gz" echo "" echo "${red}==========--------- Building the new docker image -----------==========" @@ -24,7 +33,12 @@ echo "" echo "${green}* docker-compose build${yellow}" echo "" -docker-compose build +if [ "${ACTION}" == "package" ]; then + docker-compose build + +elif [ "${ACTION}" == "pull" ]; then + docker-compose pull +fi echo "" DOCKER_IMAGE=intuit/intuit-spring-cloud-config-validator:${VERSION} @@ -47,17 +61,17 @@ docker create --name config-validator ${DOCKER_IMAGE} /bin/true echo "" echo "${red}==========--------- Export .tar.gz -----------==========" echo "" -echo "${green}* docker export config-validator | gzip > intuit-spring-cloud-config-validator-latest.tar.gz${yellow}" +echo "${green}* docker export config-validator | gzip > intuit-spring-cloud-config-validator-${VERSION}.tar.gz${yellow}" echo "" -docker export config-validator | gzip > intuit-spring-cloud-config-validator-latest.tar.gz +docker export config-validator | gzip > intuit-spring-cloud-config-validator-${VERSION}.tar.gz echo "" echo "${red}==========--------- Clean up -----------==========" echo "" -echo "${green}* docker stop config-validator && docker rm config-validator && ls -la intuit-spring-cloud-config-validator-latest.tar.gz${yellow}" +echo "${green}* docker stop config-validator && docker rm config-validator && ls -la intuit-spring-cloud-config-validator-${VERSION}.tar.gz${yellow}" echo "" -docker stop config-validator && docker rm config-validator && ls -la intuit-spring-cloud-config-validator-latest.tar.gz +docker stop config-validator && docker rm config-validator && ls -la intuit-spring-cloud-config-validator-${VERSION}.tar.gz echo "" echo "${red}==========--------- Upload to Github Enterprise -----------==========" @@ -66,8 +80,8 @@ echo "${yellow}" date echo "" -echo "${green}You can now install the new version on your Github Enterprise installation" -echo "${green}* File to upload: ${yellow}$(pwd)/intuit-spring-cloud-config-validator-latest.tar.gz" +echo "${green}You can now install version '${VERSION}' on your Github Enterprise installation" +echo "${green}* File to upload: ${yellow}$(pwd)/intuit-spring-cloud-config-validator-${VERSION}.tar.gz" echo "" echo "${green}* Your users can now enable this ${yellow}Pre-receive hook${green} on a github repo to verify it" echo "" diff --git a/tests/context.py b/tests/context.py index 8354c5d..081ea92 100644 --- a/tests/context.py +++ b/tests/context.py @@ -42,13 +42,14 @@ def printFileValidationStatus(filePath, validationObject): isValid = isConfigValid(validationObject) message = str(isValid) if isValid else str(isValid) + " ERROR: " + str(validationObject) + # https://stackoverflow.com/questions/6812031/how-to-make-unicode-string-with-python3/6812069#6812069 (added , UTF-8 param to str) # The V of successful in green - v = ShellColor.OKGREEN + str(u'\u2714'.encode('UTF-8')) + v = ShellColor.OKGREEN + str('\u2714'.encode('UTF-8'), 'UTF-8') # The X of failure in red - x = ShellColor.FAIL + str(u'\u2718'.encode('UTF-8')) + x = ShellColor.FAIL + str('\u2718'.encode('UTF-8'), 'UTF-8') shellStatus = v if isValid else x - print shellStatus + " is " + getRelativeFixturePath(filePath) + " valid? " + message + ShellColor.ENDC + print(shellStatus + " is " + getRelativeFixturePath(filePath) + " valid? " + message + ShellColor.ENDC) class ValidationAssertions: """Validation Assertions for the test cases to validate on the values.""" diff --git a/tests/test_all_valid_config_validation.py b/tests/test_all_valid_config_validation.py index 61f03e1..50e6bfb 100644 --- a/tests/test_all_valid_config_validation.py +++ b/tests/test_all_valid_config_validation.py @@ -26,8 +26,8 @@ def test_that_validation_index_is_dictionary(self): self.assertTrue(len(self.validationIndex) > 0) def test_all_properties_are_valid(self): - print ShellColor.WARNING + "All config files are valid" + ShellColor.ENDC - for filePath, validationObject in self.validationIndex.iteritems(): + print(ShellColor.WARNING + "All config files are valid" + ShellColor.ENDC) + for filePath, validationObject in self.validationIndex.items(): printFileValidationStatus(filePath, validationObject) # Verify if the directory is in the file path diff --git a/tests/test_invalid_matrix_json_validation.py b/tests/test_invalid_matrix_json_validation.py index 8eb2434..de9d30f 100644 --- a/tests/test_invalid_matrix_json_validation.py +++ b/tests/test_invalid_matrix_json_validation.py @@ -26,8 +26,8 @@ def test_that_validation_index_is_dictionary(self): self.assertTrue(len(self.validationIndex) > 0) def test_all_matrix_json_files_are_invalid(self): - print ShellColor.WARNING + "The android matrix file is invalid" + ShellColor.ENDC - for filePath, validationObject in self.validationIndex.iteritems(): + print(ShellColor.WARNING + "The android matrix file is invalid" + ShellColor.ENDC) + for filePath, validationObject in self.validationIndex.items(): isValid = isConfigValid(validationObject) printFileValidationStatus(filePath, validationObject) diff --git a/tests/test_invalid_properties_validation.py b/tests/test_invalid_properties_validation.py index effa1d1..0031432 100644 --- a/tests/test_invalid_properties_validation.py +++ b/tests/test_invalid_properties_validation.py @@ -26,8 +26,8 @@ def test_that_validation_index_is_dictionary(self): self.assertTrue(len(self.validationIndex) > 0) def test_some_yaml_yml_files_are_invalid(self): - print ShellColor.WARNING + "Properties files are invalid without associated values" + ShellColor.ENDC - for filePath, validationObject in self.validationIndex.iteritems(): + print(ShellColor.WARNING + "Properties files are invalid without associated values" + ShellColor.ENDC) + for filePath, validationObject in self.validationIndex.items(): isValid = isConfigValid(validationObject) printFileValidationStatus(filePath, validationObject) diff --git a/tests/test_invalid_yaml_yml_duplicate_keys.py b/tests/test_invalid_yaml_yml_duplicate_keys.py index 50967ce..4f0fcb3 100644 --- a/tests/test_invalid_yaml_yml_duplicate_keys.py +++ b/tests/test_invalid_yaml_yml_duplicate_keys.py @@ -26,8 +26,8 @@ def test_that_validation_index_is_dictionary(self): self.assertTrue(len(self.validationIndex) > 0) def test_some_yaml_yml_files_are_invalid(self): - print ShellColor.WARNING + "Some Yaml Single documents are invalid" + ShellColor.ENDC - for filePath, validationObject in self.validationIndex.iteritems(): + print(ShellColor.WARNING + "Some Yaml Single documents are invalid" + ShellColor.ENDC) + for filePath, validationObject in self.validationIndex.items(): isValid = isConfigValid(validationObject) printFileValidationStatus(filePath, validationObject) diff --git a/tests/test_invalid_yaml_yml_multi_document_validation.py b/tests/test_invalid_yaml_yml_multi_document_validation.py index 5898bc5..52dd845 100644 --- a/tests/test_invalid_yaml_yml_multi_document_validation.py +++ b/tests/test_invalid_yaml_yml_multi_document_validation.py @@ -26,8 +26,8 @@ def test_that_validation_index_is_dictionary(self): self.assertTrue(len(self.validationIndex) > 0) def test_some_yaml_yml_files_are_invalid(self): - print ShellColor.WARNING + "Some Yaml Multi documents are invalid" + ShellColor.ENDC - for filePath, validationObject in self.validationIndex.iteritems(): + print(ShellColor.WARNING + "Some Yaml Multi documents are invalid" + ShellColor.ENDC) + for filePath, validationObject in self.validationIndex.items(): isValid = isConfigValid(validationObject) printFileValidationStatus(filePath, validationObject) diff --git a/tests/test_invalid_yaml_yml_single_document_validation.py b/tests/test_invalid_yaml_yml_single_document_validation.py index bc5e345..58c3367 100644 --- a/tests/test_invalid_yaml_yml_single_document_validation.py +++ b/tests/test_invalid_yaml_yml_single_document_validation.py @@ -26,8 +26,8 @@ def test_that_validation_index_is_dictionary(self): self.assertTrue(len(self.validationIndex) > 0) def test_some_yaml_yml_files_are_invalid(self): - print ShellColor.WARNING + "Some Yaml Single documents are invalid" + ShellColor.ENDC - for filePath, validationObject in self.validationIndex.iteritems(): + print(ShellColor.WARNING + "Some Yaml Single documents are invalid" + ShellColor.ENDC) + for filePath, validationObject in self.validationIndex.items(): isValid = isConfigValid(validationObject) printFileValidationStatus(filePath, validationObject) diff --git a/validate_config_files.py b/validate_config_files.py index 18c5f8f..33701f8 100755 --- a/validate_config_files.py +++ b/validate_config_files.py @@ -32,7 +32,7 @@ def isOnTestCases(): * python -m unittest discover -v tests """ # http://stackoverflow.com/questions/4858100/how-to-list-imported-modules/4858123#4858123 - return 'unittest' in sys.modules.keys() or 'tests.sys' in sys.modules.keys() + return 'unittest' in list(sys.modules.keys()) or 'tests.sys' in list(sys.modules.keys()) @staticmethod def getCurrentDirPath(): @@ -56,7 +56,7 @@ def getCurrentDirPath(): # If the execution is on github if not ExecutionContext.isOnGithub(): - print "=> Validating repo " + currentDirPath + print("=> Validating repo " + currentDirPath) else: # https://help.github.com/enterprise/2.6/admin/guides/developer-workflow/creating-a-pre-receive-hook-script/#writing-a-pre-receive-hook-script @@ -72,20 +72,20 @@ def getCurrentDirPath(): # Deleting a branch should NOT validate anything... skipping... if "0000000" in commit: - print "Deleting branch... Skip validation" + print("Deleting branch... Skip validation") exit(0) if "0000000" in base: - print "Validating new branch..." + print("Validating new branch...") - print "Processing commit=" + commit + " ref=" + ref + print("Processing commit=" + commit + " ref=" + ref) currentDirPath = Validator.processPreReceivehookFilesInGithub(base, commit) if "0000000" not in base: - print "=> Validating " + base + ".." + commit + print("=> Validating " + base + ".." + commit) else: - print "=> Validating SHA " + commit + print("=> Validating SHA " + commit) return currentDirPath @@ -118,7 +118,7 @@ def listConfigFilesInGitCommits(base, commit): # Filter the non-empty, non-repeated elements as the command returns a\nb\n\c # http://stackoverflow.com/questions/33944647/what-is-the-most-pythonic-way-to-filter-a-set/33944663#33944663 - return [x for x in set(results.strip().split('\n')) if x != ''] + return [x for x in set(results.decode('utf-8').strip().split('\n')) if x != ''] @staticmethod def openCommitFileContent(fileName, commit = "HEAD"): @@ -169,7 +169,10 @@ def isYamlFileValid(filePath): yamlLintConfig = YamlLintConfig(yaml.safe_dump(conf)) # ymlDocs = yaml.load(open(filePath), Loader = yaml.Loader) - lintError = list(linter.run(open(filePath), yamlLintConfig)); + lintError = None + with open(filePath) as yamlFile: + lintError = list(linter.run(yamlFile, yamlLintConfig)) + yamlFile.close() if lintError: return lintError @@ -196,7 +199,7 @@ def saveFileContent(fileName, content, contextDir): # Save the file in the context with open(filePath, "w") as text_file: - text_file.write(content) + text_file.write(content.decode('UTF-8')) return filePath @@ -269,7 +272,7 @@ def listAllConfigFiles(dirPath): # Valid configuration files configMatches = ["**/*.json", "**/*.yaml", "**/*.yml", "**/*.properties"] - print "Filtering Spring Cloud Config Server's files: ", configMatches + print("Filtering Spring Cloud Config Server's files: ", configMatches) # Get all the types config files based on the matches. allConfigs = [] @@ -313,9 +316,9 @@ def run(dirPath = None): """Runs the validation on a given directory, printing the report about each file verified""" # Starting the process - print "#####################################################" - print "#### Intuit Spring Cloud Config Validator " + VERSION + " #####" - print "#####################################################" + print("#####################################################") + print("#### Intuit Spring Cloud Config Validator " + VERSION + " #####") + print("#####################################################") #for key in os.environ.keys(): # print "%30s %s \n" % (key,os.environ[key]) @@ -332,15 +335,15 @@ def explain(currentDirPath, validationIndex): noErrors = True # Iterate over the index of the verifications - for filePath, isValid in validationIndex.iteritems(): + for filePath, isValid in validationIndex.items(): filePath = filePath if not ExecutionContext.isOnGithub() else str.replace(filePath, currentDirPath + "/", "") if isValid == True: - print "(v) File " + filePath + " is valid!" + print("(v) File " + filePath + " is valid!") else: isValid = isValid if not ExecutionContext.isOnGithub() else str.replace(str(isValid), currentDirPath + "/", "") # Only when we are running in github - print "(x) File " + filePath + " is invalid: " + str(isValid) + print("(x) File " + filePath + " is invalid: " + str(isValid)) noErrors = False # Exist with the value for errors