From 7b753b9116e21b08bd61002b06fe1e9358a743d0 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Thu, 31 Jan 2019 14:21:42 -0800 Subject: [PATCH 1/4] Remove badge for separate checks --- README.rst | 17 - badge_server/get_badge_data.py | 0 badge_server/main.py | 432 +----------------- badge_server/templates/dependency-result.html | 119 ----- .../templates/google-compatibility.html | 141 ------ .../templates/self-compatibility.html | 101 ---- badge_server/test_badge_server.py | 166 ------- 7 files changed, 1 insertion(+), 975 deletions(-) create mode 100644 badge_server/get_badge_data.py delete mode 100644 badge_server/templates/dependency-result.html delete mode 100644 badge_server/templates/google-compatibility.html delete mode 100644 badge_server/templates/self-compatibility.html diff --git a/README.rst b/README.rst index 94331e5c..8ac8658a 100644 --- a/README.rst +++ b/README.rst @@ -23,23 +23,6 @@ Dependency Management Toolkit for Google Cloud Python Projects .. |github_head| image:: https://python-compatibility-tools.appspot.com/one_badge_image?package=git%2Bgit://github.com/GoogleCloudPlatform/cloud-opensource-python.git%23subdirectory=compatibility_lib&force_run_check=1 :target: https://python-compatibility-tools.appspot.com/one_badge_target?package=git%2Bgit://github.com/GoogleCloudPlatform/cloud-opensource-python.git%23subdirectory=compatibility_lib -- Compatibility check results - -.. csv-table:: - :header: "CHECK_TYPE", "RESULT" - :widths: 20, 30 - - "Self Compatibility", |self_compatibility| - "Google Compatibility", |google_compatibility| - "Dependency Version Status", |dependency_version_status| - -.. |self_compatibility| image:: https://python-compatibility-tools.appspot.com/self_compatibility_badge_image?package=compatibility-lib - :target: https://python-compatibility-tools.appspot.com/self_compatibility_badge_target?package=compatibility-lib -.. |google_compatibility| image:: https://python-compatibility-tools.appspot.com/google_compatibility_badge_image?package=compatibility-lib - :target: https://python-compatibility-tools.appspot.com/google_compatibility_badge_target?package=compatibility-lib -.. |dependency_version_status| image:: https://python-compatibility-tools.appspot.com/self_dependency_badge_image?package=compatibility-lib - :target: https://python-compatibility-tools.appspot.com/self_dependency_badge_target?package=compatibility-lib - ----------------- Compatibility Lib ----------------- diff --git a/badge_server/get_badge_data.py b/badge_server/get_badge_data.py new file mode 100644 index 00000000..e69de29b diff --git a/badge_server/main.py b/badge_server/main.py index db63e880..eb143057 100644 --- a/badge_server/main.py +++ b/badge_server/main.py @@ -12,24 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" -URL for creating the badge: -[TODO] Switch to use pybadges once figure out the module not found issue. -'https://img.shields.io/badge/{name}-{status}-{color}.svg' -""" -import datetime -import logging -import requests -import threading - import flask import pybadges import utils as badge_utils -from compatibility_lib import configs from compatibility_lib import utils -from compatibility_lib import package as package_module app = flask.Flask(__name__) @@ -63,55 +51,6 @@ def _get_result_from_cache( return result -def _get_pair_status_for_packages(pkg_sets): - """Get the pairwise dependency compatibility check result for packages. - - Rules: - - Return warning status if not compatible with any of the listed - Google owned Python packages. Whole list in compatibility_lib.config. - - Ignore the warning status if: - - The package doesn't support one of the Python versions. - - The package's pair is not self compatible, which means the - pairwise conflict isn't related to the package being checked. - - Return success status if compatible with all the list of Google owned - Python packages. - """ - version_and_res = { - 'py2': { - 'status': 'SUCCESS', - 'details': {}, - }, - 'py3': { - 'status': 'SUCCESS', - 'details': {}, - } - } - for pkg_set in pkg_sets: - pkgs = [package_module.Package(pkg) for pkg in pkg_set] - pair_res = badge_utils.store.get_pair_compatibility(pkgs) - - for res in pair_res: - py_version = badge_utils.PY_VER_MAPPING[res.python_major_version] - # Status showing one of the check failures - if res.status.value != 'SUCCESS': - # Ignore the package that not support for given py_ver - if pkg_set[1] in \ - configs.PKG_PY_VERSION_NOT_SUPPORTED.get( - res.python_major_version): - continue - # Ignore the package that are not self compatible - self_status = _get_result_from_cache( - package_name=pkg_set[1], - badge_type=badge_utils.BadgeType.SELF_COMP_BADGE) - if self_status[py_version]['status'] != 'SUCCESS': - continue - version_and_res[py_version]['status'] = res.status.value - version_and_res[py_version]['details'][pkg_set[1]] = \ - res.details if res.details is not None \ - else badge_utils.EMPTY_DETAILS - return version_and_res - - def _get_all_results_from_cache(package_name, commit_number=None): """Get all the check results from cache. @@ -184,38 +123,10 @@ def one_badge_image(): is_github = True commit_number = badge_utils._calculate_commit_number(package_name) - - force_run_check = flask.request.args.get('force_run_check') - # Remove the last '/' from the url root - url_prefix = flask.request.url_root[:-1] - # Call the url for each badge to run the checks. This will populate the - # individual caches, which are used to calculate the final image state. - # Self compatibility badge - requests.get(url_prefix + flask.url_for( - 'self_compatibility_badge_image', - package=package_name, - force_run_check=force_run_check, - commit_number=commit_number)) - # Google compatibility badge - requests.get(url_prefix + flask.url_for( - 'google_compatibility_badge_image', - package=package_name, - force_run_check=force_run_check, - commit_number=commit_number)) - # Self dependency badge - requests.get(url_prefix + flask.url_for( - 'self_dependency_badge_image', - package=package_name, - force_run_check=force_run_check, - commit_number=commit_number)) - status, timestamp, _, _, _ = _get_all_results_from_cache( package_name, commit_number=commit_number) color = badge_utils.STATUS_COLOR_MAPPING[status] - details_link = url_prefix + flask.url_for('one_badge_target', - package=package_name) - # Include the check timestamp for github head if is_github and timestamp: badge_name = '{} {}'.format(badge_name, timestamp) @@ -224,8 +135,7 @@ def one_badge_image(): pybadges.badge( left_text=badge_name, right_text=status, - right_color=color, - whole_link=details_link)) + right_color=color)) response.content_type = badge_utils.SVG_CONTENT_TYPE response.headers['Cache-Control'] = 'no-cache' response.add_etag() @@ -250,345 +160,5 @@ def one_badge_target(): commit_number=commit_number) -@app.route('/self_compatibility_badge_image') -def self_compatibility_badge_image(): - """Badge showing whether a package is compatible with itself.""" - package_name = flask.request.args.get('package') - force_run_check = flask.request.args.get('force_run_check') - commit_number = flask.request.args.get('commit_number') - - badge_name = flask.request.args.get('badge_name') - package_key = '{}_{}'.format( - package_name, commit_number) if commit_number else package_name - - if badge_name is None: - badge_name = 'self compatibility' - - version_and_res = badge_utils._build_default_result( - badge_type=badge_utils.BadgeType.SELF_COMP_BADGE, - status='CALCULATING', - details=None) - - def run_check(): - # First see if this package is already stored in BigQuery. - package = package_module.Package(package_name) - compatibility_status = badge_utils.store.get_self_compatibility( - package) - if compatibility_status: - for res in compatibility_status: - py_version = badge_utils.PY_VER_MAPPING[ - res.python_major_version] - version_and_res[py_version]['status'] = res.status.value - version_and_res[py_version]['details'] = res.details \ - if res.details is not None else badge_utils.EMPTY_DETAILS - - # If not pre stored in BigQuery, run the check for the package. - else: - py2_res = badge_utils.checker.check([package_name], '2') - py3_res = badge_utils.checker.check([package_name], '3') - - version_and_res['py2']['status'] = py2_res.get('result') - py2_description = py2_res.get('description') - py2_details = badge_utils.EMPTY_DETAILS if py2_description \ - is None else py2_description - version_and_res['py2']['details'] = py2_details - version_and_res['py3']['status'] = py3_res.get('result') - py3_description = py3_res.get('description') - py3_details = badge_utils.EMPTY_DETAILS if py3_description \ - is None else py3_description - version_and_res['py3']['details'] = py3_details - - # Add the timestamp - version_and_res['timestamp'] = datetime.datetime.now().strftime( - badge_utils.TIMESTAMP_FORMAT) - - # Write the result to Cloud Datastore - cache.set('{}_self_comp_badge'.format(package_key), version_and_res) - - if not utils._is_package_in_whitelist([package_name]): - self_comp_res = badge_utils._build_default_result( - badge_type=badge_utils.BadgeType.SELF_COMP_BADGE, - status='UNKNOWN', - details=badge_utils.PACKAGE_NOT_SUPPORTED) - else: - self_comp_res = cache.get('{}_self_comp_badge'.format(package_key)) - - if self_comp_res is None: - details = version_and_res - else: - details = self_comp_res - - # Run the check if details is None or forced to populate the cache or - # the cache is outdated. - if self_comp_res is None or force_run_check is not None: - threading.Thread(target=run_check).start() - elif self_comp_res is not None: - timestamp = self_comp_res.get('timestamp') - if not badge_utils._is_github_cache_valid(timestamp): - threading.Thread(target=run_check).start() - - badge = badge_utils._get_badge(details, badge_name) - response = flask.make_response(badge) - response.content_type = badge_utils.SVG_CONTENT_TYPE - response.headers['Cache-Control'] = 'no-cache' - response.add_etag() - - return response - - -@app.route('/self_compatibility_badge_target') -def self_compatibility_badge_target(): - """Return the dict which contains the self compatibility status and details - for py2 and py3. - - e.g. { - 'py2':{ - 'status': 'SUCCESS', - 'details': None, - }, - 'py3':{ - 'status': 'CHECK_WARNING', - 'details': '...', - } - } - """ - package_name = flask.request.args.get('package') - result_dict = _get_result_from_cache( - package_name=package_name, - badge_type=badge_utils.BadgeType.SELF_COMP_BADGE) - - return flask.render_template( - 'self-compatibility.html', - package_name=package_name, - result=result_dict) - - -@app.route('/self_dependency_badge_image') -def self_dependency_badge_image(): - """Badge showing whether a package is has outdated dependencies.""" - - package_name = flask.request.args.get('package') - force_run_check = flask.request.args.get('force_run_check') - commit_number = flask.request.args.get('commit_number') - - badge_name = flask.request.args.get('badge_name') - package_key = '{}_{}'.format( - package_name, commit_number) if commit_number else package_name - - if badge_name is None: - badge_name = 'dependency status' - - def run_check(): - res = { - 'status': 'UP_TO_DATE', - 'details': {}, - 'timestamp': '', - } - details = {} - outdated = badge_utils.highlighter.check_package(package_name) - deprecated_deps_list = badge_utils.finder.get_deprecated_dep( - package_name)[1] - deprecated_deps = ', '.join(deprecated_deps_list) - - max_level = badge_utils.priority_level.UP_TO_DATE - for dep in outdated: - dep_detail = {} - level = dep.priority.level - if level.value > max_level.value: - max_level = level - dep_detail['installed_version'] = dep.installed_version - dep_detail['latest_version'] = dep.latest_version - dep_detail['priority'] = dep.priority.level.name - dep_detail['detail'] = dep.priority.details - details[dep.name] = dep_detail - res['status'] = max_level.name - res['details'] = details - res['deprecated_deps'] = deprecated_deps - res['timestamp'] = datetime.datetime.now().strftime( - badge_utils.TIMESTAMP_FORMAT) - - # Write the result to Cloud Datastore - cache.set('{}_dependency_badge'.format(package_key), res) - - if not utils._is_package_in_whitelist([package_name]): - dependency_res = badge_utils._build_default_result( - badge_type=badge_utils.BadgeType.DEP_BADGE, - status='UNKNOWN', - details={}) - else: - dependency_res = cache.get('{}_dependency_badge'.format(package_key)) - - if dependency_res is None: - details = badge_utils.DEFAULT_DEPENDENCY_RESULT - else: - details = dependency_res - - # Run the check if dependency_res is None or forced to populate the cache - # or the cache is outdated. - if dependency_res is None or force_run_check is not None: - threading.Thread(target=run_check).start() - elif dependency_res is not None: - timestamp = dependency_res.get('timestamp') - if not badge_utils._is_github_cache_valid(timestamp): - threading.Thread(target=run_check).start() - - badge = badge_utils._get_badge(details, badge_name) - response = flask.make_response(badge) - response.content_type = badge_utils.SVG_CONTENT_TYPE - response.headers['Cache-Control'] = 'no-cache' - response.add_etag() - - return response - - -@app.route('/self_dependency_badge_target') -def self_dependency_badge_target(): - """Return a dict that contains dependency status and details.""" - package_name = flask.request.args.get('package') - result_dict = _get_result_from_cache( - package_name=package_name, - badge_type=badge_utils.BadgeType.DEP_BADGE) - - return flask.render_template( - 'dependency-result.html', - package_name=package_name, - result=result_dict) - - -@app.route('/google_compatibility_badge_image') -def google_compatibility_badge_image(): - """Badge showing whether a package is compatible with Google OSS Python - packages. If all packages success, status is SUCCESS; else set status - to one of the failure types, details can be found at the target link.""" - package_name = flask.request.args.get('package') - force_run_check = flask.request.args.get('force_run_check') - commit_number = flask.request.args.get('commit_number') - - badge_name = flask.request.args.get('badge_name') - package_key = '{}_{}'.format( - package_name, commit_number) if commit_number else package_name - - if badge_name is None: - badge_name = 'google compatibility' - - def run_check(): - pkg_sets = [[package_name, pkg] for pkg in configs.PKG_LIST] - if package_name in configs.PKG_LIST: - result = _get_pair_status_for_packages(pkg_sets) - else: - version_and_res = { - 'py2': { - 'status': 'SUCCESS', - 'details': {}, - }, - 'py3': { - 'status': 'SUCCESS', - 'details': {}, - }, - 'timestamp': '', - } - - for py_ver in [2, 3]: - results = list(badge_utils.checker.get_pairwise_compatibility( - py_ver, pkg_sets)) - logging.warning(results) - py_version = badge_utils.PY_VER_MAPPING[py_ver] - - for res in results: - res_item = res[0] - status = res_item.get('result') - package = res_item.get('packages')[1] - if status != 'SUCCESS': - # Ignore the package that not support for given py_ver - if package in \ - configs.PKG_PY_VERSION_NOT_SUPPORTED.get( - py_ver): - continue - - # Ignore the package that are not self compatible - self_status = _get_result_from_cache( - package_name=package_name, - badge_type=badge_utils.BadgeType.SELF_COMP_BADGE) - if self_status[py_version]['status'] not in [ - 'SUCCESS', 'CALCULATING']: - continue - # Status showing one of the check failures - version_and_res[ - py_version]['status'] = res_item.get('result') - description = res_item.get('description') - details = badge_utils.EMPTY_DETAILS if description \ - is None else description - version_and_res[ - py_version]['details'][package] = details - version_and_res['timestamp'] = datetime.datetime.now().strftime( - badge_utils.TIMESTAMP_FORMAT) - result = version_and_res - - # Write the result to Cloud Datastore - cache.set('{}_google_comp_badge'.format(package_key), result) - - google_comp_res = cache.get('{}_google_comp_badge'.format(package_key)) - - if not utils._is_package_in_whitelist([package_name]): - google_comp_res = badge_utils._build_default_result( - badge_type=badge_utils.BadgeType.GOOGLE_COMP_BADGE, - status='UNKNOWN', - details={}) - - if google_comp_res is None: - details = badge_utils._build_default_result( - badge_type=badge_utils.BadgeType.GOOGLE_COMP_BADGE, - status='CALCULATING', - details={}) - else: - details = google_comp_res - - # Run the check if google_comp_res is None or forced to populate the cache - # or the cache is outdated. - if google_comp_res is None or force_run_check is not None: - threading.Thread(target=run_check).start() - elif google_comp_res is not None: - timestamp = google_comp_res.get('timestamp') - if not badge_utils._is_github_cache_valid(timestamp): - threading.Thread(target=run_check).start() - - badge = badge_utils._get_badge(details, badge_name) - response = flask.make_response(badge) - response.content_type = badge_utils.SVG_CONTENT_TYPE - response.headers['Cache-Control'] = 'no-cache' - response.add_etag() - - return response - - -@app.route('/google_compatibility_badge_target') -def google_compatibility_badge_target(): - """Return the dict which contains the compatibility status with google - packages and details for py2 and py3. - - e.g. { - 'py2':{ - 'status': 'SUCCESS', - 'details': None, - }, - 'py3':{ - 'status': 'CHECK_WARNING', - 'details': { - 'package1': '...', - }, - } - } - """ - package_name = flask.request.args.get('package') - result_dict = _get_result_from_cache( - package_name=package_name, - badge_type=badge_utils.BadgeType.GOOGLE_COMP_BADGE) - - return flask.render_template( - 'google-compatibility.html', - package_name=package_name, - result=result_dict) - - if __name__ == '__main__': app.run(host='0.0.0.0', port=8080) diff --git a/badge_server/templates/dependency-result.html b/badge_server/templates/dependency-result.html deleted file mode 100644 index d625ec02..00000000 --- a/badge_server/templates/dependency-result.html +++ /dev/null @@ -1,119 +0,0 @@ - - - - - - - - - - - - - Dependency Check Result - - - - - - - - - - -
-
- -
-
-
-
-

Package name: {{ package_name }} -

-
-
-
-
-
- - - -
-
-
-

- Dependency Check Result: {{ result.status }} -

-
- {% if result.status in ['LOW_PRIORITY', 'HIGH_PRIORITY'] %} - - - - - - - - - - - - {% for dep, detail in result.details.items() %} - - - - - - - - {% endfor %} - -
Dependency NamePriority to UpdateInstalled VersionLatest VersionDetails
-
-
{{ dep }}
-
- 		- {{ detail.priority }} - -
-
{{ detail.installed_version }}
-
- 		-
-
{{ detail.latest_version }}
-
- 		- {{ detail.detail }} -
- {% elif result.status == 'UP_TO_DATE' %} -

All dependencies are up to date!

- {% endif %} -
-
- {% if result.deprecated_deps | length > 0 %} -
-
-
- Deprecated Dependencies -
-
-

{{ result.deprecated_deps }}

-
-
-
- {% endif %} -
-
- -
-
- - - - - - - - - - - diff --git a/badge_server/templates/google-compatibility.html b/badge_server/templates/google-compatibility.html deleted file mode 100644 index ffea3849..00000000 --- a/badge_server/templates/google-compatibility.html +++ /dev/null @@ -1,141 +0,0 @@ - - - - - - - - - - - - - Google Compatibility Result - - - - - - - - - - -
-
- -
-
-
-
-

Package name: {{ package_name }} -

-
-
-
-
-
- - -
-
-
-
-

Google Compatibility Check Results

-
-
-

Click on the tabs to see results for different Python versions.

- - -
-
-
-

Check Result: {{ result.py2.status }}

-
- - - - - - - - - {% for dep_name, detail in result.py2.details.items() %} - - - - - {% endfor %} - -
Package NameDetails
-
-
{{ dep_name }}
-
- 		- {{ detail }} -
-
-
-
-
-
-

Check Result: {{ result.py3.status }}

-
- - - - - - - - - {% for dep_name, detail in result.py3.details.items() %} - - - - - {% endfor %} - -
Dependency NameDetails
-
-
{{ dep_name }}
-
- 		- {{ detail }} -
-
-
-
-
-
-
-
-
-
-
- - - - - - - - - - - diff --git a/badge_server/templates/self-compatibility.html b/badge_server/templates/self-compatibility.html deleted file mode 100644 index 4704478f..00000000 --- a/badge_server/templates/self-compatibility.html +++ /dev/null @@ -1,101 +0,0 @@ - - - - - - - - - - - - - Self Compatibility Result - - - - - - - - - - -
-
- -
-
-
-
-

Package name: {{ package_name }} -

-
-
-
-
-
- - - -
-
-
-

- Self Compatibility Check Results

-
- - - - - - - - - - - - - - - - - - - - -
Python VersionResultDetails
-
-
Py2
-
- 		- {{ result.py2.status }} - - {{ result.py2.details }} -
-
-
Py3
-
- 		- {{ result.py3.status }} - - {{ result.py3.details }} -
-
-
-
-
- -
-
- - - - - - - - - - - diff --git a/badge_server/test_badge_server.py b/badge_server/test_badge_server.py index 217a0e54..28c151b1 100644 --- a/badge_server/test_badge_server.py +++ b/badge_server/test_badge_server.py @@ -16,7 +16,6 @@ import os import unittest -from compatibility_lib import fake_compatibility_store os.environ["RUN_LOCALLY"] = 'true' @@ -26,171 +25,6 @@ class TestBadgeServer(unittest.TestCase): - def setUp(self): - self.mock_checker = mock.Mock(autospec=True) - self.fake_store = fake_compatibility_store.CompatibilityStore() - self.patch_checker = mock.patch( - 'main.badge_utils.checker', self.mock_checker) - self.patch_store = mock.patch( - 'main.badge_utils.store', self.fake_store) - - def test__get_pair_status_for_packages_success(self): - pkg_sets = [ - ['opencensus', 'google-api-core'], - ['opencensus', 'google-api-python-client'] - ] - expected = { - 'py2': {'status': 'SUCCESS', 'details': {}}, - 'py3': {'status': 'SUCCESS', 'details': {}} - } - - with self.patch_checker, self.patch_store: - version_and_res = main._get_pair_status_for_packages( - pkg_sets) - - self.assertEqual(version_and_res, expected) - - def test__get_pair_status_for_packages_warning(self): - from compatibility_lib import compatibility_store - from compatibility_lib import package - - PACKAGE_1 = package.Package("package1") - PACKAGE_2 = package.Package("package2") - - pkg_sets = [ - ['package1', 'package2'], - ] - expected = { - 'py2': { - 'status': 'CHECK_WARNING', - 'details': {'package2': 'NO DETAILS'} - }, - 'py3': { - 'status': 'CHECK_WARNING', - 'details': {'package2': 'NO DETAILS'} - } - } - cr_py2 = compatibility_store.CompatibilityResult( - packages=[PACKAGE_1, PACKAGE_2], - python_major_version=2, - status=compatibility_store.Status.CHECK_WARNING) - cr_py3 = compatibility_store.CompatibilityResult( - packages=[PACKAGE_1, PACKAGE_2], - python_major_version=3, - status=compatibility_store.Status.CHECK_WARNING) - pair_result = [cr_py2, cr_py3] - self.fake_store._packages_to_compatibility_result[ - frozenset([PACKAGE_1, PACKAGE_2])] = pair_result - - mock_self_res = mock.Mock() - self_res = { - 'py2': { - 'status': 'SUCCESS', 'details': {} - }, - 'py3': { - 'status': 'SUCCESS', 'details': {} - } - } - mock_self_res.return_value = self_res - patch_self_status = mock.patch( - 'main._get_result_from_cache', - mock_self_res) - - with self.patch_checker, self.patch_store, patch_self_status: - version_and_res = main._get_pair_status_for_packages( - pkg_sets) - - self.assertEqual(version_and_res, expected) - - def test__get_pair_status_for_packages_install_error(self): - from compatibility_lib import compatibility_store - from compatibility_lib import package - - PACKAGE_1 = package.Package("package1") - PACKAGE_2 = package.Package("tensorflow") - - pkg_sets = [ - ['package1', 'tensorflow'], - ] - expected = { - 'py2': { - 'status': 'SUCCESS', 'details': {} - }, - 'py3': { - 'status': 'SUCCESS', 'details': {} - } - } - cr_py2 = compatibility_store.CompatibilityResult( - packages=[PACKAGE_1, PACKAGE_2], - python_major_version=2, - status=compatibility_store.Status.INSTALL_ERROR) - cr_py3 = compatibility_store.CompatibilityResult( - packages=[PACKAGE_1, PACKAGE_2], - python_major_version=3, - status=compatibility_store.Status.SUCCESS) - pair_result = [cr_py2, cr_py3] - self.fake_store._packages_to_compatibility_result[ - frozenset([PACKAGE_1, PACKAGE_2])] = pair_result - - with self.patch_checker, self.patch_store: - version_and_res = main._get_pair_status_for_packages( - pkg_sets) - - self.assertEqual(version_and_res, expected) - - def test__get_pair_status_for_packages_self_conflict(self): - # If the pair package is not self compatible, the package being checked - # should not be marked as CHECK_WARNING. - from compatibility_lib import compatibility_store - from compatibility_lib import package - - PACKAGE_1 = package.Package("package1") - PACKAGE_2 = package.Package("tensorflow") - - pkg_sets = [ - ['package1', 'tensorflow'], - ] - expected = { - 'py2': { - 'status': 'SUCCESS', 'details': {} - }, - 'py3': { - 'status': 'SUCCESS', 'details': {} - } - } - cr_py2 = compatibility_store.CompatibilityResult( - packages=[PACKAGE_1, PACKAGE_2], - python_major_version=2, - status=compatibility_store.Status.CHECK_WARNING) - cr_py3 = compatibility_store.CompatibilityResult( - packages=[PACKAGE_1, PACKAGE_2], - python_major_version=3, - status=compatibility_store.Status.CHECK_WARNING) - pair_result = [cr_py2, cr_py3] - self.fake_store._packages_to_compatibility_result[ - frozenset([PACKAGE_1, PACKAGE_2])] = pair_result - - mock_self_res = mock.Mock() - self_res = { - 'py2': { - 'status': 'CHECK_WARNING', 'details': {} - }, - 'py3': { - 'status': 'CHECK_WARNING', 'details': {} - } - } - mock_self_res.return_value = self_res - patch_self_status = mock.patch( - 'main._get_result_from_cache', - mock_self_res) - - with self.patch_checker, self.patch_store, patch_self_status: - version_and_res = main._get_pair_status_for_packages( - pkg_sets) - - self.assertEqual(version_and_res, expected) - - def test__get_badge_use_py2(self): package_name = 'package-1' res = { From a82befeffccc73c3ade2129b84686cd50ec8407f Mon Sep 17 00:00:00 2001 From: Angela Li Date: Tue, 5 Feb 2019 15:32:46 -0800 Subject: [PATCH 2/4] Run checks for both pypi and github head versions --- badge_server/get_badge_data.py | 0 .../compatibility_checker.py | 33 ++++++++--- .../compatibility_lib/compatibility_store.py | 3 + .../compatibility_lib/configs.py | 4 +- .../get_compatibility_data.py | 57 ++++++++++++++++--- system_test/test_badge_server.py | 36 ------------ 6 files changed, 80 insertions(+), 53 deletions(-) delete mode 100644 badge_server/get_badge_data.py diff --git a/badge_server/get_badge_data.py b/badge_server/get_badge_data.py deleted file mode 100644 index e69de29b..00000000 diff --git a/compatibility_lib/compatibility_lib/compatibility_checker.py b/compatibility_lib/compatibility_lib/compatibility_checker.py index a228b95e..709f353c 100644 --- a/compatibility_lib/compatibility_lib/compatibility_checker.py +++ b/compatibility_lib/compatibility_lib/compatibility_checker.py @@ -37,6 +37,7 @@ def __init__(self, max_workers=20): def check(self, packages, python_version): """Call the checker server to get back status results.""" + print(packages) if not utils._is_package_in_whitelist(packages): UNKNOWN_STATUS_RESULT['packages'] = packages @@ -50,12 +51,22 @@ def check(self, packages, python_version): } result = requests.get(SERVER_URL, params=data) content = result.content.decode('utf-8') + print('Done', packages) return json.loads(content) def filter_packages(self, packages, python_version): - return [pkg for pkg in packages if pkg not in - configs.PKG_PY_VERSION_NOT_SUPPORTED[int(python_version)]] + """Filter out the packages not supported by the given py version.""" + filtered_packages = [] + for pkg in packages: + if 'github.com' in pkg: + pkg_name = configs.WHITELIST_URLS[pkg] + else: + pkg_name = pkg + if pkg_name not in configs.PKG_PY_VERSION_NOT_SUPPORTED[ + int(python_version)]: + filtered_packages.append(pkg) + return filtered_packages @retrying.retry(wait_exponential_multiplier=5000, wait_exponential_max=20000) @@ -69,8 +80,10 @@ def get_self_compatibility(self, python_version, packages=None): """Get the self compatibility data for each package.""" if packages is None: packages = configs.PKG_LIST - # Remove the package not supported in the python_version - packages = self.filter_packages(packages, python_version) + + # Remove the package not supported in the python_version + packages = self.filter_packages(packages, python_version) + with concurrent.futures.ThreadPoolExecutor( max_workers=self.max_workers) as p: pkg_set_results = p.map( @@ -83,13 +96,19 @@ def get_self_compatibility(self, python_version, packages=None): def get_pairwise_compatibility(self, python_version, pkg_sets=None): """Get pairwise compatibility data for each pair of packages.""" if pkg_sets is None: - packages = self.filter_packages(configs.PKG_LIST, python_version) - pkg_sets = itertools.combinations(packages, 2) + pkg_sets = itertools.combinations(configs.PKG_LIST, 2) + + filtered_pkgs = [] + for pkgs in pkg_sets: + if list(pkgs) != self.filter_packages(pkgs, python_version): + continue + filtered_pkgs.append(pkgs) + with concurrent.futures.ThreadPoolExecutor( max_workers=self.max_workers) as p: pkg_set_results = p.map( self.retrying_check, - ((list(pkg_set), python_version) for pkg_set in pkg_sets)) + ((list(pkg_set), python_version) for pkg_set in filtered_pkgs)) for result in zip(pkg_set_results): yield result diff --git a/compatibility_lib/compatibility_lib/compatibility_store.py b/compatibility_lib/compatibility_lib/compatibility_store.py index 4dac76cc..2222e33a 100644 --- a/compatibility_lib/compatibility_lib/compatibility_store.py +++ b/compatibility_lib/compatibility_lib/compatibility_store.py @@ -24,6 +24,7 @@ from google.cloud import bigquery from google.cloud.bigquery import table +from compatibility_lib import configs from compatibility_lib import package _DATASET_NAME = 'compatibility_checker' @@ -478,6 +479,8 @@ def _get_package_version(self, result: CompatibilityResult) -> str: raise ValueError('multiple packages found in CompatibilityResult') install_name = result.packages[0].install_name + if 'github.com' in install_name: + install_name = configs.WHITELIST_URLS[install_name] install_name_sanitized = install_name.split('[')[0] for pkg, version_info in result.dependency_info.items(): diff --git a/compatibility_lib/compatibility_lib/configs.py b/compatibility_lib/compatibility_lib/configs.py index d44ff340..bb13cff1 100644 --- a/compatibility_lib/compatibility_lib/configs.py +++ b/compatibility_lib/compatibility_lib/configs.py @@ -171,18 +171,16 @@ def _format_url(repo_name, setuppy_path=''): _format_url('google/apitools'): 'google-apitools', _format_url('GoogleCloudPlatform/gsutil'): 'gsutil', _format_url('census-instrumentation/opencensus-python'): 'opencensus', - _format_url('protocolbuffers/protobuf', 'python'): 'protobuf', _format_url('google/protorpc'): 'protorpc', _format_url('tensorflow/tensorflow', 'tensorflow/tools/pip_package'): 'tensorflow', - _format_url('tensorflow/tensorflow', - 'tensorflow/contrib/tpu/profiler/pip_package'): 'tensorflow', _format_url('GoogleCloudPlatform/cloud-opensource-python', 'compatibility_lib'): 'compatibility-lib', # TODO: The following projects do not use setup.py # googleapis-common-protos # grpc-google-iam-v1 # grpcio + # protobuf # tensorboard - not sure what the build process is # _format_url('tensorflow/tensorboard', 'tensorboard/pip_package'): # 'tensorboard', diff --git a/compatibility_lib/compatibility_lib/get_compatibility_data.py b/compatibility_lib/compatibility_lib/get_compatibility_data.py index 88deaae0..4736f151 100644 --- a/compatibility_lib/compatibility_lib/get_compatibility_data.py +++ b/compatibility_lib/compatibility_lib/get_compatibility_data.py @@ -14,14 +14,16 @@ """Get self and pairwise compatibility data and write to bigquery.""" +import argparse import datetime +import itertools from compatibility_lib import compatibility_checker from compatibility_lib import compatibility_store from compatibility_lib import configs from compatibility_lib import package -checker = compatibility_checker.CompatibilityChecker() +checker = compatibility_checker.CompatibilityChecker(max_workers=800) store = compatibility_store.CompatibilityStore() PY2 = '2' @@ -53,15 +55,40 @@ def _result_dict_to_compatibility_result(results, python_version): return res_list -def write_to_status_table(): +def get_package_pairs(check_pypi, check_github): + """Get package pairs for pypi and github head.""" + self_packages = [] + pair_packages = [] + if check_pypi: + # Get pypi packages for single checks + self_packages.extend(configs.PKG_LIST) + # Get pypi packages for pairwise checks + pypi_pairs = list(itertools.combinations(configs.PKG_LIST, 2)) + pair_packages.extend(pypi_pairs) + if check_github: + # Get github head packages for single checks + self_packages.extend(list(configs.WHITELIST_URLS.keys())) + # Get github head packages for pairwise checks + for gh_url in configs.WHITELIST_URLS: + pairs = [] + gh_name = configs.WHITELIST_URLS[gh_url] + for pypi_pkg in configs.PKG_LIST: + if pypi_pkg != gh_name: + pairs.append((gh_url, pypi_pkg)) + pair_packages.extend(pairs) + + return self_packages, pair_packages + + +def write_to_status_table(check_pypi, check_github): """Get the compatibility status for PyPI versions.""" # Write self compatibility status to BigQuery + self_packages, pair_packages = get_package_pairs(check_pypi, check_github) self_res_list = [] - packages = configs.PKG_LIST for py_version in [PY2, PY3]: results = checker.get_self_compatibility( python_version=py_version, - packages=packages) + packages=self_packages) res_list = _result_dict_to_compatibility_result(results, py_version) self_res_list.extend(res_list) @@ -69,11 +96,27 @@ def write_to_status_table(): # Write pairwise compatibility status to BigQuery for py_version in [PY2, PY3]: - # For PyPI released versions - results = checker.get_pairwise_compatibility(py_version) + # For PyPI released versions and github head versions + results = checker.get_pairwise_compatibility( + python_version=py_version, pkg_sets=pair_packages) res_list = _result_dict_to_compatibility_result(results, py_version) store.save_compatibility_statuses(res_list) if __name__ == '__main__': - write_to_status_table() + parser = argparse.ArgumentParser(description='Determine what to check.') + parser.add_argument( + '--pypi', + type=bool, + default=False, + help='Check PyPI released packages or not.') + parser.add_argument( + '--github', + type=bool, + default=False, + help='Check GitHub head packages or not.') + args = parser.parse_args() + + check_pypi = args.pypi + check_github = args.github + write_to_status_table(check_pypi, check_github) diff --git a/system_test/test_badge_server.py b/system_test/test_badge_server.py index 130a3a84..1d4b320f 100644 --- a/system_test/test_badge_server.py +++ b/system_test/test_badge_server.py @@ -64,42 +64,6 @@ def tearDown(self): # Kill the application process os.killpg(os.getpgid(self.process.pid), signal.SIGTERM) - @retry(wait_fixed=RETRY_WAIT_PERIOD, - stop_max_attempt_number=RETRY_MAX_ATTEMPT) - def test_self_compatibility_badge(self): - response = requests.get( - '{}self_compatibility_badge_image?package={}'.format( - BASE_URL, PACKAGE_FOR_TEST)) - status_code = response.status_code - content = response.content - - self.assertEqual(status_code, 200) - self.assertIn(b"CALCULATING", content) - - @retry(wait_fixed=RETRY_WAIT_PERIOD, - stop_max_attempt_number=RETRY_MAX_ATTEMPT) - def test_google_compatibility_badge(self): - response = requests.get( - '{}google_compatibility_badge_image?package={}'.format( - BASE_URL, PACKAGE_FOR_TEST)) - status_code = response.status_code - content = response.content - - self.assertEqual(status_code, 200) - self.assertIn(b"CALCULATING", content) - - @retry(wait_fixed=RETRY_WAIT_PERIOD, - stop_max_attempt_number=RETRY_MAX_ATTEMPT) - def test_self_dependency_badge(self): - response = requests.get( - '{}self_dependency_badge_image?package={}'.format( - BASE_URL, PACKAGE_FOR_TEST)) - status_code = response.status_code - content = response.content - - self.assertEqual(status_code, 200) - self.assertIn(b"CALCULATING", content) - @retry(wait_fixed=RETRY_WAIT_PERIOD, stop_max_attempt_number=RETRY_MAX_ATTEMPT) def test_one_badge(self): From 2ce194b12854bba9a1b97320a6b0bc847eb727e6 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Tue, 5 Feb 2019 15:39:43 -0800 Subject: [PATCH 3/4] fix --- compatibility_lib/compatibility_lib/compatibility_checker.py | 2 +- .../compatibility_lib/test_get_compatibility_data.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/compatibility_lib/compatibility_lib/compatibility_checker.py b/compatibility_lib/compatibility_lib/compatibility_checker.py index 709f353c..8ae94c8d 100644 --- a/compatibility_lib/compatibility_lib/compatibility_checker.py +++ b/compatibility_lib/compatibility_lib/compatibility_checker.py @@ -64,7 +64,7 @@ def filter_packages(self, packages, python_version): else: pkg_name = pkg if pkg_name not in configs.PKG_PY_VERSION_NOT_SUPPORTED[ - int(python_version)]: + int(python_version)]: filtered_packages.append(pkg) return filtered_packages diff --git a/compatibility_lib/compatibility_lib/test_get_compatibility_data.py b/compatibility_lib/compatibility_lib/test_get_compatibility_data.py index ce5647c3..106b4791 100644 --- a/compatibility_lib/compatibility_lib/test_get_compatibility_data.py +++ b/compatibility_lib/compatibility_lib/test_get_compatibility_data.py @@ -92,7 +92,8 @@ def test_write_to_status_table(self): with self.patch_checker, self.patch_store: from compatibility_lib import get_compatibility_data - get_compatibility_data.write_to_status_table() + get_compatibility_data.write_to_status_table( + check_pypi=True, check_github=False) self.assertTrue(self.mock_checker.get_self_compatibility.called) self.assertTrue(self.mock_checker.get_pairwise_compatibility.called) From 465fa827082263a282aeeda65282ff87b8a2bf28 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Thu, 7 Feb 2019 14:55:10 -0800 Subject: [PATCH 4/4] generating singles and pairs and run them together --- .../compatibility_checker.py | 70 ++++++++++++------- .../compatibility_lib/compatibility_store.py | 4 +- .../get_compatibility_data.py | 33 +++------ 3 files changed, 57 insertions(+), 50 deletions(-) diff --git a/compatibility_lib/compatibility_lib/compatibility_checker.py b/compatibility_lib/compatibility_lib/compatibility_checker.py index 8ae94c8d..e12e52c6 100644 --- a/compatibility_lib/compatibility_lib/compatibility_checker.py +++ b/compatibility_lib/compatibility_lib/compatibility_checker.py @@ -19,6 +19,7 @@ import json import requests import retrying +import time from compatibility_lib import configs from compatibility_lib import utils @@ -38,6 +39,7 @@ def __init__(self, max_workers=20): def check(self, packages, python_version): """Call the checker server to get back status results.""" print(packages) + start = time.time() if not utils._is_package_in_whitelist(packages): UNKNOWN_STATUS_RESULT['packages'] = packages @@ -51,9 +53,9 @@ def check(self, packages, python_version): } result = requests.get(SERVER_URL, params=data) content = result.content.decode('utf-8') - print('Done', packages) + print(round(time.time() - start, 2), packages) - return json.loads(content) + return json.loads(content), python_version def filter_packages(self, packages, python_version): """Filter out the packages not supported by the given py version.""" @@ -68,47 +70,63 @@ def filter_packages(self, packages, python_version): filtered_packages.append(pkg) return filtered_packages - @retrying.retry(wait_exponential_multiplier=5000, - wait_exponential_max=20000) + @retrying.retry(wait_random_min=1000, + wait_random_max=2000) def retrying_check(self, args): """Retrying logic for sending requests to checker server.""" packages = args[0] python_version = args[1] return self.check(packages, python_version) - def get_self_compatibility(self, python_version, packages=None): - """Get the self compatibility data for each package.""" + def collect_check_packages( + self, python_version=None, packages=None, pkg_sets=None): + # Generating single packages if packages is None: packages = configs.PKG_LIST - # Remove the package not supported in the python_version - packages = self.filter_packages(packages, python_version) - - with concurrent.futures.ThreadPoolExecutor( - max_workers=self.max_workers) as p: - pkg_set_results = p.map( - self.retrying_check, - (([pkg], python_version) for pkg in packages)) - - for result in zip(pkg_set_results): - yield result - - def get_pairwise_compatibility(self, python_version, pkg_sets=None): - """Get pairwise compatibility data for each pair of packages.""" + check_singles = [] + if python_version is None: + for py_ver in ['2', '3']: + # Remove the package not supported in the python_version + packages = self.filter_packages(packages, py_ver) + for pkg in packages: + check_singles.append(([pkg], py_ver)) + else: + check_singles = [([pkg], python_version) for pkg in packages] + + # Generating pairs if pkg_sets is None: pkg_sets = itertools.combinations(configs.PKG_LIST, 2) - filtered_pkgs = [] - for pkgs in pkg_sets: - if list(pkgs) != self.filter_packages(pkgs, python_version): - continue - filtered_pkgs.append(pkgs) + check_pairs = [] + if python_version is None: + for py_ver in ['2', '3']: + filtered_pkgs = [] + for pkgs in pkg_sets: + if list(pkgs) != self.filter_packages(pkgs, + py_ver): + continue + filtered_pkgs.append(pkgs) + for pkg_set in filtered_pkgs: + check_pairs.append((list(pkg_set), py_ver)) + else: + check_pairs = [(list(pkg_set), python_version) \ + for pkg_set in pkg_sets] + + res = tuple(check_singles) + tuple(check_pairs) + return res + + def get_compatibility( + self, python_version=None, packages=None, pkg_sets=None): + """Get the compatibility data for each package and package pairs.""" + check_packages = self.collect_check_packages( + python_version, packages, pkg_sets) with concurrent.futures.ThreadPoolExecutor( max_workers=self.max_workers) as p: pkg_set_results = p.map( self.retrying_check, - ((list(pkg_set), python_version) for pkg_set in filtered_pkgs)) + tuple(check_packages)) for result in zip(pkg_set_results): yield result diff --git a/compatibility_lib/compatibility_lib/compatibility_store.py b/compatibility_lib/compatibility_lib/compatibility_store.py index 2222e33a..f0a0cd57 100644 --- a/compatibility_lib/compatibility_lib/compatibility_store.py +++ b/compatibility_lib/compatibility_lib/compatibility_store.py @@ -439,8 +439,8 @@ def save_compatibility_statuses( install_name_to_compatibility_result[install_name]) new_version_string = self._get_package_version(cs) - old_version = version.StrictVersion(old_version_string) - new_version = version.StrictVersion(new_version_string) + old_version = version.LooseVersion(old_version_string) + new_version = version.LooseVersion(new_version_string) if new_version > old_version: install_name_to_compatibility_result[install_name] = cs diff --git a/compatibility_lib/compatibility_lib/get_compatibility_data.py b/compatibility_lib/compatibility_lib/get_compatibility_data.py index 4736f151..9bfca120 100644 --- a/compatibility_lib/compatibility_lib/get_compatibility_data.py +++ b/compatibility_lib/compatibility_lib/get_compatibility_data.py @@ -30,17 +30,18 @@ PY3 = '3' -def _result_dict_to_compatibility_result(results, python_version): +def _result_dict_to_compatibility_result(results): res_list = [] for item in results: res_dict = item[0] - check_result = res_dict.get('result') + result_content, python_version = res_dict + check_result = result_content.get('result') packages_list = [package.Package(pkg) - for pkg in res_dict.get('packages')] - details = res_dict.get('description') + for pkg in result_content.get('packages')] + details = result_content.get('description') timestamp = datetime.datetime.now().isoformat() - dependency_info = res_dict.get('dependency_info') + dependency_info = result_content.get('dependency_info') compatibility_result = compatibility_store.CompatibilityResult( packages=packages_list, @@ -84,23 +85,11 @@ def write_to_status_table(check_pypi, check_github): """Get the compatibility status for PyPI versions.""" # Write self compatibility status to BigQuery self_packages, pair_packages = get_package_pairs(check_pypi, check_github) - self_res_list = [] - for py_version in [PY2, PY3]: - results = checker.get_self_compatibility( - python_version=py_version, - packages=self_packages) - res_list = _result_dict_to_compatibility_result(results, py_version) - self_res_list.extend(res_list) - - store.save_compatibility_statuses(self_res_list) - - # Write pairwise compatibility status to BigQuery - for py_version in [PY2, PY3]: - # For PyPI released versions and github head versions - results = checker.get_pairwise_compatibility( - python_version=py_version, pkg_sets=pair_packages) - res_list = _result_dict_to_compatibility_result(results, py_version) - store.save_compatibility_statuses(res_list) + results = checker.get_compatibility( + packages=self_packages, pkg_sets=pair_packages) + res_list = _result_dict_to_compatibility_result(results) + + store.save_compatibility_statuses(res_list) if __name__ == '__main__':