From 2da57e05af9933829828e73a85feb570c1337c13 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 30 Oct 2025 19:39:42 +0000 Subject: [PATCH 1/4] chore(librarian): onboard to librarian, including generate --- .../generator-input/.repo-metadata.json | 18 + .librarian/generator-input/noxfile.py | 556 ++++++++++++++++++ .librarian/generator-input/owlbot.py | 352 +++++++++++ .librarian/generator-input/setup.py | 104 ++++ .librarian/state.yaml | 10 +- noxfile.py | 4 +- owlbot.py | 2 +- 7 files changed, 1041 insertions(+), 5 deletions(-) create mode 100644 .librarian/generator-input/.repo-metadata.json create mode 100644 .librarian/generator-input/noxfile.py create mode 100644 .librarian/generator-input/owlbot.py create mode 100644 .librarian/generator-input/setup.py diff --git a/.librarian/generator-input/.repo-metadata.json b/.librarian/generator-input/.repo-metadata.json new file mode 100644 index 000000000..8d12e4cc0 --- /dev/null +++ b/.librarian/generator-input/.repo-metadata.json @@ -0,0 +1,18 @@ +{ + "name": "pubsub", + "name_pretty": "Google Cloud Pub/Sub", + "product_documentation": "https://cloud.google.com/pubsub/docs/", + "client_documentation": "https://cloud.google.com/python/docs/reference/pubsub/latest", + "issue_tracker": "https://issuetracker.google.com/savedsearches/559741", + "release_level": "stable", + "language": "python", + "repo": "googleapis/python-pubsub", + "distribution_name": "google-cloud-pubsub", + "api_id": "pubsub.googleapis.com", + "requires_billing": true, + "default_version": "v1", + "codeowner_team": "@googleapis/api-pubsub", + "api_shortname": "pubsub", + "library_type": "GAPIC_COMBO", + "api_description": "is designed to provide reliable, many-to-many, asynchronous messaging between applications. Publisher applications can send messages to a topic and other applications can subscribe to that topic to receive the messages. By decoupling senders and receivers, Google Cloud Pub/Sub allows developers to communicate between independently written applications." +} diff --git a/.librarian/generator-input/noxfile.py b/.librarian/generator-input/noxfile.py new file mode 100644 index 000000000..fd552166c --- /dev/null +++ b/.librarian/generator-input/noxfile.py @@ -0,0 +1,556 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generated by synthtool. DO NOT EDIT! + +from __future__ import absolute_import + +import os +import pathlib +import re +import shutil +from typing import Dict, List +import warnings + +import nox + +FLAKE8_VERSION = "flake8==6.1.0" +BLACK_VERSION = "black[jupyter]==23.7.0" +ISORT_VERSION = "isort==5.11.0" +LINT_PATHS = ["google", "tests", "noxfile.py", "setup.py"] + +MYPY_VERSION = "mypy==1.10.0" + +DEFAULT_PYTHON_VERSION = "3.14" + +UNIT_TEST_PYTHON_VERSIONS: List[str] = [ + "3.7", + "3.8", + "3.9", + "3.10", + "3.11", + "3.12", + "3.13", + "3.14", +] +UNIT_TEST_STANDARD_DEPENDENCIES = [ + "mock", + "asyncmock", + "pytest", + "pytest-cov", + "pytest-asyncio", +] +UNIT_TEST_EXTERNAL_DEPENDENCIES: List[str] = [] +UNIT_TEST_LOCAL_DEPENDENCIES: List[str] = [] +UNIT_TEST_DEPENDENCIES: List[str] = [ + "flaky", +] +UNIT_TEST_EXTRAS: List[str] = [] +UNIT_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {} + +SYSTEM_TEST_PYTHON_VERSIONS: List[str] = ["3.12"] +SYSTEM_TEST_STANDARD_DEPENDENCIES: List[str] = [ + "mock", + "pytest", + "google-cloud-testutils", +] +SYSTEM_TEST_EXTERNAL_DEPENDENCIES: List[str] = [ + "psutil", + "flaky", +] +SYSTEM_TEST_LOCAL_DEPENDENCIES: List[str] = [] +SYSTEM_TEST_DEPENDENCIES: List[str] = [] +SYSTEM_TEST_EXTRAS: List[str] = [] +SYSTEM_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {} + +CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() + +nox.options.sessions = [ + "unit", + "system", + "cover", + "lint", + "lint_setup_py", + "blacken", + "mypy", + # https://github.com/googleapis/python-pubsub/pull/552#issuecomment-1016256936 + # "mypy_samples", # TODO: uncomment when the check passes + "docs", + "docfx", + "format", +] + +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def mypy(session): + """Run type checks with mypy.""" + session.install("-e", ".[all]") + session.install(MYPY_VERSION) + + # Version 2.1.1 of google-api-core version is the first type-checked release. + # Version 2.2.0 of google-cloud-core version is the first type-checked release. + session.install( + "google-api-core[grpc]>=2.1.1", "google-cloud-core>=2.2.0", "types-requests" + ) + + # Just install the type info directly, since "mypy --install-types" might + # require an additional pass. + # Exclude types-protobuf==4.24.0.20240106 + # See https://github.com/python/typeshed/issues/11254 + session.install("types-protobuf!=4.24.0.20240106", "types-setuptools") + + # TODO: Only check the hand-written layer, the generated code does not pass + # mypy checks yet. + # https://github.com/googleapis/gapic-generator-python/issues/1092 + # TODO: Re-enable mypy checks once we merge, since incremental checks are failing due to protobuf upgrade + # session.run("mypy", "-p", "google.cloud", "--exclude", "google/pubsub_v1/") + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def mypy_samples(session): + """Run type checks with mypy.""" + + session.install("-e", ".[all]") + + session.install("pytest") + session.install(MYPY_VERSION) + + # Just install the type info directly, since "mypy --install-types" might + # require an additional pass. + session.install( + "types-mock", "types-protobuf", "types-setuptools", "types-requests" + ) + + session.run( + "mypy", + "--config-file", + str(CURRENT_DIRECTORY / "samples" / "snippets" / "mypy.ini"), + "--no-incremental", # Required by warn-unused-configs from mypy.ini to work + "samples/", + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def lint(session): + """Run linters. + + Returns a failure if the linters find linting errors or sufficiently + serious code quality issues. + """ + session.install(FLAKE8_VERSION, BLACK_VERSION) + session.run( + "black", + "--check", + *LINT_PATHS, + ) + session.run("flake8", "google", "tests") + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def blacken(session): + """Run black. Format code to uniform standard.""" + session.install(BLACK_VERSION) + session.run( + "black", + *LINT_PATHS, + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def format(session): + """ + Run isort to sort imports. Then run black + to format code to uniform standard. + """ + session.install(BLACK_VERSION, ISORT_VERSION) + # Use the --fss option to sort imports using strict alphabetical order. + # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + session.run( + "isort", + "--fss", + *LINT_PATHS, + ) + session.run( + "black", + *LINT_PATHS, + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def lint_setup_py(session): + """Verify that setup.py is valid (including RST check).""" + session.install("setuptools", "docutils", "pygments") + session.run("python", "setup.py", "check", "--restructuredtext", "--strict") + + +def install_unittest_dependencies(session, *constraints): + standard_deps = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_DEPENDENCIES + session.install(*standard_deps, *constraints) + + if UNIT_TEST_EXTERNAL_DEPENDENCIES: + warnings.warn( + "'unit_test_external_dependencies' is deprecated. Instead, please " + "use 'unit_test_dependencies' or 'unit_test_local_dependencies'.", + DeprecationWarning, + ) + session.install(*UNIT_TEST_EXTERNAL_DEPENDENCIES, *constraints) + + if UNIT_TEST_LOCAL_DEPENDENCIES: + session.install(*UNIT_TEST_LOCAL_DEPENDENCIES, *constraints) + + if UNIT_TEST_EXTRAS_BY_PYTHON: + extras = UNIT_TEST_EXTRAS_BY_PYTHON.get(session.python, []) + elif UNIT_TEST_EXTRAS: + extras = UNIT_TEST_EXTRAS + else: + extras = [] + + if extras: + session.install("-e", f".[{','.join(extras)}]", *constraints) + else: + session.install("-e", ".", *constraints) + + +@nox.session(python=UNIT_TEST_PYTHON_VERSIONS) +@nox.parametrize( + "protobuf_implementation", + ["python", "upb", "cpp"], +) +def unit(session, protobuf_implementation): + # Install all test dependencies, then install this package in-place. + + if protobuf_implementation == "cpp" and session.python in ( + "3.11", + "3.12", + "3.13", + "3.14", + ): + session.skip("cpp implementation is not supported in python 3.11+") + + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + install_unittest_dependencies(session, "-c", constraints_path) + + # TODO(https://github.com/googleapis/synthtool/issues/1976): + # Remove the 'cpp' implementation once support for Protobuf 3.x is dropped. + # The 'cpp' implementation requires Protobuf<4. + if protobuf_implementation == "cpp": + session.install("protobuf<4") + + # Run py.test against the unit tests. + session.run( + "py.test", + "--quiet", + f"--junitxml=unit_{session.python}_sponge_log.xml", + "--cov=google/cloud", + "--cov=tests/unit", + "--cov-append", + "--cov-config=.coveragerc", + "--cov-report=", + "--cov-fail-under=0", + os.path.join("tests", "unit"), + *session.posargs, + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + }, + ) + + +def install_systemtest_dependencies(session, *constraints): + # Use pre-release gRPC for system tests. + # Exclude version 1.52.0rc1 which has a known issue. + # See https://github.com/grpc/grpc/issues/32163 + session.install("--pre", "grpcio!=1.52.0rc1") + + session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_EXTERNAL_DEPENDENCIES: + session.install(*SYSTEM_TEST_EXTERNAL_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_LOCAL_DEPENDENCIES: + session.install("-e", *SYSTEM_TEST_LOCAL_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_DEPENDENCIES: + session.install("-e", *SYSTEM_TEST_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_EXTRAS_BY_PYTHON: + extras = SYSTEM_TEST_EXTRAS_BY_PYTHON.get(session.python, []) + elif SYSTEM_TEST_EXTRAS: + extras = SYSTEM_TEST_EXTRAS + else: + extras = [] + + if extras: + session.install("-e", f".[{','.join(extras)}]", *constraints) + else: + session.install("-e", ".", *constraints) + + +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +def system(session): + """Run the system test suite.""" + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + system_test_path = os.path.join("tests", "system.py") + system_test_folder_path = os.path.join("tests", "system") + + # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. + if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false": + session.skip("RUN_SYSTEM_TESTS is set to false, skipping") + # Install pyopenssl for mTLS testing. + if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": + session.install("pyopenssl") + + system_test_exists = os.path.exists(system_test_path) + system_test_folder_exists = os.path.exists(system_test_folder_path) + # Sanity check: only run tests if found. + if not system_test_exists and not system_test_folder_exists: + session.skip("System tests were not found") + + install_systemtest_dependencies(session, "-c", constraints_path) + + # Run py.test against the system tests. + if system_test_exists: + session.run( + "py.test", + "--verbose", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_path, + *session.posargs, + ) + if os.path.exists(system_test_folder_path): + session.run( + "py.test", + "--verbose", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_folder_path, + *session.posargs, + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def cover(session): + """Run the final coverage report. + + This outputs the coverage report aggregating coverage from the unit + test runs (not system test runs), and then erases coverage data. + """ + session.install("coverage", "pytest-cov") + session.run("coverage", "report", "--show-missing", "--fail-under=99") + + session.run("coverage", "erase") + + +# py > 3.10 not supported yet +@nox.session(python="3.10") +def docs(session): + """Build the docs for this library.""" + + session.install("-e", ".") + session.install( + # We need to pin to specific versions of the `sphinxcontrib-*` packages + # which still support sphinx 4.x. + # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 + # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. + "sphinxcontrib-applehelp==1.0.4", + "sphinxcontrib-devhelp==1.0.2", + "sphinxcontrib-htmlhelp==2.0.1", + "sphinxcontrib-qthelp==1.0.3", + "sphinxcontrib-serializinghtml==1.1.5", + "sphinx==4.5.0", + "alabaster", + "recommonmark", + ) + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run( + "sphinx-build", + "-W", # warnings as errors + "-T", # show full traceback on exception + "-N", # no colors + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) + + +# py > 3.10 not supported yet +@nox.session(python="3.10") +def docfx(session): + """Build the docfx yaml files for this library.""" + + session.install("-e", ".") + session.install( + # We need to pin to specific versions of the `sphinxcontrib-*` packages + # which still support sphinx 4.x. + # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 + # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. + "sphinxcontrib-applehelp==1.0.4", + "sphinxcontrib-devhelp==1.0.2", + "sphinxcontrib-htmlhelp==2.0.1", + "sphinxcontrib-qthelp==1.0.3", + "sphinxcontrib-serializinghtml==1.1.5", + "gcp-sphinx-docfx-yaml", + "alabaster", + "recommonmark", + ) + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run( + "sphinx-build", + "-T", # show full traceback on exception + "-N", # no colors + "-D", + ( + "extensions=sphinx.ext.autodoc," + "sphinx.ext.autosummary," + "docfx_yaml.extension," + "sphinx.ext.intersphinx," + "sphinx.ext.coverage," + "sphinx.ext.napoleon," + "sphinx.ext.todo," + "sphinx.ext.viewcode," + "recommonmark" + ), + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) + + +@nox.session(python="3.14") +@nox.parametrize( + "protobuf_implementation", + ["python", "upb", "cpp"], +) +def prerelease_deps(session, protobuf_implementation): + """Run all tests with prerelease versions of dependencies installed.""" + + if protobuf_implementation == "cpp" and session.python in ( + "3.11", + "3.12", + "3.13", + "3.14", + ): + session.skip("cpp implementation is not supported in python 3.11+") + + # Install all dependencies + session.install("-e", ".[all, tests, tracing]") + unit_deps_all = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_EXTERNAL_DEPENDENCIES + session.install(*unit_deps_all) + system_deps_all = ( + SYSTEM_TEST_STANDARD_DEPENDENCIES + SYSTEM_TEST_EXTERNAL_DEPENDENCIES + ) + session.install(*system_deps_all) + + # Because we test minimum dependency versions on the minimum Python + # version, the first version we test with in the unit tests sessions has a + # constraints file containing all dependencies and extras. + with open( + CURRENT_DIRECTORY + / "testing" + / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", + encoding="utf-8", + ) as constraints_file: + constraints_text = constraints_file.read() + + # Ignore leading whitespace and comment lines. + constraints_deps = [ + match.group(1) + for match in re.finditer( + r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE + ) + ] + + session.install(*constraints_deps) + + prerel_deps = [ + "protobuf", + # dependency of grpc + "six", + "grpc-google-iam-v1", + "googleapis-common-protos", + "grpcio", + "grpcio-status", + "google-api-core", + "google-auth", + "proto-plus", + "google-cloud-testutils", + # dependencies of google-cloud-testutils" + "click", + ] + + for dep in prerel_deps: + session.install("--pre", "--no-deps", "--upgrade", dep) + + # Remaining dependencies + other_deps = [ + "requests", + ] + session.install(*other_deps) + + # Print out prerelease package versions + session.run( + "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" + ) + session.run("python", "-c", "import grpc; print(grpc.__version__)") + session.run("python", "-c", "import google.auth; print(google.auth.__version__)") + + session.run( + "py.test", + "tests/unit", + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + }, + ) + + system_test_path = os.path.join("tests", "system.py") + system_test_folder_path = os.path.join("tests", "system") + + # Only run system tests if found. + if os.path.exists(system_test_path): + session.run( + "py.test", + "--verbose", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_path, + *session.posargs, + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + }, + ) + if os.path.exists(system_test_folder_path): + session.run( + "py.test", + "--verbose", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_folder_path, + *session.posargs, + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + }, + ) diff --git a/.librarian/generator-input/owlbot.py b/.librarian/generator-input/owlbot.py new file mode 100644 index 000000000..5e6af7955 --- /dev/null +++ b/.librarian/generator-input/owlbot.py @@ -0,0 +1,352 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from pathlib import Path +import re +import shutil +import textwrap + +import synthtool as s +import synthtool.gcp as gcp +from synthtool.languages import python + +# ---------------------------------------------------------------------------- +# Copy the generated client from the owl-bot staging directory +# ---------------------------------------------------------------------------- + +clean_up_generated_samples = True + +# Load the default version defined in .repo-metadata.json. +default_version = json.load(open(".repo-metadata.json", "rt")).get( + "default_version" +) + +for library in s.get_staging_dirs(default_version): + if clean_up_generated_samples: + shutil.rmtree("samples/generated_samples", ignore_errors=True) + clean_up_generated_samples = False + + # DEFAULT SCOPES and SERVICE_ADDRESS are being used. so let's force them in. + s.replace( + library / f"google/pubsub_{library.name}/services/*er/*client.py", + r"""DEFAULT_ENDPOINT = \"pubsub\.googleapis\.com\"""", + """ + # The scopes needed to make gRPC calls to all of the methods defined in + # this service + _DEFAULT_SCOPES = ( + 'https://www.googleapis.com/auth/cloud-platform', + 'https://www.googleapis.com/auth/pubsub', + ) + + SERVICE_ADDRESS = "pubsub.googleapis.com:443" + \"""The default address of the service.\""" + + \g<0>""", + ) + + # Modify GRPC options in transports. + count = s.replace( + [ + library / f"google/pubsub_{library.name}/services/*/transports/grpc*", + library / f"tests/unit/gapic/pubsub_{library.name}/*", + ], + "options=\[.*?\]", + """options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ("grpc.max_metadata_size", 4 * 1024 * 1024), + ("grpc.keepalive_time_ms", 30000), + ]""", + flags=re.MULTILINE | re.DOTALL, + ) + + if count < 15: + raise Exception("Expected replacements for gRPC channel options not made.") + + # If the emulator is used, force an insecure gRPC channel to avoid SSL errors. + clients_to_patch = [ + library / f"google/pubsub_{library.name}/services/publisher/client.py", + library / f"google/pubsub_{library.name}/services/subscriber/client.py", + library / f"google/pubsub_{library.name}/services/schema_service/client.py", + ] + err_msg = ( + "Expected replacements for gRPC channel to use with the emulator not made." + ) + + count = s.replace(clients_to_patch, r"import os", "import functools\n\g<0>") + + if count < len(clients_to_patch): + raise Exception(err_msg) + + count = s.replace( + clients_to_patch, + f"from \.transports\.base", + "\nimport grpc\n\g<0>", + ) + + if count < len(clients_to_patch): + raise Exception(err_msg) + + # TODO(https://github.com/googleapis/python-pubsub/issues/1349): Move the emulator + # code below to test files. + count = s.replace( + clients_to_patch, + r"# initialize with the provided callable or the passed in class", + """\g<0> + + emulator_host = os.environ.get("PUBSUB_EMULATOR_HOST") + if emulator_host: + if issubclass(transport_init, type(self)._transport_registry["grpc"]): # type: ignore + channel = grpc.insecure_channel(target=emulator_host) + else: + channel = grpc.aio.insecure_channel(target=emulator_host) + transport_init = functools.partial(transport_init, channel=channel) + + """, + ) + + if count < len(clients_to_patch): + raise Exception(err_msg) + + # Monkey patch the streaming_pull() GAPIC method to disable pre-fetching stream + # results. + s.replace( + library / f"google/pubsub_{library.name}/services/subscriber/client.py", + ( + r"# Wrap the RPC method.*\n" + r"\s+# and friendly error.*\n" + r"\s+rpc = self\._transport\._wrapped_methods\[self\._transport\.streaming_pull\]" + ), + """ + # Wrappers in api-core should not automatically pre-fetch the first + # stream result, as this breaks the stream when re-opening it. + # https://github.com/googleapis/python-pubsub/issues/93#issuecomment-630762257 + self._transport.streaming_pull._prefetch_first_result_ = False + + \g<0>""", + ) + + # Emit deprecation warning if return_immediately flag is set with synchronous pull. + s.replace( + library / f"google/pubsub_{library.name}/services/subscriber/*client.py", + r"from google.pubsub_v1 import gapic_version as package_version", + "import warnings\n\g<0>", + ) + + count = s.replace( + library / f"google/pubsub_{library.name}/services/subscriber/*client.py", + r""" + ([^\n\S]+(?:async\ )?def\ pull\(.*?->\ pubsub\.PullResponse:.*?) + ((?P[^\n\S]+)\#\ Wrap\ the\ RPC\ method) + """, + textwrap.dedent( + """ + \g<1> + \gif request.return_immediately: + \g warnings.warn( + \g "The return_immediately flag is deprecated and should be set to False.", + \g category=DeprecationWarning, + \g ) + + \g<2>""" + ), + flags=re.MULTILINE | re.DOTALL | re.VERBOSE, + ) + + if count != 2: + raise Exception("Too many or too few replacements in pull() methods.") + + # Silence deprecation warnings in pull() method flattened parameter tests. + s.replace( + library / f"tests/unit/gapic/pubsub_{library.name}/test_subscriber.py", + "import os", + "\g<0>\nimport warnings", + ) + + count = s.replace( + library / f"tests/unit/gapic/pubsub_{library.name}/test_subscriber.py", + textwrap.dedent( + r""" + ([^\n\S]+# Call the method with a truthy value for each flattened field, + [^\n\S]+# using the keyword arguments to the method\.) + \s+(client\.pull\(.*?\))""" + ), + """\n\g<1> + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=DeprecationWarning) + \g<2>""", + flags=re.MULTILINE | re.DOTALL, + ) + + if count < 1: + raise Exception("Catch warnings replacement failed.") + + count = s.replace( + library / f"tests/unit/gapic/pubsub_{library.name}/test_subscriber.py", + textwrap.dedent( + r""" + ([^\n\S]+# Call the method with a truthy value for each flattened field, + [^\n\S]+# using the keyword arguments to the method\.) + \s+response = (await client\.pull\(.*?\))""" + ), + """\n\g<1> + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=DeprecationWarning) + \g<2>""", + flags=re.MULTILINE | re.DOTALL, + ) + + if count < 1: + raise Exception("Catch warnings replacement failed.") + + # Make sure that client library version is present in user agent header. + count = s.replace( + [ + library + / f"google/pubsub_{library.name}/services/publisher/async_client.py", + library / f"google/pubsub_{library.name}/services/publisher/client.py", + library + / f"google/pubsub_{library.name}/services/publisher/transports/base.py", + library + / f"google/pubsub_{library.name}/services/schema_service/async_client.py", + library / f"google/pubsub_{library.name}/services/schema_service/client.py", + library + / f"google/pubsub_{library.name}/services/schema_service/transports/base.py", + library + / f"google/pubsub_{library.name}/services/subscriber/async_client.py", + library / f"google/pubsub_{library.name}/services/subscriber/client.py", + library + / f"google/pubsub_{library.name}/services/subscriber/transports/base.py", + ], + r"""gapic_version=package_version.__version__""", + "client_library_version=package_version.__version__", + ) + + if count < 1: + raise Exception("client_library_version replacement failed.") + + # Allow timeout to be an instance of google.api_core.timeout.* + count = s.replace( + library / f"google/pubsub_{library.name}/types/__init__.py", + r"from \.pubsub import \(", + "from typing import Union\n\n\g<0>", + ) + + if count < 1: + raise Exception("Catch timeout replacement 1 failed.") + + count = s.replace( + library / f"google/pubsub_{library.name}/types/__init__.py", + r"__all__ = \(\n", + textwrap.dedent( + '''\ + TimeoutType = Union[ + int, + float, + "google.api_core.timeout.ConstantTimeout", + "google.api_core.timeout.ExponentialTimeout", + ] + """The type of the timeout parameter of publisher client methods.""" + + \g<0> "TimeoutType",''' + ), + ) + + if count < 1: + raise Exception("Catch timeout replacement 2 failed.") + + count = s.replace( + library / f"google/pubsub_{library.name}/services/publisher/*client.py", + r"from google.api_core import retry as retries.*\n", + "\g<0>from google.api_core import timeout as timeouts # type: ignore\n", + ) + + if count < 1: + raise Exception("Catch timeout replacement 3 failed.") + + count = s.replace( + library / f"google/pubsub_{library.name}/services/publisher/*client.py", + f"from google\.pubsub_{library.name}\.types import pubsub", + f"\g<0>\nfrom google.pubsub_{library.name}.types import TimeoutType", + ) + + if count < 1: + raise Exception("Catch timeout replacement 4 failed.") + + count = s.replace( + library / f"google/pubsub_{library.name}/services/publisher/*client.py", + r"(\s+)timeout: Union\[float, object\] = gapic_v1.method.DEFAULT.*\n", + f"\g<1>timeout: TimeoutType = gapic_{library.name}.method.DEFAULT,", + ) + + if count < 1: + raise Exception("Catch timeout replacement 5 failed.") + + count = s.replace( + library / f"google/pubsub_{library.name}/services/publisher/*client.py", + r"([^\S\r\n]+)timeout \(float\): (.*)\n", + ("\g<1>timeout (TimeoutType):\n" "\g<1> \g<2>\n"), + ) + + if count < 1: + raise Exception("Catch timeout replacement 6 failed.") + + # Override the default max retry deadline for publisher methods. + count = s.replace( + library / f"google/pubsub_{library.name}/services/publisher/transports/base.py", + r"deadline=60\.0", + "deadline=600.0", + ) + if count < 9: + raise Exception( + "Default retry deadline not overriden for all publisher methods." + ) + + # The namespace package declaration in google/cloud/__init__.py should be excluded + # from coverage. + count = s.replace( + library / ".coveragerc", + "google/pubsub/__init__.py", + """google/cloud/__init__.py + google/pubsub/__init__.py""", + ) + + if count < 1: + raise Exception(".coveragerc replacement failed.") + + s.move([library], excludes=["noxfile.py", "README.rst", "docs/**/*", "setup.py", "testing/constraints-3.7.txt", "testing/constraints-3.8.txt"]) +s.remove_staging_dirs() + +# ---------------------------------------------------------------------------- +# Add templated files +# ---------------------------------------------------------------------------- + +templated_files = gcp.CommonTemplates().py_library( + microgenerator=True, + samples=True, + cov_level=99, + versions=gcp.common.detect_versions(path="./google", default_first=True), + unit_test_python_versions=["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"], + unit_test_dependencies=["flaky"], + system_test_python_versions=["3.12"], + system_test_external_dependencies=["psutil","flaky"], +) +s.move(templated_files, excludes=[".coveragerc", ".github/**", "README.rst", "docs/**", ".kokoro/**"]) + +python.py_samples(skip_readmes=True) + +# run format session for all directories which have a noxfile +for noxfile in Path(".").glob("**/noxfile.py"): + s.shell.run(["nox", "-s", "blacken"], cwd=noxfile.parent, hide_output=False) diff --git a/.librarian/generator-input/setup.py b/.librarian/generator-input/setup.py new file mode 100644 index 000000000..6dbea105a --- /dev/null +++ b/.librarian/generator-input/setup.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import io +import os + +import setuptools # type: ignore + +package_root = os.path.abspath(os.path.dirname(__file__)) + +name = "google-cloud-pubsub" + + +description = "Google Cloud Pub/Sub API client library" + +version = {} +with open(os.path.join(package_root, "google/pubsub/gapic_version.py")) as fp: + exec(fp.read(), version) +version = version["__version__"] + +if version[0] == "0": + release_status = "Development Status :: 4 - Beta" +else: + release_status = "Development Status :: 5 - Production/Stable" + +dependencies = [ + "grpcio >= 1.51.3, < 2.0.0; python_version < '3.14'", # https://github.com/googleapis/python-pubsub/issues/609 + "grpcio >= 1.75.1, < 2.0.0; python_version >= '3.14'", + # google-api-core >= 1.34.0 is allowed in order to support google-api-core 1.x + "google-auth >= 2.14.1, <3.0.0", + "google-api-core[grpc] >= 1.34.0, <3.0.0,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*", + "proto-plus >= 1.22.0, <2.0.0", + "proto-plus >= 1.22.2, <2.0.0; python_version>='3.11'", + "proto-plus >= 1.25.0, < 2.0.0; python_version >= '3.13'", + "protobuf>=3.20.2,<7.0.0,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", + "grpc-google-iam-v1 >= 0.12.4, < 1.0.0", + "grpcio-status >= 1.33.2", + "opentelemetry-api <= 1.22.0; python_version<='3.7'", + "opentelemetry-api >= 1.27.0; python_version>='3.8'", + "opentelemetry-sdk <= 1.22.0; python_version<='3.7'", + "opentelemetry-sdk >= 1.27.0; python_version>='3.8'", +] +extras = {"libcst": "libcst >= 0.3.10"} +url = "https://github.com/googleapis/python-pubsub" + +package_root = os.path.abspath(os.path.dirname(__file__)) + +readme_filename = os.path.join(package_root, "README.rst") +with io.open(readme_filename, encoding="utf-8") as readme_file: + readme = readme_file.read() + +packages = [ + package + for package in setuptools.find_namespace_packages() + if package.startswith("google") +] + +setuptools.setup( + name=name, + version=version, + description=description, + long_description=readme, + author="Google LLC", + author_email="googleapis-packages@google.com", + license="Apache 2.0", + url=url, + classifiers=[ + release_status, + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Operating System :: OS Independent", + "Topic :: Internet", + ], + platforms="Posix; MacOS X; Windows", + packages=packages, + install_requires=dependencies, + extras_require=extras, + python_requires=">=3.7", + scripts=["scripts/fixup_pubsub_v1_keywords.py"], + include_package_data=True, + zip_safe=False, +) diff --git a/.librarian/state.yaml b/.librarian/state.yaml index 4d7702c55..a00bd3709 100644 --- a/.librarian/state.yaml +++ b/.librarian/state.yaml @@ -2,9 +2,15 @@ image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-li libraries: - id: google-cloud-pubsub version: 2.32.0 - apis: [] + last_generated_commit: d300b151a973ce0425ae4ad07b3de957ca31bec6 + apis: + - path: google/pubsub/v1 + service_config: pubsub_v1.yaml source_roots: - . preserve_regex: [] - remove_regex: [] + remove_regex: + - google/pubsub + - google/pubsub_v1 + - tests/unit/gapic tag_format: v{version} diff --git a/noxfile.py b/noxfile.py index 7455daf83..fd552166c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -30,11 +30,11 @@ FLAKE8_VERSION = "flake8==6.1.0" BLACK_VERSION = "black[jupyter]==23.7.0" ISORT_VERSION = "isort==5.11.0" -LINT_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] +LINT_PATHS = ["google", "tests", "noxfile.py", "setup.py"] MYPY_VERSION = "mypy==1.10.0" -DEFAULT_PYTHON_VERSION = "3.13" +DEFAULT_PYTHON_VERSION = "3.14" UNIT_TEST_PYTHON_VERSIONS: List[str] = [ "3.7", diff --git a/owlbot.py b/owlbot.py index abaf534e2..5e6af7955 100644 --- a/owlbot.py +++ b/owlbot.py @@ -326,7 +326,7 @@ if count < 1: raise Exception(".coveragerc replacement failed.") - s.move([library], excludes=["**/gapic_version.py", "noxfile.py", "README.rst", "docs/**/*", "setup.py", "testing/constraints-3.7.txt", "testing/constraints-3.8.txt"]) + s.move([library], excludes=["noxfile.py", "README.rst", "docs/**/*", "setup.py", "testing/constraints-3.7.txt", "testing/constraints-3.8.txt"]) s.remove_staging_dirs() # ---------------------------------------------------------------------------- From bf88f54b6d50078282429bfc1ffc00e85a4365bb Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 30 Oct 2025 19:44:20 +0000 Subject: [PATCH 2/4] update github actions --- .github/workflows/lint.yml | 2 +- .github/workflows/unittest.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 46a3ff38f..6204983fd 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,7 +12,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v6 with: - python-version: "3.13" + python-version: "3.14" - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index d59bbb1b8..04e411304 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -45,7 +45,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.14" - name: Install coverage run: | python -m pip install --upgrade setuptools pip wheel From c9e9a79b5ea99f3526f28dab1e2d534650d0cca1 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 30 Oct 2025 19:48:36 +0000 Subject: [PATCH 3/4] bump version/last generated commit --- .librarian/state.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.librarian/state.yaml b/.librarian/state.yaml index a00bd3709..bfadaea65 100644 --- a/.librarian/state.yaml +++ b/.librarian/state.yaml @@ -1,8 +1,8 @@ image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator:latest libraries: - id: google-cloud-pubsub - version: 2.32.0 - last_generated_commit: d300b151a973ce0425ae4ad07b3de957ca31bec6 + version: 2.33.0 + last_generated_commit: fee5b32df810adbd07d6a20bd97d9239937ef6e4 apis: - path: google/pubsub/v1 service_config: pubsub_v1.yaml From e06f9909550fb4d0b5e35e5507097ecf1bd96292 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 30 Oct 2025 20:43:23 +0000 Subject: [PATCH 4/4] apply generated changes --- google/pubsub/gapic_version.py | 2 +- google/pubsub_v1/gapic_version.py | 2 +- testing/constraints-3.14.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/google/pubsub/gapic_version.py b/google/pubsub/gapic_version.py index 0c5de5c03..1eb64a9d6 100644 --- a/google/pubsub/gapic_version.py +++ b/google/pubsub/gapic_version.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2022 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/google/pubsub_v1/gapic_version.py b/google/pubsub_v1/gapic_version.py index 0c5de5c03..1eb64a9d6 100644 --- a/google/pubsub_v1/gapic_version.py +++ b/google/pubsub_v1/gapic_version.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2022 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/testing/constraints-3.14.txt b/testing/constraints-3.14.txt index 1dba0484d..2ae5a677e 100644 --- a/testing/constraints-3.14.txt +++ b/testing/constraints-3.14.txt @@ -7,7 +7,7 @@ # Then this file should have google-cloud-foo>=1 google-api-core>=2 google-auth>=2 +grpcio>=1 proto-plus>=1 protobuf>=6 grpc-google-iam-v1>=0 -grpcio >= 1.75.1