diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index b951cb150..baf6be015 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -197,14 +197,14 @@ blocks: jobs: - name: Build and Tests with 'classic' group protocol commands: - - sem-version python 3.9 + - sem-version python 3.11 # use a virtualenv - python3 -m venv _venv && source _venv/bin/activate - chmod u+r+x tools/source-package-verification.sh - tools/source-package-verification.sh - name: Build and Tests with 'consumer' group protocol commands: - - sem-version python 3.9 + - sem-version python 3.11 - sem-version java 17 # use a virtualenv - python3 -m venv _venv && source _venv/bin/activate @@ -213,7 +213,7 @@ blocks: - tools/source-package-verification.sh - name: Build, Test, and Report coverage commands: - - sem-version python 3.9 + - sem-version python 3.11 # use a virtualenv - python3 -m venv _venv && source _venv/bin/activate - chmod u+r+x tools/source-package-verification.sh @@ -237,7 +237,7 @@ blocks: jobs: - name: Build commands: - - sem-version python 3.9 + - sem-version python 3.11 # use a virtualenv - python3 -m venv _venv && source _venv/bin/activate - chmod u+r+x tools/source-package-verification.sh @@ -256,7 +256,7 @@ blocks: jobs: - name: Build commands: - - sem-version python 3.9 + - sem-version python 3.11 # use a virtualenv - python3 -m venv _venv && source _venv/bin/activate - chmod u+r+x tools/source-package-verification.sh @@ -275,7 +275,7 @@ blocks: jobs: - name: Build commands: - - sem-version python 3.9 + - sem-version python 3.11 # use a virtualenv - python3 -m venv _venv && source _venv/bin/activate - chmod u+r+x tools/source-package-verification.sh @@ -302,7 +302,7 @@ blocks: - name: Build and Tests commands: # Setup Python environment - - sem-version python 3.9 + - sem-version python 3.11 - python3 -m venv _venv && source _venv/bin/activate # Install ducktape framework and additional dependencies diff --git a/DEVELOPER.md b/DEVELOPER.md index 9bc1d8c6b..88fef7810 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -212,8 +212,11 @@ tox -e black,isort # Check linting tox -e flake8 +# Check typing +tox -e mypy + # Run all formatting and linting checks -tox -e black,isort,flake8 +tox -e black,isort,flake8,mypy ``` ## Documentation build diff --git a/MANIFEST.in b/MANIFEST.in index 7e9bbf313..cd0c97ef2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,7 @@ include README.md include LICENSE include src/confluent_kafka/src/*.[ch] +include src/confluent_kafka/py.typed +include src/confluent_kafka/cimpl.pyi prune tests -prune docs \ No newline at end of file +prune docs diff --git a/Makefile b/Makefile index 3615e2b93..100c0dd14 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,9 @@ all: @echo "Targets:" @echo " clean" @echo " docs" + @echo " mypy" + @echo " style-check" + @echo " style-fix" clean: @@ -14,6 +17,9 @@ clean: docs: $(MAKE) -C docs html +mypy: + python3 -m mypy src/confluent_kafka + style-check: @(tools/style-format.sh \ $$(git ls-tree -r --name-only HEAD | egrep '\.(c|h|py)$$') ) diff --git a/requirements/requirements-avro.txt b/requirements/requirements-avro.txt index ccb70d0c4..8eefd943d 100644 --- a/requirements/requirements-avro.txt +++ b/requirements/requirements-avro.txt @@ -1,4 +1,4 @@ fastavro < 1.8.0; python_version == "3.7" fastavro < 2; python_version > "3.7" requests -avro>=1.11.1,<2 \ No newline at end of file +avro>=1.11.1,<2 diff --git a/requirements/requirements-tests.txt b/requirements/requirements-tests.txt index e1a85440d..099213039 100644 --- a/requirements/requirements-tests.txt +++ b/requirements/requirements-tests.txt @@ -2,7 +2,9 @@ urllib3<3 flake8 mypy +attrs types-cachetools +types-requests orjson pytest pytest-timeout diff --git a/src/confluent_kafka/deserializing_consumer.py b/src/confluent_kafka/deserializing_consumer.py index 18423d336..e1c0895a4 100644 --- a/src/confluent_kafka/deserializing_consumer.py +++ b/src/confluent_kafka/deserializing_consumer.py @@ -105,7 +105,11 @@ def poll(self, timeout: float = -1) -> Optional[Message]: if error is not None: raise ConsumeError(error, kafka_message=msg) - ctx = SerializationContext(msg.topic(), MessageField.VALUE, msg.headers()) + topic = msg.topic() + if topic is None: + raise TypeError("Message topic is None") + ctx = SerializationContext(topic, MessageField.VALUE, msg.headers()) + value = msg.value() if self._value_deserializer is not None: try: diff --git a/src/confluent_kafka/schema_registry/common/avro.py b/src/confluent_kafka/schema_registry/common/avro.py index f7ab32dfe..dc514f6ab 100644 --- a/src/confluent_kafka/schema_registry/common/avro.py +++ b/src/confluent_kafka/schema_registry/common/avro.py @@ -5,7 +5,7 @@ from collections import defaultdict from copy import deepcopy from io import BytesIO -from typing import Dict, Optional, Set, Tuple, Union +from typing import Dict, Optional, Set, Tuple, Union, cast from fastavro import repository, validate from fastavro.schema import load_schema @@ -217,11 +217,17 @@ def _resolve_union(schema: AvroSchema, message: AvroMessage) -> Tuple[Optional[A for subschema in schema: try: if is_wrapped_union: - if isinstance(subschema, dict) and subschema["name"] == message[0]: - return (subschema, message[1]) + if isinstance(subschema, dict): + dict_schema = cast(dict, subschema) + tuple_message = cast(tuple, message) + if dict_schema["name"] == tuple_message[0]: + return (dict_schema, tuple_message[1]) elif is_typed_union: - if isinstance(subschema, dict) and subschema["name"] == message['-type']: - return (subschema, message) + if isinstance(subschema, dict): + dict_schema = cast(dict, subschema) + dict_message = cast(dict, message) + if dict_schema["name"] == dict_message['-type']: + return (dict_schema, dict_message) else: validate(message, subschema) return (subschema, message) diff --git a/src/confluent_kafka/schema_registry/rules/cel/string_format.py b/src/confluent_kafka/schema_registry/rules/cel/string_format.py index 5f45beb83..8a544a29b 100644 --- a/src/confluent_kafka/schema_registry/rules/cel/string_format.py +++ b/src/confluent_kafka/schema_registry/rules/cel/string_format.py @@ -44,11 +44,11 @@ def __init__(self, locale: str): def format(self, fmt: celtypes.Value, args: celtypes.Value) -> celpy.Result: if not isinstance(fmt, celtypes.StringType): return celpy.native_to_cel( # type: ignore[attr-defined] - celpy.new_error("format() requires a string as the first argument") + celpy.new_error("format() requires a string as the first argument") # type: ignore[attr-defined] ) # type: ignore[attr-defined] if not isinstance(args, celtypes.ListType): return celpy.native_to_cel( # type: ignore[attr-defined] - celpy.new_error("format() requires a list as the second argument") + celpy.new_error("format() requires a list as the second argument") # type: ignore[attr-defined] ) # type: ignore[attr-defined] # printf style formatting i = 0 diff --git a/src/confluent_kafka/schema_registry/rules/encryption/encrypt_executor.py b/src/confluent_kafka/schema_registry/rules/encryption/encrypt_executor.py index 6a0b03436..454613004 100644 --- a/src/confluent_kafka/schema_registry/rules/encryption/encrypt_executor.py +++ b/src/confluent_kafka/schema_registry/rules/encryption/encrypt_executor.py @@ -365,7 +365,7 @@ def _is_expired(self, ctx: RuleContext, dek: Optional[Dek]) -> bool: ctx.rule_mode != RuleMode.READ and self._dek_expiry_days > 0 and dek is not None - and (now - dek.ts) / MILLIS_IN_DAY > self._dek_expiry_days + and (now - (dek.ts or 0)) / MILLIS_IN_DAY > self._dek_expiry_days ) # type: ignore[operator] def transform(self, ctx: RuleContext, field_type: FieldType, field_value: Any) -> Any: diff --git a/tests/ducktape/README.md b/tests/ducktape/README.md index 9bbbfad70..d09224004 100644 --- a/tests/ducktape/README.md +++ b/tests/ducktape/README.md @@ -51,7 +51,7 @@ The bounds configuration supports different environments with different performa "max_error_rate": 0.02, "min_success_rate": 0.98, "max_p99_latency_ms": 3000.0, - "max_memory_growth_mb": 800.0, + "max_memory_growth_mb": 1000.0, "max_buffer_full_rate": 0.05, "min_messages_per_poll": 10.0 }, @@ -62,7 +62,7 @@ The bounds configuration supports different environments with different performa "max_error_rate": 0.01, "min_success_rate": 0.99, "max_p99_latency_ms": 2500.0, - "max_memory_growth_mb": 600.0, + "max_memory_growth_mb": 700.0, "max_buffer_full_rate": 0.03, "min_messages_per_poll": 15.0 }, diff --git a/tests/ducktape/consumer_benchmark_metrics.py b/tests/ducktape/consumer_benchmark_metrics.py index 8cdd10154..5bcdded66 100644 --- a/tests/ducktape/consumer_benchmark_metrics.py +++ b/tests/ducktape/consumer_benchmark_metrics.py @@ -230,7 +230,7 @@ def __init__( max_p95_latency_ms: float = 10000.0, min_success_rate: float = 0.90, max_error_rate: float = 0.05, - max_memory_growth_mb: float = 600.0, + max_memory_growth_mb: float = 700.0, min_messages_per_consume: float = 0.5, max_empty_consume_rate: float = 0.5, ): diff --git a/tests/ducktape/producer_benchmark_bounds.json b/tests/ducktape/producer_benchmark_bounds.json index 5fca529ae..afbddc376 100644 --- a/tests/ducktape/producer_benchmark_bounds.json +++ b/tests/ducktape/producer_benchmark_bounds.json @@ -7,7 +7,7 @@ "max_error_rate": 0.02, "min_success_rate": 0.98, "max_p99_latency_ms": 7000.0, - "max_memory_growth_mb": 800.0, + "max_memory_growth_mb": 1000.0, "max_buffer_full_rate": 0.05, "min_messages_per_poll": 5.0 }, @@ -18,7 +18,7 @@ "max_error_rate": 0.01, "min_success_rate": 0.99, "max_p99_latency_ms": 12000.0, - "max_memory_growth_mb": 800.0, + "max_memory_growth_mb": 1000.0, "max_buffer_full_rate": 0.03, "min_messages_per_poll": 5.0 }, diff --git a/tests/ducktape/run_ducktape_test.py b/tests/ducktape/run_ducktape_test.py index bdff04e83..20faf6a42 100755 --- a/tests/ducktape/run_ducktape_test.py +++ b/tests/ducktape/run_ducktape_test.py @@ -135,7 +135,7 @@ def run_all_tests(args): print("=" * 70) for test_type in test_types: - print(f"\n{'='*20} Running {test_type.upper()} Tests {'='*20}") + print(f"\n{'='*20} Running {test_type.upper()} Tests {'='*20}") # noqa: E226 # Create a new args object for this test type test_args = argparse.Namespace(test_type=test_type, test_method=args.test_method, debug=args.debug) @@ -148,7 +148,7 @@ def run_all_tests(args): else: print(f"\nāœ… {test_type.upper()} tests passed!") - print(f"\n{'='*70}") + print(f"\n{'='*70}") # noqa: E226 if overall_success: print("šŸŽ‰ All tests completed successfully!") return 0 diff --git a/tests/ducktape/transaction_benchmark_bounds.json b/tests/ducktape/transaction_benchmark_bounds.json index 81fbc3148..623be3ca3 100644 --- a/tests/ducktape/transaction_benchmark_bounds.json +++ b/tests/ducktape/transaction_benchmark_bounds.json @@ -7,7 +7,7 @@ "max_error_rate": 0.02, "min_success_rate": 0.98, "max_p99_latency_ms": 8000.0, - "max_memory_growth_mb": 800.0, + "max_memory_growth_mb": 1000.0, "max_buffer_full_rate": 0.05, "min_messages_per_poll": 0.0 }, @@ -18,7 +18,7 @@ "max_error_rate": 0.01, "min_success_rate": 0.99, "max_p99_latency_ms": 7000.0, - "max_memory_growth_mb": 600.0, + "max_memory_growth_mb": 700.0, "max_buffer_full_rate": 0.03, "min_messages_per_poll": 0.0 }, diff --git a/tools/source-package-verification.sh b/tools/source-package-verification.sh index 3ca34a73f..530107f26 100755 --- a/tools/source-package-verification.sh +++ b/tools/source-package-verification.sh @@ -5,6 +5,7 @@ # set -e +pip install --upgrade pip pip install -r requirements/requirements-tests-install.txt pip install -U build @@ -52,9 +53,14 @@ if [[ $OS_NAME == linux && $ARCH == x64 ]]; then echo "Checking code formatting ..." # Check all tracked files (Python and C) all_files=$(git ls-tree -r --name-only HEAD | egrep '\.(py|c|h)$') + clang-format --version tools/style-format.sh $all_files || exit 1 echo "Building documentation ..." flake8 --exclude ./_venv,*_pb2.py,./build + + echo "Running mypy type checking ..." + python3.11 -m mypy src/confluent_kafka + pip install -r requirements/requirements-docs.txt make docs diff --git a/tox.ini b/tox.ini index 73123be2e..e9d4d3e17 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = flake8,black,isort,py37,py38,py39,py310,py311,py312,py313 +envlist = flake8,black,isort,mypy,py37,py38,py39,py310,py311,py312,py313 [testenv] passenv = @@ -20,6 +20,16 @@ commands = deps = flake8 commands = flake8 +[testenv:mypy] +deps = + mypy + types-cachetools + types-requests~=2.32.0 +commands = + # Need attrs to be explicitly installed for mypy to find frozen class init definitions + pip install attrs + mypy src/confluent_kafka + [testenv:black] deps = black>=24.0.0 commands = black --check --diff .