diff --git a/.analysis_options b/.analysis_options deleted file mode 100644 index a10d4c5..0000000 --- a/.analysis_options +++ /dev/null @@ -1,2 +0,0 @@ -analyzer: - strong-mode: true diff --git a/.gitignore b/.gitignore index c1dc09d..4c735a2 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,8 @@ packages .packages notes.md coverage/ +.atom/ +.vscode/ +.idea/ +package_config.json +.DS_Store diff --git a/.travis.yml b/.travis.yml index c17cdfa..df7b64b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,3 @@ -sudo: required - -dist: trusty - language: dart services: @@ -11,20 +7,15 @@ dart: - stable before_install: - - docker build -t kafka-cluster tool/kafka-cluster/ - - docker run -d --name kafka-cluster -p 2181:2181 -p 9092:9092 -p 9093:9093 --env ADVERTISED_HOST=127.0.0.1 kafka-cluster - - docker ps -a - - sleep 5 - - docker exec kafka-cluster bash -c '$KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper=localhost:2181 --topic dartKafkaTest --partitions 3 --replication-factor 2' - - docker exec kafka-cluster bash -c '$KAFKA_HOME/bin/kafka-topics.sh --list --zookeeper=localhost:2181' + - ./tool/rebuild.sh script: - - pub run test -r expanded test/all.dart - - pub global activate coverage - - dart --observe=8111 test/all.dart & - - sleep 20 - - pub global run coverage:collect_coverage --port=8111 -o coverage.json --resume-isolates - - pub global run coverage:format_coverage --package-root=packages --report-on lib --in coverage.json --out lcov.info --lcov + - pub run test test/all.dart + # - pub global activate coverage + # - dart --observe=8111 test/all.dart & + # - sleep 10 + # - pub global run coverage:collect_coverage --port=8111 -o coverage.json --resume-isolates + # - pub global run coverage:format_coverage --packages=.packages --report-on lib --in coverage.json --out lcov.info --lcov after_success: - - bash <(curl -s https://codecov.io/bash) + # - bash <(curl -s https://codecov.io/bash) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 46227d6..519c146 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -4,7 +4,7 @@ Requirements: * Docker Toolbox (OS X) -## Starting Kafka container locally +## Kafka 0.8.x ``` docker build -t kafka-cluster tool/kafka-cluster/ @@ -22,3 +22,28 @@ Now you should be able to run tests with: ``` pub run test -j 1 ``` + +## Kafka 0.10.0.0 + +We're using `ches/kafka` base image so instructions are a bit different. + +``` +# Zookeeper is in a separate container now +docker run -d --name zookeeper --publish 2181:2181 jplock/zookeeper:3.4.6 + +# Build our image +docker build -t kafka tool/kafka/ + +ZK_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' zookeeper) + +# or for fish users +export ZK_IP=(docker inspect --format '{{ .NetworkSettings.IPAddress }}' zookeeper) + +# Start Kafka container +docker run -d --name kafka --publish 9092:9092 --publish 9093:9093 \ + --env KAFKA_ADVERTISED_HOST_NAME=127.0.0.1 \ + --env ZOOKEEPER_IP=$ZK_IP \ + kafka +``` + +Kafka brokers will be available on `127.0.0.1:9092` and `127.0.0.1:9093`. diff --git a/README.md b/README.md index a597b51..40c3c75 100644 --- a/README.md +++ b/README.md @@ -1,198 +1,79 @@ # Dart Kafka -[![Build Status](https://travis-ci.org/pulyaevskiy/dart-kafka.svg?branch=master)](https://travis-ci.org/pulyaevskiy/dart-kafka) +[![Build Status](https://travis-ci.org/dart-kafka/kafka.svg?branch=kafka-0.10)](https://travis-ci.org/dart-kafka/kafka) [![Coverage](https://codecov.io/gh/pulyaevskiy/dart-kafka/branch/master/graph/badge.svg)](https://codecov.io/gh/pulyaevskiy/dart-kafka) [![License](https://img.shields.io/badge/license-BSD--2-blue.svg)](https://raw.githubusercontent.com/pulyaevskiy/dart-kafka/master/LICENSE) Kafka client library written in Dart. -### Current status - -This library is a work-in-progress and has not been used in production yet. - ### Things that are not supported yet. * Snappy compression. +* SSL ## Installation -There is no Pub package yet, but it will be published as soon as APIs are -stable enough. - -For now you can use git dependency in your `pubspec.yaml`: - -```yaml -dependencies: - kafka: - git: https://github.com/pulyaevskiy/dart-kafka.git -``` - -And then import it as usual: - -```dart -import 'package:kafka/kafka.dart'; -``` - -## Features +_To be updated with first release._ -This library provides several high-level API objects to interact with Kafka: - -* __KafkaSession__ - responsible for managing connections to Kafka brokers and - coordinating all requests. Also provides access to metadata information. -* __Producer__ - publishes messages to Kafka topics -* __Consumer__ - consumes messages from Kafka topics and stores it's state (current - offsets). Leverages ConsumerMetadata API via ConsumerGroup. -* __Fetcher__ - consumes messages from Kafka without storing state. -* __OffsetMaster__ - provides convenience on top of Offset API allowing to easily - retrieve earliest and latest offsets of particular topic-partitions. -* __ConsumerGroup__ - provides convenience on top of Consumer Metadata API to easily - fetch or commit consumer offsets. ## Producer -Simple implementation of Kafka producer. Supports auto-detection of leaders for -topic-partitions and creates separate `ProduceRequest`s for each broker. -Requests are sent in parallel and all responses are aggregated in special -`ProduceResult` object. +Producer publishes messages to the Kafka cluster. Here is a simple example +of using the producer to send `String` records: ```dart -// file:produce.dart -import 'dart:io'; -import 'package:kafka/kafka.dart'; - -main(List arguments) async { - var host = new ContactPoint('127.0.0.1', 9092); - var session = new KafkaSession([host]); - - var producer = new Producer(session, 1, 1000); - var result = await producer.produce([ - new ProduceEnvelope('topicName', 0, [new Message('msgForPartition0'.codeUnits)]), - new ProduceEnvelope('topicName', 1, [new Message('msgForPartition1'.codeUnits)]) - ]); - print(result.hasErrors); - print(result.offsets); - session.close(); // make sure to always close the session when the work is done. -} -``` - -Result: - -```shell -$ dart produce.dart -$ false -$ {dartKafkaTest: {0: 213075, 1: 201680}} -``` - -## Consumer - -High-level implementation of Kafka consumer which stores it's state using -Kafka's ConsumerMetadata API. - -> If you don't want to keep state of consumed offsets take a look at `Fetcher` -> which was designed specifically for this use case. - -Consumer returns messages as a `Stream`, so all standard stream operations -should be applicable. However Kafka topics are ordered streams of messages -with sequential offsets. Consumer implementation allows to preserve order of -messages received from server. For this purpose all messages are wrapped in -special `MessageEnvelope` object with following methods: - -``` -/// Signals to consumer that message has been processed and it's offset can -/// be committed. -void commit(String metadata); - -/// Signals that message has been processed and we are ready for -/// the next one. Offset of this message will **not** be committed. -void ack(); - -/// Signals to consumer to cancel any further deliveries and close the stream. -void cancel(); -``` - -One must call `commit()` or `ack()` for each processed message, otherwise -Consumer won't send the next message to the stream. - -Simplest example of a consumer: - -```dart -import 'dart:io'; import 'dart:async'; + import 'package:kafka/kafka.dart'; -void main(List arguments) async { - var host = new ContactPoint('127.0.0.1', 9092); - var session = new KafkaSession([host]); - var group = new ConsumerGroup(session, 'consumerGroupName'); - var topics = { - 'topicName': [0, 1] // list of partitions to consume from. - }; - - var consumer = new Consumer(session, group, topics, 100, 1); - await for (MessageEnvelope envelope in consumer.consume(limit: 3)) { - // Assuming that messages were produces by Producer from previous example. - var value = new String.fromCharCodes(envelope.message.value); - print('Got message: ${envelope.offset}, ${value}'); - envelope.commit('metadata'); // Important. - } - session.close(); // make sure to always close the session when the work is done. +Future main() async { + var config = new ProducerConfig(bootstrapServers: ['127.0.0.1:9092']); + var producer = new Producer( + new StringSerializer(), new StringSerializer(), config); + var record = new ProducerRecord('example-topic', 0, 'key', 'value'); + producer.add(record); + var result = await record.result; + print(result); + await producer.close(); } ``` -It is also possible to consume messages in batches for improved efficiency: - -```dart -import 'dart:io'; -import 'dart:async'; -import 'package:kafka/kafka.dart'; +The producer implements `StreamSink` interface so it is possible to send +individual records via `add()` as well as streams of records via +`addStream()`. -void main(List arguments) async { - var host = new ContactPoint('127.0.0.1', 9092); - var session = new KafkaSession([host]); - var group = new ConsumerGroup(session, 'consumerGroupName'); - var topics = { - 'topicName': [0, 1] // list of partitions to consume from. - }; - - var consumer = new Consumer(session, group, topics, 100, 1); - await for (BatchEnvelope batch in consumer.batchConsume(20)) { - batch.items.forEach((MessageEnvelope envelope) { - // use envelope as usual - }); - batch.commit('metadata'); // use batch control methods instead of individual messages. - } - session.close(); // make sure to always close the session when the work is done. -} -``` +The producer buffers records internally so that they can be sent in bulk to +the server. This does not necessarily mean increased latency for record +delivery. When a record is added with `add()` it is sent immediately +(although an asynchronous gap is present between call to `add()` and actual +send) as long as there is available slot in IO pool. By default producer +has a limit of up to 5 in-flight requests per broker connection, +so delay can occur only if all the slots are already occupied. -### Consumer offset reset strategy +## Note on new API design -Due to the fact that Kafka topics can be configured to delete old messages -periodically, it is possible that your consumer offset may become invalid ( -just because there is no such message/offset in Kafka topic anymore). +Public API of this library has been completely re-written since original +version (which supported only Kafka 0.8.x and was never published on Pub). -In such cases `Consumer` provides configurable strategy with following options: +New design is trying to accomplish two main goals: -* `OffsetOutOfRangeBehavior.throwError` -* `OffsetOutOfRangeBehavior.resetToEarliest` (default) -* `OffsetOutOfRangeBehavior.resetToLatest` +#### 1. Follow official Java client semantics and contract. -By default if it gets `OffsetOutOfRange` server error it will reset it's offsets -to earliest available in the consumed topic and partitions, which essentially -means consuming all available messages from the beginning. +`Producer` and `Consumer` are trying to preserve characteristics of +original Java implementations. This is why configurations for both +are almost identical to the ones in the official client. The way serialization is implemented also based on Java code. -To modify this behavior simply set `onOffsetOutOfRange` property of consumer to -one of the above values: +#### 2. Streams-compatible public API. -``` -var consumer = new Consumer(session, group, topics, 100, 1); -consumer.onOffsetOutOfRange = OffsetOutOfRangeBehavior.throwError; -``` +The main reason is to allow better interoperability with other libraries. +`Producer` implements `StreamSink` for this specific reason, so instead of +having a `send()` method (as in Java client) there are `add()` and +`addStream()`. ## Supported protocol versions -Current version targets version `0.8.2` of the Kafka protocol. There is no plans -to support earlier versions. +Current version targets version `0.10` of the Kafka protocol. +There is no plans to support earlier versions. ## License diff --git a/analysis_options.yaml b/analysis_options.yaml new file mode 100644 index 0000000..94bd6be --- /dev/null +++ b/analysis_options.yaml @@ -0,0 +1,4 @@ +analyzer: + strong-mode: + implicit-casts: true + implicit-dynamic: true diff --git a/example/offsets.dart b/example/offsets.dart new file mode 100644 index 0000000..79ac4f9 --- /dev/null +++ b/example/offsets.dart @@ -0,0 +1,15 @@ +import 'dart:async'; + +import 'package:kafka/kafka.dart'; + +Future main() async { + var session = Session(['127.0.0.1:9092']); + var master = OffsetMaster(session); + var offsets = await master.fetchEarliest([ + TopicPartition('simple_topic', 0), + TopicPartition('simple_topic', 1), + TopicPartition('simple_topic', 2), + ]); + print(offsets); + await session.close(); // Always close session in the end. +} diff --git a/example/simple_consumer.dart b/example/simple_consumer.dart new file mode 100644 index 0000000..bfd408d --- /dev/null +++ b/example/simple_consumer.dart @@ -0,0 +1,25 @@ +import 'dart:async'; + +import 'package:kafka/kafka.dart'; +import 'package:logging/logging.dart'; + +Future main() async { + Logger.root.level = Level.ALL; + Logger.root.onRecord.listen(print); + + var session = Session(['127.0.0.1:9092']); + var consumer = Consumer( + 'simple_consumer', StringDeserializer(), StringDeserializer(), session); + + await consumer.subscribe(['simple_topic']); + var queue = consumer.poll(); + while (await queue.moveNext()) { + var records = queue.current; + for (var record in records.records) { + print( + "[${record.topic}:${record.partition}], offset: ${record.offset}, ${record.key}, ${record.value}, ts: ${record.timestamp}"); + } + await consumer.commit(); + } + await session.close(); +} diff --git a/example/simple_producer.dart b/example/simple_producer.dart new file mode 100644 index 0000000..9ce9809 --- /dev/null +++ b/example/simple_producer.dart @@ -0,0 +1,22 @@ +import 'dart:async'; +import 'package:kafka/kafka.dart'; + +Future main() async { + // Logger.root.level = Level.ALL; + // Logger.root.onRecord.listen(print); + + var config = ProducerConfig(bootstrapServers: ['127.0.0.1:9092']); + var producer = + Producer(StringSerializer(), StringSerializer(), config); + + for (var i = 0; i < 10; i++) { + // Loop through a list of partitions. + for (var p in [0, 1, 2]) { + var rec = + ProducerRecord('simple_topic', p, 'key:${p},$i', 'value:${p},$i'); + producer.add(rec); + rec.result.then(print); + } + } + await producer.close(); +} diff --git a/kafka.bnf.txt b/kafka.bnf.txt new file mode 100644 index 0000000..bca4258 --- /dev/null +++ b/kafka.bnf.txt @@ -0,0 +1,250 @@ +# types: string, int8, int16, int32, int64, bytes + +Packet => Size Payload + Size => int32 + Payload => Request | Response + +Request => ApiKey ApiVersion CorrelationId ClientId RequestMessage + ApiKey => int16 + ApiVersion => int16 + CorrelationId => int32 + ClientId => string + RequestMessage => ProduceRequest_v0 | FetchRequest_v0 | OffsetRequest | MetadataRequest_v0 | OffsetCommitRequestV0 | OffsetCommitRequestV1 | OffsetCommitRequestV2 | OffsetFetchRequest | GroupCoordinatorRequest | JoinGroupRequest | HeartbeatRequest | LeaveGroupRequest | SyncGroupRequest | DescribeGroupsRequest | ListGroupsRequest + +Response => CorrelationId ResponseMessage + CorrelationId => int32 + ResponseMessage => ProduceResponse | FetchResponse | OffsetResponse | MetadataResponse | OffsetCommitResponse | OffsetFetchResponse | GroupCoordinatorResponse | JoinGroupResponse | HeartbeatResponse | LeaveGroupResponse | SyncGroupResponse | DescribeGroupsResponse | ListGroupsResponse + +Message => Offset MessageSize Crc MagicByte Attributes Key Value + Offset => int64 + MessageSize => int32 + Crc => int32 + MagicByte => int8 + Attributes => int8 + Key => bytes + Value => bytes + +MetadataRequest_v0 => [TopicName] + TopicName => string + +MetadataResponse => [Broker] [TopicMetadata] + Broker => NodeId Host Port + NodeId => int32 + Host => string + Port => int32 + TopicMetadata => ErrorCode TopicName [PartitionMetadata] + PartitionMetadata => ErrorCode Partition Leader Replicas Isr + Partition => int32 + Leader => int32 + Replicas => [int32] + Isr => [int32] + ErrorCode => int16 + TopicName => string + +ProduceRequest => RequiredAcks Timeout [TopicMessageSet] + TopicMessageSet => TopicName [PartitionMessageSet] + PartitionMessageSet => Partition MessageSetSize [Message] + RequiredAcks => int16 + Timeout => int32 + Partition => int32 + MessageSetSize => int32 + TopicName => string + +ProduceResponse => [ProduceResponseTopic] + ProduceResponseTopic => TopicName [ProduceResponsePartition] + ProduceResponsePartition => Partition ErrorCode Offset + TopicName => string + Partition => int32 + ErrorCode => int16 + Offset => int64 + +FetchRequest => ReplicaId MaxWaitTime MinBytes [FetchRequestTopic] + FetchRequestTopic => TopicName [FetchRequestPartition] + FetchRequestPartition => Partition FetchOffset MaxBytes + ReplicaId => int32 + MaxWaitTime => int32 + MinBytes => int32 + TopicName => string + Partition => int32 + FetchOffset => int64 + MaxBytes => int32 + +FetchResponse => [FetchResponseTopic] + FetchResponseTopic => TopicName [FetchResponsePartition] + FetchResponsePartition => Partition ErrorCode HighWatermarkOffset MessageSetSize [Message] + TopicName => string + Partition => int32 + ErrorCode => int16 + HighWatermarkOffset => int64 + MessageSetSize => int32 + +OffsetRequest => ReplicaId [OffsetRequestTopic] + OffsetRequestTopic => TopicName [OffsetRequestPartition] + OffsetRequestPartition => Partition Time MaxNumberOfOffsets + ReplicaId => int32 + TopicName => string + Partition => int32 + Time => int64 + MaxNumberOfOffsets => int32 + +OffsetResponse => [TopicOffsets] + TopicOffsets => TopicName [PartitionOffsets] + PartitionOffsets => Partition ErrorCode [Offset] + TopicName => string + Partition => int32 + ErrorCode => int16 + Offset => int64 + +GroupCoordinatorRequest => GroupId + GroupId => string + +GroupCoordinatorResponse => ErrorCode CoordinatorId CoordinatorHost CoordinatorPort + ErrorCode => int16 + CoordinatorId => int32 + CoordinatorHost => string + CoordinatorPort => int32 + +OffsetCommitRequestV0 => ConsumerGroupId [OCReqV0Topic] + OCReqV0Topic => TopicName [OCReqV0Partition] + OCReqV0Partition => Partition Offset Metadata + ConsumerGroupId => string + TopicName => string + Partition => int32 + Offset => int64 + Metadata => string + +OffsetCommitRequestV1 => ConsumerGroupId ConsumerGroupGenerationId ConsumerId [OCReqV1Topic] + OCReqV1Topic => TopicName [OCReqV1Partition] + OCReqV1Partition => Partition Offset TimeStamp Metadata + ConsumerGroupId => string + ConsumerGroupGenerationId => int32 + ConsumerId => string + TopicName => string + Partition => int32 + Offset => int64 + TimeStamp => int64 + Metadata => string + +OffsetCommitRequestV2 => ConsumerGroupId ConsumerGroupGenerationId ConsumerId RetentionTime [OCReqV2Topic] + OCReqV2Topic => TopicName [OCReqV2Partition] + OCReqV2Partition => Partition Offset Metadata + ConsumerGroupId => string + ConsumerGroupGenerationId => int32 + ConsumerId => string + RetentionTime => int64 + TopicName => string + Partition => int32 + Offset => int64 + Metadata => string + +OffsetCommitResponse => [OCRspTopic] + OCRspTopic => TopicName [OCRspPartition] + OCRspPartition => Partition ErrorCode + TopicName => string + Partition => int32 + ErrorCode => int16 + +OffsetFetchRequest => ConsumerGroup [OFReqTopic] + OFReqTopic => TopicName [Partition] + ConsumerGroup => string + TopicName => string + Partition => int32 + +OffsetFetchResponse => [TopicOffset] + TopicOffset => TopicName [PartitionOffset] + PartitionOffset => Partition Offset Metadata ErrorCode + TopicName => string + Partition => int32 + Offset => int64 + Metadata => string + ErrorCode => int16 + +JoinGroupRequest => GroupId SessionTimeout MemberId ProtocolType [GroupProtocol] + GroupId => string + SessionTimeout => int32 + MemberId => string + ProtocolType => string + GroupProtocol => ProtocolName ProtocolMetadata + ProtocolName => string + ProtocolMetadata => bytes + +JoinGroupResponse => ErrorCode GenerationId ProtocolName LeaderId MemberId [GroupMemberMetadata] + ErrorCode => int16 + GenerationId => int32 + ProtocolName => string + LeaderId => string + MemberId => string + +GroupMemberMetadata => MemberId ProtocolMetadata + MemberId => string + ProtocolMetadata => bytes + +# embeded in 'bytes' content of +# JoinGroupRequest.GroupProtocol.ProtocolMetadata +# JoinGroupResponse.Members.ProtocolMetadata +ConsumerGroupProtocolMetadata => Version [TopicName] UserData + Version => int16 + TopicName => string + UserData => bytes + +SyncGroupRequest => GroupId GenerationId MemberId [GroupAssignment] + GroupId => string + GenerationId => int32 + MemberId => string + +GroupAssignment => MemberId MemberAssignment + MemberId => string + MemberAssignment => bytes + +## embeded in 'bytes' content of +## SyncGroupRequest.GroupAssignment.MemberAssignment +ConsumerGroupMemberAssignment => Version [ConsumerGroupPartitionAssignment] UserData + Version => int16 + ConsumerGroupPartitionAssignment => TopicName [Partition] + TopicName => string + Partition => int32 + UserData => bytes + +SyncGroupResponse => ErrorCode MemberAssignment + ErrorCode => int16 + MemberAssignment => bytes + +HeartbeatRequest => GroupId GenerationId MemberId + GroupId => string + GenerationId => int32 + MemberId => string + +HeartbeatResponse => ErrorCode + ErrorCode => int16 + +LeaveGroupRequest => GroupId MemberId + GroupId => string + MemberId => string + +LeaveGroupResponse => ErrorCode + ErrorCode => int16 + +ListGroupsRequest => + +ListGroupsResponse => ErrorCode [GroupInfo] + ErrorCode => int16 + GroupInfo => GroupId ProtocolType + GroupId => string + ProtocolType => string + +DescribeGroupsRequest => [GroupId] + GroupId => string + +DescribeGroupsResponse => [GroupDescription] + GroupDescription => ErrorCode GroupId State ProtocolType Protocol [GroupMemberDescription] + ErrorCode => int16 + GroupId => string + State => string + ProtocolType => string + Protocol => string + GroupMemberDescription => MemberId ClientId ClientHost MemberMetadata MemberAssignment + MemberId => string + ClientId => string + ClientHost => string + MemberMetadata => bytes + MemberAssignment => bytes diff --git a/lib/common.dart b/lib/common.dart deleted file mode 100644 index c996387..0000000 --- a/lib/common.dart +++ /dev/null @@ -1,18 +0,0 @@ -/// Common dependencies for other Kafka libraries withing this package. -library kafka.common; - -import 'package:logging/logging.dart'; - -part 'src/common/errors.dart'; -part 'src/common/messages.dart'; -part 'src/common/metadata.dart'; -part 'src/common/offsets.dart'; - -/// String identifier used to pass to Kafka server in API calls. -const String dartKafkaId = 'dart_kafka'; - -/// Logger for this library. -/// -/// Doesn't do anything by default. You should set log level and add your handler -/// in order to get logs. -final Logger kafkaLogger = new Logger('Kafka'); diff --git a/lib/kafka.dart b/lib/kafka.dart index bb1e99e..8aee191 100644 --- a/lib/kafka.dart +++ b/lib/kafka.dart @@ -1,25 +1,21 @@ -/// ## Apache Kafka client library for Dartlang -/// -/// This library implements Kafka binary protocol and provides -/// high-level abstractions for producing and consuming messages. library kafka; -import 'dart:async'; -import 'dart:collection'; -import 'dart:io'; - -import 'package:quiver/collection.dart'; - -import 'common.dart'; - -import 'protocol.dart'; - -export 'common.dart' hide groupBy, kafkaLogger; -export 'protocol.dart' show TopicMetadata; - -part 'src/consumer.dart'; -part 'src/consumer_group.dart'; -part 'src/fetcher.dart'; -part 'src/offset_master.dart'; -part 'src/producer.dart'; -part 'src/session.dart'; +export 'src/common.dart'; +export 'src/consumer.dart'; +export 'src/consumer_metadata_api.dart'; +export 'src/consumer_offset_api.dart'; +export 'src/errors.dart'; +export 'src/fetch_api.dart'; +export 'src/group_membership_api.dart'; +export 'src/list_offset_api.dart'; +export 'src/messages.dart'; +export 'src/metadata.dart'; +export 'src/metadata_api.dart'; +export 'src/offset_commit_api.dart'; +export 'src/offset_master.dart'; +export 'src/partition_assignor.dart'; +export 'src/produce_api.dart'; +export 'src/producer.dart'; +export 'src/serialization.dart'; +export 'src/session.dart'; +export 'src/versions_api.dart'; diff --git a/lib/protocol.dart b/lib/protocol.dart deleted file mode 100644 index 33c3c46..0000000 --- a/lib/protocol.dart +++ /dev/null @@ -1,27 +0,0 @@ -/// Subpackage with implementation of Kafka protocol. -/// -/// Users of this package are not supposed to import this library directly and -/// use main 'kafka' package instead. -library kafka.protocol; - -import 'dart:collection'; -import 'dart:convert'; -import 'dart:io'; -import 'dart:math'; -import 'dart:typed_data'; - -import 'common.dart'; - -part 'src/protocol/bytes_builder.dart'; -part 'src/protocol/bytes_reader.dart'; -part 'src/protocol/common.dart'; -part 'src/protocol/consumer_metadata_api.dart'; -part 'src/protocol/fetch_api.dart'; -part 'src/protocol/group_membership_api.dart'; -part 'src/protocol/messages.dart'; -part 'src/protocol/metadata_api.dart'; -part 'src/protocol/offset_api.dart'; -part 'src/protocol/offset_commit_api.dart'; -part 'src/protocol/offset_fetch_api.dart'; -part 'src/protocol/produce_api.dart'; -part 'src/util/crc32.dart'; diff --git a/lib/src/common.dart b/lib/src/common.dart new file mode 100644 index 0000000..35884b9 --- /dev/null +++ b/lib/src/common.dart @@ -0,0 +1,98 @@ +import 'package:quiver/core.dart'; + +import 'util/tuple.dart'; + +/// Numeric codes of Kafka API requests. +class ApiKey { + static const produce = 0; + static const fetch = 1; + static const offsets = 2; + static const metadata = 3; + static const leaderAndIsr = 4; + static const stopReplica = 5; + static const updateMetadata = 6; + static const controlledShutdown = 7; + static const offsetCommit = 8; + static const offsetFetch = 9; + static const groupCoordinator = 10; + static const joinGroup = 11; + static const heartbeat = 12; + static const leaveGroup = 13; + static const syncGroup = 14; + static const describeGroups = 15; + static const listGroups = 16; + static const saslHandshake = 17; + static const apiVersions = 18; + static const createTopics = 19; + static const deleteTopics = 20; +} + +/// Represents single broker in Kafka cluster. +class Broker { + /// The unique identifier of this broker. + final int id; + + /// The hostname of this broker. + final String host; + + /// The port number this broker accepts connections on. + final int port; + + Broker._(this.id, this.host, this.port); + + static final Map _cache = new Map(); + + factory Broker(int id, String host, int port) { + var key = tuple3(id, host, port); + if (!_cache.containsKey(key)) { + _cache[key] = new Broker._(id, host, port); + } + + return _cache[key]; + } + + @override + int get hashCode => hash3(id, host, port); + + @override + bool operator ==(o) => + o is Broker && o.id == id && o.host == host && o.port == port; + + @override + String toString() => 'Broker{$id, $host:$port}'; +} + +/// Represents one partition in a topic. +class TopicPartition { + /// The name of Kafka topic. + final String topic; + + /// The partition ID. + final int partition; + + static final Map _cache = new Map(); + + TopicPartition._(this.topic, this.partition); + + factory TopicPartition(String topic, int partition) { + var key = hash2(topic, partition); + if (!_cache.containsKey(key)) { + _cache[key] = new TopicPartition._(topic, partition); + } + + return _cache[key]; + } + + @override + bool operator ==(o) { + return (o is TopicPartition && + o.topic == topic && + o.partition == partition); + } + + @override + int get hashCode => hash2(topic, partition); + + @override + String toString() => "TopicPartition{$topic:$partition}"; +} diff --git a/lib/src/common/errors.dart b/lib/src/common/errors.dart deleted file mode 100644 index c0d5c72..0000000 --- a/lib/src/common/errors.dart +++ /dev/null @@ -1,128 +0,0 @@ -part of kafka.common; - -/// Used to indicate there is a mismatch in CRC sum of a message (message is -/// corrupted). -class MessageCrcMismatchError extends StateError { - MessageCrcMismatchError(String message) : super(message); -} - -/// Represents error returned by Kafka server. -class KafkaServerError { - static const int NoError = 0; - static const int Unknown = -1; - static const int OffsetOutOfRange = 1; - static const int InvalidMessage = 2; - static const int UnknownTopicOrPartition = 3; - static const int InvalidMessageSize = 4; - static const int LeaderNotAvailable = 5; - static const int NotLeaderForPartition = 6; - static const int RequestTimedOut = 7; - static const int BrokerNotAvailable = 8; - static const int ReplicaNotAvailable = 9; - static const int MessageSizeTooLarge = 10; - static const int StaleControllerEpoch = 11; - static const int OffsetMetadataTooLarge = 12; - static const int OffsetsLoadInProgress = 14; - static const int ConsumerCoordinatorNotAvailable = 15; - static const int NotCoordinatorForConsumer = 16; - static const int InvalidTopicCode = 17; - static const int RecordListTooLargeCode = 18; - static const int NotEnoughReplicasCode = 19; - static const int NotEnoughReplicasAfterAppendCode = 20; - static const int InvalidRequiredAcksCode = 21; - static const int IllegalGenerationCode = 22; - static const int InconsistentGroupProtocolCode = 23; - static const int InvalidGroupIdCode = 24; - static const int UnknownMemberIdCode = 25; - static const int InvalidSessionTimeoutCode = 26; - static const int RebalanceInProgressCode = 27; - static const int InvalidCommitOffsetSizeCode = 28; - static const int TopicAuthorizationFailedCode = 29; - static const int GroupAuthorizationFailedCode = 30; - static const int ClusterAuthorizationFailedCode = 31; - static const int InvalidTimestamp = 32; - static const int UnsupportedSaslMechanism = 33; - static const int IllegalSaslState = 34; - static const int UnsupportedVersion = 35; - - /// Numeric code of this server error. - final int code; - - static final Map _instances = new Map(); - - static const Map _errorTexts = const { - 0: 'NoError', - -1: 'Unknown', - 1: 'OffsetOutOfRange', - 2: 'InvalidMessage', - 3: 'UnknownTopicOrPartition', - 4: 'InvalidMessageSize', - 5: 'LeaderNotAvailable', - 6: 'NotLeaderForPartition', - 7: 'RequestTimedOut', - 8: 'BrokerNotAvailable', - 9: 'ReplicaNotAvailable', - 10: 'MessageSizeTooLarge', - 11: 'StaleControllerEpoch', - 12: 'OffsetMetadataTooLarge', - 14: 'OffsetsLoadInProgress', - 15: 'ConsumerCoordinatorNotAvailable', - 16: 'NotCoordinatorForConsumer', - 17: 'InvalidTopicCode', - 18: 'RecordListTooLargeCode', - 19: 'NotEnoughReplicasCode', - 20: 'NotEnoughReplicasAfterAppendCode', - 21: 'InvalidRequiredAcksCode', - 22: 'IllegalGenerationCode', - 23: 'InconsistentGroupProtocolCode', - 24: 'InvalidGroupIdCode', - 25: 'UnknownMemberIdCode', - 26: 'InvalidSessionTimeoutCode', - 27: 'RebalanceInProgressCode', - 28: 'InvalidCommitOffsetSizeCode', - 29: 'TopicAuthorizationFailedCode', - 30: 'GroupAuthorizationFailedCode', - 31: 'ClusterAuthorizationFailedCode', - 32: 'InvalidTimestamp', - 33: 'UnsupportedSaslMechanism', - 34: 'IllegalSaslState', - 35: 'UnsupportedVersion', - }; - - /// String representation of this server error. - String get message => _errorTexts[code]; - - KafkaServerError._(this.code); - - /// Creates instance of KafkaServerError from numeric error code. - factory KafkaServerError(int code) { - if (!_instances.containsKey(code)) { - _instances[code] = new KafkaServerError._(code); - } - - return _instances[code]; - } - - @override - String toString() => 'KafkaServerError: ${message}(${code})'; - - bool get isError => code != NoError; - bool get isNoError => code == NoError; - bool get isUnknown => code == Unknown; - bool get isOffsetOutOfRange => code == OffsetOutOfRange; - bool get isInvalidMessage => code == InvalidMessage; - bool get isUnknownTopicOrPartition => code == UnknownTopicOrPartition; - bool get isInvalidMessageSize => code == InvalidMessageSize; - bool get isLeaderNotAvailable => code == LeaderNotAvailable; - bool get isNotLeaderForPartition => code == NotLeaderForPartition; - bool get isRequestTimedOut => code == RequestTimedOut; - bool get isBrokerNotAvailable => code == BrokerNotAvailable; - bool get isReplicaNotAvailable => code == ReplicaNotAvailable; - bool get isMessageSizeTooLarge => code == MessageSizeTooLarge; - bool get isStaleControllerEpoch => code == StaleControllerEpoch; - bool get isOffsetMetadataTooLarge => code == OffsetMetadataTooLarge; - bool get isOffsetsLoadInProgress => code == OffsetsLoadInProgress; - bool get isConsumerCoordinatorNotAvailable => - code == ConsumerCoordinatorNotAvailable; - bool get isNotCoordinatorForConsumer => code == NotCoordinatorForConsumer; -} diff --git a/lib/src/common/messages.dart b/lib/src/common/messages.dart deleted file mode 100644 index 0bfcbfa..0000000 --- a/lib/src/common/messages.dart +++ /dev/null @@ -1,95 +0,0 @@ -part of kafka.common; - -/// Compression types supported by Kafka. -enum KafkaCompression { none, gzip, snappy } - -/// Kafka Message Attributes. Only [KafkaCompression] is supported by the -/// server at the moment. -class MessageAttributes { - /// Compression codec. - final KafkaCompression compression; - - /// Creates new instance of MessageAttributes. - MessageAttributes([this.compression = KafkaCompression.none]); - - /// Creates MessageAttributes from the raw byte. - MessageAttributes.fromByte(int byte) : compression = getCompression(byte); - - static KafkaCompression getCompression(int byte) { - var c = byte & 3; - var map = { - 0: KafkaCompression.none, - 1: KafkaCompression.gzip, - 2: KafkaCompression.snappy, - }; - return map[c]; - } - - /// Converts this attributes into byte. - int toInt() { - return _compressionToInt(); - } - - int _compressionToInt() { - switch (this.compression) { - case KafkaCompression.none: - return 0; - case KafkaCompression.gzip: - return 1; - case KafkaCompression.snappy: - return 2; - } - } -} - -/// Kafka Message as defined in the protocol. -class Message { - /// Metadata attributes about this message. - final MessageAttributes attributes; - - /// Actual message contents. - final List value; - - /// Optional message key that was used for partition assignment. - /// The key can be `null`. - final List key; - - /// Default internal constructor. - Message._(this.attributes, this.key, this.value); - - /// Creates new [Message]. - factory Message(List value, - {MessageAttributes attributes, List key}) { - attributes ??= new MessageAttributes(); - return new Message._(attributes, key, value); - } -} - -/// Envelope used for publishing messages to Kafka. -class ProduceEnvelope { - /// Name of the topic. - final String topicName; - - /// Partition ID. - final int partitionId; - - /// List of messages to publish. - final List messages; - - /// Compression codec to be used. - final KafkaCompression compression; - - /// Creates new envelope containing list of messages. - /// - /// You can optionally set [compression] codec which will be used to encode - /// messages. - ProduceEnvelope(this.topicName, this.partitionId, this.messages, - {this.compression: KafkaCompression.none}) { - messages.forEach((m) { - if (m.attributes.compression != KafkaCompression.none) { - throw new StateError( - 'ProduceEnvelope: compression can not be set on individual messages in ProduceEnvelope, use ProduceEnvelope.compression instead.'); - } - }); - } -} diff --git a/lib/src/common/metadata.dart b/lib/src/common/metadata.dart deleted file mode 100644 index 2d1f2b0..0000000 --- a/lib/src/common/metadata.dart +++ /dev/null @@ -1,58 +0,0 @@ -part of kafka.common; - -/// Represents single node in a Kafka cluster. -class Broker { - /// Unique ID of this broker within cluster. - final int id; - - /// Host name or IP address of this broker. - final String host; - - /// Port number of this broker. - final int port; - - static final Map _instances = new Map(); - - /// Creates new instance of Kafka broker. - factory Broker(int id, String host, int port) { - var key = '${host}:${port}'; - if (!_instances.containsKey(key)) { - _instances[key] = new Broker._(id, host, port); - } else { - if (_instances[key].id != id) throw new StateError('Broker ID mismatch.'); - } - - return _instances[key]; - } - - Broker._(this.id, this.host, this.port); - - @override - toString() => 'KafkaBroker: ${host}:${port} (id: ${id})'; -} - -class TopicPartition { - final String topicName; - final int partitionId; - - static final Map _cache = new Map(); - - TopicPartition._(this.topicName, this.partitionId); - - factory TopicPartition(String topicName, int partitionId) { - var key = topicName + partitionId.toString(); - if (!_cache.containsKey(key)) { - _cache[key] = new TopicPartition._(topicName, partitionId); - } - - return _cache[key]; - } - - @override - bool operator ==(other) { - return (other.topicName == topicName && other.partitionId == partitionId); - } - - @override - int get hashCode => (topicName + partitionId.toString()).hashCode; -} diff --git a/lib/src/common/offsets.dart b/lib/src/common/offsets.dart deleted file mode 100644 index b32a595..0000000 --- a/lib/src/common/offsets.dart +++ /dev/null @@ -1,26 +0,0 @@ -part of kafka.common; - -Map groupBy(Iterable list, f(element)) { - var grouped = new Map(); - for (var e in list) { - var key = f(e); - if (!grouped.containsKey(key)) { - grouped[key] = new List(); - } - grouped[key].add(e); - } - - return grouped; -} - -/// Data structure representing consumer offset. -class ConsumerOffset { - final String topicName; - final int partitionId; - final int offset; - final String metadata; - final int errorCode; - - ConsumerOffset(this.topicName, this.partitionId, this.offset, this.metadata, - [this.errorCode]); -} diff --git a/lib/src/consumer.dart b/lib/src/consumer.dart index 4b33fac..68dd3ce 100644 --- a/lib/src/consumer.dart +++ b/lib/src/consumer.dart @@ -1,488 +1,464 @@ -part of kafka; +import 'dart:async'; +import 'package:logging/logging.dart'; +import 'consumer_streamiterator.dart'; +import 'common.dart'; +import 'consumer_group.dart'; +import 'consumer_offset_api.dart'; +import 'errors.dart'; +import 'fetch_api.dart'; +import 'group_membership_api.dart'; +import 'offset_master.dart'; +import 'serialization.dart'; +import 'session.dart'; +import 'util/group_by.dart'; + +final Logger _logger = new Logger('Consumer'); + +/// Consumes messages from Kafka cluster. +/// +/// Consumer interacts with the server to allow multiple members of the same group +/// load balance consumption by distributing topics and partitions evenly across +/// all members. +/// +/// ## Subscription and rebalance +/// +/// Before it can start consuming messages, Consumer must [subscribe] to a set +/// of topics. Every instance subscribes as a distinct member of it's [group]. +/// Each member receives assignment which contains a unique subset of topics and +/// partitions it must consume. So if there is only one member then it gets +/// assigned all topics and partitions when subscribed. +/// +/// When another member wants to join the same group a rebalance is triggered by +/// the Kafka server. During rebalance all members must rejoin the group and +/// receive their updated assignments. Rebalance is handled transparently by +/// this client and does not require any additional action. +/// +/// ## Usage example +/// +/// TODO: write an example +abstract class Consumer { + /// The consumer group name. + String get group; -/// Determines behavior of [Consumer] when it receives `OffsetOutOfRange` API -/// error. -enum OffsetOutOfRangeBehavior { - /// Consumer will throw [KafkaServerError] with error code `1`. - throwError, + /// Starts polling Kafka servers for new messages. + /// + /// Must first call [subscribe] to indicate which topics must be consumed. + StreamIterator> poll(); + + /// Subscribe this consumer to a set of [topics]. + /// + /// This function evaluates lazily, subscribing to the specified topics only + /// when [poll] is called. + void subscribe(List topics); - /// Consumer will reset it's offsets to the earliest available for particular - /// topic-partition. - resetToEarliest, + /// Unsubscribes from all currently assigned partitions and leaves + /// consumer group. + /// + /// Unsubscribe triggers rebalance of all existing members of this consumer + /// group. + Future unsubscribe(); + + /// Commits current offsets to the server. + Future commit(); - /// Consumer will reset it's offsets to the latest available for particular - /// topic-partition. - resetToLatest + /// Seek to the first offset for all of the currently assigned partitions. + /// + /// This function evaluates lazily, seeking to the first offset in all + /// partitions only when [poll] is called. + /// + /// Requires active subscription, see [subscribe] for more details. + void seekToBeginning(); + + /// Seek to the last offset for all of the currently assigned partitions. + /// + /// This function evaluates lazily, seeking to the last offset in all + /// partitions only when [poll] is called. + /// + /// Requires active subscription, see [subscribe] for more details. + void seekToEnd(); + + factory Consumer(String group, Deserializer keyDeserializer, + Deserializer valueDeserializer, Session session) { + return new _ConsumerImpl( + group, keyDeserializer, valueDeserializer, session); + } } -/// High-level Kafka consumer class. +/// Defines type of consumer state function. +typedef Future _ConsumerState(); + +/// Default implementation of Kafka consumer. /// -/// Provides convenience layer on top of Kafka's low-level APIs. -class Consumer { - /// Instance of [KafkaSession] used to send requests. - final KafkaSession session; +/// Implements a finite state machine which is started by a call to [poll]. +class _ConsumerImpl implements Consumer { + static const int DEFAULT_MAX_BYTES = 36864; + static const int DEFAULT_MAX_WAIT_TIME = 10000; + static const int DEFAULT_MIN_BYTES = 1; + + final Session session; + final Deserializer keyDeserializer; + final Deserializer valueDeserializer; + final int requestMaxBytes; + + final ConsumerGroup _group; - /// Consumer group this consumer belongs to. - final ConsumerGroup consumerGroup; + _ConsumerState _activeState; - /// Topics and partitions to consume. - final Map> topicPartitions; + _ConsumerImpl( + String group, this.keyDeserializer, this.valueDeserializer, this.session, + {int requestMaxBytes}) + : _group = new ConsumerGroup(session, group), + requestMaxBytes = requestMaxBytes ?? DEFAULT_MAX_BYTES; - /// Maximum amount of time in milliseconds to block waiting if insufficient - /// data is available at the time the request is issued. - final int maxWaitTime; + /// The consumer group name. + String get group => _group.name; - /// Minimum number of bytes of messages that must be available - /// to give a response. - final int minBytes; + GroupSubscription _subscription; - /// Determines this consumer's strategy of handling `OffsetOutOfRange` API - /// errors. + /// Current consumer subscription. + GroupSubscription get subscription => _subscription; + + /// List of topics to subscribe to when joining the group. /// - /// Default value is `resetToEarliest` which will automatically reset offset - /// of ConsumerGroup for particular topic-partition to the earliest offset - /// available. + /// Set by initial call to [subscribe] and used during initial + /// subscribe and possible resubscriptions. + List _topics; + + StreamController> _streamController; + ConsumerStreamIterator _streamIterator; + + /// Whether user canceled stream subscription. /// - /// See [OffsetOutOfRangeBehavior] for details on each value. - OffsetOutOfRangeBehavior onOffsetOutOfRange = - OffsetOutOfRangeBehavior.resetToEarliest; - - /// Creates new consumer identified by [consumerGroup]. - Consumer(this.session, this.consumerGroup, this.topicPartitions, - this.maxWaitTime, this.minBytes); - - /// Consumes messages from Kafka. If [limit] is specified consuming - /// will stop after exactly [limit] messages have been retrieved. If no - /// specific limit is set it'll default to `-1` and will consume all incoming - /// messages continuously. - Stream consume({int limit: -1}) { - var controller = new _MessageStreamController(limit); - - Future> list = _buildWorkers(); - list.then((workers) { - if (workers.isEmpty) { - controller.close(); - return; - } - var remaining = workers.length; - var futures = workers.map((w) => w.run(controller)).toList(); - futures.forEach((Future f) { - f.then((_) { - remaining--; - if (remaining == 0) { - kafkaLogger - ?.info('Consumer: All workers are done. Closing stream.'); - controller.close(); - } - }, onError: (error, stackTrace) { - controller.addError(error, stackTrace); - }); - }); - }, onError: (error, stackTrace) { - controller.addError(error, stackTrace); - }); + /// This triggers shutdown of polling phase. + bool _isCanceled = false; + + /// Whether resubscription is required due to rebalance event received from + /// the server. + bool _resubscriptionNeeded = false; - return controller.stream; + @override + void subscribe(List topics) { + assert(_subscription == null, 'Already subscribed.'); + _topics = new List.from(topics, growable: false); } - /// Consume messages in batches. - /// - /// This will create a stream of [BatchEnvelope] objects. Each batch - /// will contain up to [maxBatchSize] of `MessageEnvelope`s. + /// State of this consumer during (re)subscription. /// - /// Note that calling `commit`, `ack`, or `cancel` on individual message - /// envelope will take no effect. Instead one should use corresponding methods - /// on the BatchEnvelope itself. - /// - /// Currently batches are formed on per broker basis, meaning each batch will - /// always contain messages from one particular broker. - Stream batchConsume(int maxBatchSize) { - var controller = new _BatchStreamController(); - - Future> list = _buildWorkers(); - list.then((workers) { - if (workers.isEmpty) { - controller.close(); - return; - } - var remaining = workers.length; - var futures = - workers.map((w) => w.runBatched(controller, maxBatchSize)).toList(); - futures.forEach((Future f) { - f.then((_) { - kafkaLogger.info('Consumer: worker finished.'); - remaining--; - if (remaining == 0) { - kafkaLogger - ?.info('Consumer: All workers are done. Closing stream.'); - controller.close(); - } - }, onError: (error, stackTrace) { - controller.addError(error, stackTrace); - }); - }); - }, onError: (error, stackTrace) { - controller.addError(error, stackTrace); + /// Consumer enters this state initially when listener is added to the stream + /// and may re-enter this state in case of a rebalance event triggerred by + /// the server. + Future _resubscribeState() { + _logger + .info('Subscribing to topics ${_topics} as a member of group $group'); + var protocols = [new GroupProtocol.roundrobin(0, _topics.toSet())]; + return _group.join(30000, 3000, '', 'consumer', protocols).then((result) { + // TODO: resume heartbeat timer. + _subscription = result; + _resubscriptionNeeded = false; + _logger.info('Subscription result: ${subscription}.'); + // Switch to polling state + _activeState = _pollState; }); - - return controller.stream; } - Future> _buildWorkers() async { - var meta = await session.getMetadata(topicPartitions.keys.toSet()); - var topicsByBroker = new Map>>(); - - topicPartitions.forEach((topic, partitions) { - partitions.forEach((p) { - var leader = meta.getTopicMetadata(topic).getPartition(p).leader; - var broker = meta.getBroker(leader); - if (topicsByBroker.containsKey(broker) == false) { - topicsByBroker[broker] = new Map>(); - } - if (topicsByBroker[broker].containsKey(topic) == false) { - topicsByBroker[broker][topic] = new Set(); - } - topicsByBroker[broker][topic].add(p); - }); - }); + @override + StreamIterator> poll() { + assert(_topics != null, + 'No topics set for subscription. Must first call subscribe().'); + assert(_streamController == null, 'Already polling.'); - var workers = new List<_ConsumerWorker>(); - topicsByBroker.forEach((host, topics) { - var worker = new _ConsumerWorker( - session, host, topics, maxWaitTime, minBytes, - group: consumerGroup); - worker.onOffsetOutOfRange = onOffsetOutOfRange; - workers.add(worker); - }); + _streamController = new StreamController>( + onListen: onListen, onCancel: onCancel); + _streamIterator = + new ConsumerStreamIterator(_streamController.stream); - return workers; + return _streamIterator; } -} -class _MessageStreamController { - final int limit; - final StreamController _controller = - new StreamController(); - int _added = 0; - bool _cancelled = false; - - _MessageStreamController(this.limit); - - bool get canAdd => - (_cancelled == false && ((limit == -1) || (_added < limit))); - Stream get stream => _controller.stream; - - /// Attempts to add [event] to the stream. - /// Returns true if adding event succeeded, false otherwise. - bool add(MessageEnvelope event) { - if (canAdd) { - _controller.add(event); - _added++; - return true; + /// Starts execution of state machine. + /// + /// Returned future completes whenever there is no active state + /// (execution completed) or unhandled error occured. + Future _run() async { + while (_activeState != null) { + await _activeState(); } - return false; } - void addError(Object error, [StackTrace stackTrace]) { - _controller.addError(error, stackTrace); + /// Only set in initial stream controler. Rebalance events create new streams + /// but we don't set onListen callback on those since our state machine + /// is already running. + void onListen() { + // Start polling only after there is active listener. + _activeState = _resubscribeState; + _run().catchError((error, stackTrace) { + _streamController.addError(error, stackTrace); + }).whenComplete(() { + // TODO: ensure cleanup here, e.g. shutdown heartbeats + var closeFuture = _streamController.close(); + _streamController = null; + _streamIterator = null; + return closeFuture; + }); } - void cancel() { - _cancelled = true; + void onCancel() { + _isCanceled = true; + // Listener canceled subscription so we need to drop any records waiting + // to be processed. + _waitingRecords.values.forEach((_) { + _.ack(); + }); } - void close() { - _controller.close(); + /// Poll state of this consumer's state machine. + /// + /// Consumer enters this state when [poll] is executed and stays in this state + /// until: + /// + /// - user cancels the poll, results in "clean exit". + /// - a rebalance error received from the server, which transitions state machine to + /// the [_resubscribeState]. + /// - an unhandled error occurs, adds error to the stream. + Future _pollState() async { + return _poll().catchError( + (error) { + // Switch to resubscribe state because server is performing rebalance. + _resubscriptionNeeded = true; + }, + test: isRebalanceError, + ).whenComplete(() { + // Check if resubscription is needed in case there were rebalance + // errors from either offset commit or heartbeat requests. + if (_resubscriptionNeeded) { + _logger.fine("switch to resubscribe state"); + // Switch to resubscribe state. + _activeState = _resubscribeState; + // Create new stream controller and attach to the stream iterator. + // This cancels subscription on existing stream and prevents delivery + // of any in-flight events to the listener. It also clears uncommitted + // offsets to prevent offset commits during rebalance. + + // Remove onCancel callback on existing controller. + _streamController.onCancel = null; + _streamController = + StreamController>(onCancel: onCancel); + _streamIterator.attachStream(_streamController.stream); + } + }); } -} -/// Worker responsible for fetching messages from one particular Kafka broker. -class _ConsumerWorker { - final KafkaSession session; - final Broker host; - final ConsumerGroup group; - final Map> topicPartitions; - final int maxWaitTime; - final int minBytes; - - OffsetOutOfRangeBehavior onOffsetOutOfRange = - OffsetOutOfRangeBehavior.resetToEarliest; - - _ConsumerWorker(this.session, this.host, this.topicPartitions, - this.maxWaitTime, this.minBytes, - {this.group}); - - Future run(_MessageStreamController controller) async { - kafkaLogger - ?.info('Consumer: Running worker on host ${host.host}:${host.port}'); - - while (controller.canAdd) { - var request = await _createRequest(); - kafkaLogger?.fine('Consumer: Sending fetch request to ${host}.'); - FetchResponse response = await session.send(host, request); - var didReset = await _checkOffsets(response); - if (didReset) { - kafkaLogger?.warning( - 'Offsets were reset to ${onOffsetOutOfRange}. Forcing re-fetch.'); - continue; - } - for (var item in response.results) { - for (var offset in item.messageSet.messages.keys) { - var message = item.messageSet.messages[offset]; - var envelope = new MessageEnvelope( - item.topicName, item.partitionId, offset, message); - if (!controller.add(envelope)) { - return; - } else { - var result = await envelope.result; - if (result.status == _ProcessingStatus.commit) { - var offsets = [ - new ConsumerOffset(item.topicName, item.partitionId, offset, - result.commitMetadata) - ]; - await group.commitOffsets(offsets, -1, ''); - } else if (result.status == _ProcessingStatus.cancel) { - controller.cancel(); - return; - } - } - } - } + /// Returns `true` if [error] requires resubscription. + bool isRebalanceError(error) => + error is RebalanceInProgressError || error is UnknownMemberIdError; + + /// Internal polling method. + Future _poll() async { + var offsets = await _fetchOffsets(subscription); + _logger.fine('Polling started from following offsets: ${offsets}'); + Map> leaders = + await _fetchPartitionLeaders(subscription, offsets); + + List brokerPolls = new List(); + for (var broker in leaders.keys) { + brokerPolls.add(_pollBroker(broker, leaders[broker])); } + await Future.wait(brokerPolls); } - Future runBatched(_BatchStreamController controller, int maxBatchSize) async { - kafkaLogger?.info( - 'Consumer: Running batch worker on host ${host.host}:${host.port}'); - - while (controller.canAdd) { - var request = await _createRequest(); - FetchResponse response = await session.send(host, request); - var didReset = await _checkOffsets(response); - if (didReset) { - kafkaLogger?.warning( - 'Offsets were reset to ${onOffsetOutOfRange}. Forcing re-fetch.'); - continue; + /// Stores references to consumer records in each polling broker that this + /// consumer is currently waiting to be processed. + /// The `onCancel` callback acknowledges all of these so that polling can + /// shutdown gracefully. + final Map> _waitingRecords = Map(); + + Future _pollBroker(Broker broker, List initialOffsets) async { + Map currentOffsets = Map.fromIterable( + initialOffsets, + key: (offset) => offset.topicPartition); + + while (true) { + if (_isCanceled || _resubscriptionNeeded) { + _logger.fine('Stoping poll on $broker.'); + break; } - for (var batch in responseToBatches(response, maxBatchSize)) { - if (!controller.add(batch)) return; - var result = await batch.result; - if (result.status == _ProcessingStatus.commit) { - await group.commitOffsets(batch.offsetsToCommit, -1, ''); - } else if (result.status == _ProcessingStatus.cancel) { - controller.cancel(); - return; - } + _logger.fine('Sending poll request on $broker'); + var request = + _buildRequest(currentOffsets.values.toList(growable: false)); + var response = await session.send(request, broker.host, broker.port); + + var records = recordsFromResponse(response.results); + + _logger + .fine('response from $broker has ${records.records.length} records'); + + if (records.records.isEmpty) continue; // empty response, continue polling + + for (var rec in records.records) { + currentOffsets[rec.topicPartition] = + ConsumerOffset(rec.topic, rec.partition, rec.offset, ''); } + + _waitingRecords[broker] = records; + _streamController.add(records); + await records.future; } } - Iterable responseToBatches( - FetchResponse response, int maxBatchSize) sync* { - BatchEnvelope batch; - for (var item in response.results) { - for (var offset in item.messageSet.messages.keys) { - var message = item.messageSet.messages[offset]; - var envelope = new MessageEnvelope( - item.topicName, item.partitionId, offset, message); - - if (batch == null) batch = new BatchEnvelope(); - if (batch.items.length < maxBatchSize) { - batch.items.add(envelope); - } - if (batch.items.length == maxBatchSize) { - yield batch; - batch = null; - } - } - } - if (batch is BatchEnvelope && batch.items.isNotEmpty) { - yield batch; - batch = null; - } + ConsumerRecords recordsFromResponse(List results) { + var records = results.expand((result) { + return result.messages.keys.map((offset) { + var message = result.messages[offset]; + var key = keyDeserializer.deserialize(message.key); + var value = valueDeserializer.deserialize(message.value); + return ConsumerRecord(result.topic, result.partition, offset, key, + value, message.timestamp); + }); + }).toList(growable: false); + return ConsumerRecords(records); } - Future _checkOffsets(FetchResponse response) async { - var topicsToReset = new Map>(); - for (var result in response.results) { - if (result.errorCode == KafkaServerError.OffsetOutOfRange) { - kafkaLogger?.warning( - 'Consumer: received API error 1 for topic ${result.topicName}:${result.partitionId}'); - if (!topicsToReset.containsKey(result.topicName)) { - topicsToReset[result.topicName] = new Set(); - } - topicsToReset[result.topicName].add(result.partitionId); - kafkaLogger?.info('Topics to reset: ${topicsToReset}'); + /// Fetches current consumer offsets from the server. + /// + /// Checks whether current offsets are valid by comparing to earliest + /// available offsets in the topics. Resets current offset if it's value is + /// lower than earliest available in the partition. + Future> _fetchOffsets( + GroupSubscription subscription) async { + _logger.finer('Fetching offsets for ${group}'); + var currentOffsets = + await _group.fetchOffsets(subscription.assignment.partitionsAsList); + var offsetMaster = new OffsetMaster(session); + var earliestOffsets = await offsetMaster + .fetchEarliest(subscription.assignment.partitionsAsList); + + List resetNeeded = new List(); + for (var earliest in earliestOffsets) { + // Current consumer offset can be either -1 or a value >= 0, where + // `-1` means that no committed offset exists for this partition. + // + var current = currentOffsets.firstWhere((_) => + _.topic == earliest.topic && _.partition == earliest.partition); + if (current.offset + 1 < earliest.offset) { + // reset to earliest + _logger.warning('Current consumer offset (${current.offset}) is less ' + 'than earliest available for partition (${earliest.offset}). ' + 'This may indicate that consumer missed some records in ${current.topicPartition}. ' + 'Resetting this offset to earliest.'); + resetNeeded.add(current.copy( + offset: earliest.offset - 1, metadata: 'resetToEarliest')); } } - if (topicsToReset.isNotEmpty) { - switch (onOffsetOutOfRange) { - case OffsetOutOfRangeBehavior.throwError: - throw new KafkaServerError(1); - case OffsetOutOfRangeBehavior.resetToEarliest: - await group.resetOffsetsToEarliest(topicsToReset); - break; - case OffsetOutOfRangeBehavior.resetToLatest: - await group.resetOffsetsToLatest(topicsToReset); - break; - } - return true; + if (resetNeeded.isNotEmpty) { + await _group.commitOffsets(resetNeeded, subscription: subscription); + return _group.fetchOffsets(subscription.assignment.partitionsAsList); } else { - return false; + return currentOffsets; } } - Future _createRequest() async { - var offsets = await group.fetchOffsets(topicPartitions); - var request = new FetchRequest(maxWaitTime, minBytes); - for (var o in offsets) { - request.add(o.topicName, o.partitionId, o.offset + 1); + FetchRequest _buildRequest(List offsets) { + var request = new FetchRequest(DEFAULT_MAX_WAIT_TIME, DEFAULT_MIN_BYTES); + for (var offset in offsets) { + request.add(offset.topicPartition, + new FetchData(offset.offset + 1, requestMaxBytes)); } - return request; } -} - -enum _ProcessingStatus { commit, ack, cancel } - -class _ProcessingResult { - final _ProcessingStatus status; - final String commitMetadata; - - _ProcessingResult.commit(String metadata) - : status = _ProcessingStatus.commit, - commitMetadata = metadata; - _ProcessingResult.ack() - : status = _ProcessingStatus.ack, - commitMetadata = ''; - _ProcessingResult.cancel() - : status = _ProcessingStatus.cancel, - commitMetadata = ''; -} - -/// Envelope for a [Message] used by high-level consumer. -class MessageEnvelope { - /// Topic name of this message. - final String topicName; - - /// Partition ID of this message. - final int partitionId; - - /// This message's offset - final int offset; - - /// Actual message received from Kafka broker. - final Message message; - - Completer<_ProcessingResult> _completer = new Completer<_ProcessingResult>(); - - /// Creates new envelope. - MessageEnvelope(this.topicName, this.partitionId, this.offset, this.message); - - Future<_ProcessingResult> get result => _completer.future; - - /// Signals that message has been processed and it's offset can - /// be committed (in case of high-level [Consumer] implementation). In case if - /// consumerGroup functionality is not used (like in the [Fetcher]) then - /// this method's behaviour will be the same as in [ack] method. - void commit(String metadata) { - _completer.complete(new _ProcessingResult.commit(metadata)); - } - /// Signals that message has been processed and we are ready for - /// the next one. This method will **not** trigger offset commit if this - /// envelope has been created by a high-level [Consumer]. - void ack() { - _completer.complete(new _ProcessingResult.ack()); + Future>> _fetchPartitionLeaders( + GroupSubscription subscription, List offsets) async { + var topics = subscription.assignment.topics; + var topicsMeta = await session.metadata.fetchTopics(topics); + var brokerOffsets = offsets + .where((_) => + subscription.assignment.partitionsAsList.contains(_.topicPartition)) + .toList(growable: false); + return groupBy(brokerOffsets, (_) { + var leaderId = topicsMeta[_.topic].partitions[_.partition].leader; + return topicsMeta.brokers[leaderId]; + }); } - /// Signals to consumer to cancel any further deliveries and close the stream. - void cancel() { - _completer.complete(new _ProcessingResult.cancel()); + @override + Future unsubscribe() { + // TODO: implement unsubscribe + return null; } -} -/// StreamController for batch consuming of messages. -class _BatchStreamController { - final StreamController _controller = - new StreamController(); - bool _cancelled = false; - - bool get canAdd => (_cancelled == false); - Stream get stream => _controller.stream; - - /// Attempts to add [batch] to the stream. - /// Returns true if adding event succeeded, false otherwise. - bool add(BatchEnvelope batch) { - if (canAdd) { - _controller.add(batch); - return true; + @override + Future commit() async { + // TODO: What should happen in case of an unexpected error in here? + // This should probably cancel polling and complete returned future + // with this unexpected error. + assert(_streamIterator != null); + assert(_streamIterator.current != null); + _logger.fine('Committing offsets.'); + var offsets = _streamIterator.offsets; + if (offsets.isNotEmpty) { + return _group + .commitOffsets(_streamIterator.offsets, subscription: _subscription) + .catchError((error) { + /// It is possible to receive a rebalance error in response to OffsetCommit + /// request. We set `_resubscriptionNeeded` to `true` so that next cycle + /// of polling can exit and switch to [_resubscribeState]. + _logger.warning( + 'Received $error on offset commit. Requiring resubscription.'); + _resubscriptionNeeded = true; + }, test: isRebalanceError).whenComplete(() { + _logger.fine('Done committing offsets.'); + + /// Clear accumulated offsets regardless of the result of OffsetCommit. + /// If commit successeded clearing current offsets is safe. + /// If commit failed we either go to resubscribe state which requires re-fetch + /// of offsets, or we have unexpected error so we need to shutdown polling and + /// cleanup internal state. + _streamIterator.clearOffsets(); + }); } - return false; } - void addError(Object error, [StackTrace stackTrace]) { - _controller.addError(error, stackTrace); + @override + void seekToBeginning() { + // TODO: implement seekToBeginning } - void cancel() { - _cancelled = true; - } - - void close() { - _controller.close(); + @override + void seekToEnd() { + // TODO: implement seekToEnd } } -/// Envelope for message batches used by `Consumer.batchConsume`. -class BatchEnvelope { - final List items = new List(); - - Completer<_ProcessingResult> _completer = new Completer<_ProcessingResult>(); - Future<_ProcessingResult> get result => _completer.future; +class ConsumerRecord { + final String topic; + final int partition; + final int offset; + final K key; + final V value; + final int timestamp; - String commitMetadata; + ConsumerRecord(this.topic, this.partition, this.offset, this.key, this.value, + this.timestamp); - /// Signals that batch has been processed and it's offsets can - /// be committed. In case if - /// consumerGroup functionality is not used (like in the [Fetcher]) then - /// this method's behaviour will be the same as in [ack] method. - void commit(String metadata) { - commitMetadata = metadata; - _completer.complete(new _ProcessingResult.commit(metadata)); - } + TopicPartition get topicPartition => new TopicPartition(topic, partition); +} - /// Signals that batch has been processed and we are ready for - /// the next one. This method will **not** trigger offset commit if this - /// envelope has been created by a high-level [Consumer]. - void ack() { - _completer.complete(new _ProcessingResult.ack()); - } +// TODO: bad name, figure out better one +class ConsumerRecords { + final Completer _completer = new Completer(); - /// Signals to consumer to cancel any further deliveries and close the stream. - void cancel() { - _completer.complete(new _ProcessingResult.cancel()); - } + /// Collection of consumed records. + final List> records; - Iterable get offsetsToCommit { - var grouped = new Map(); - for (var envelope in items) { - var key = new TopicPartition(envelope.topicName, envelope.partitionId); - if (!grouped.containsKey(key)) { - grouped[key] = envelope.offset; - } else if (grouped[key] < envelope.offset) { - grouped[key] = envelope.offset; - } - } + ConsumerRecords(this.records); - List offsets = []; - for (var key in grouped.keys) { - offsets.add(new ConsumerOffset( - key.topicName, key.partitionId, grouped[key], commitMetadata)); - } + Future get future => _completer.future; - return offsets; + void ack() { + _completer.complete(true); } + + bool get isCompleted => _completer.isCompleted; } diff --git a/lib/src/consumer_group.dart b/lib/src/consumer_group.dart index 771a691..4c5ef3e 100644 --- a/lib/src/consumer_group.dart +++ b/lib/src/consumer_group.dart @@ -1,92 +1,118 @@ -part of kafka; +import 'dart:async'; + +import 'package:logging/logging.dart'; + +import 'common.dart'; +import 'consumer_offset_api.dart'; +import 'errors.dart'; +import 'group_membership_api.dart'; +import 'offset_commit_api.dart'; +import 'offset_master.dart'; +import 'partition_assignor.dart'; +import 'session.dart'; + +final Logger _logger = new Logger('ConsumerGroup'); class ConsumerGroup { - final KafkaSession session; + /// The session to communicate with Kafka cluster. + final Session session; + + /// The unique name of this group. final String name; - Broker _coordinatorHost; + /// Optional retention time for committed offsets. If `null` then broker's + /// offset retention time will be used as default. + final Duration retentionTime; + + Future _coordinatorHost; - ConsumerGroup(this.session, this.name); + ConsumerGroup(this.session, this.name, {this.retentionTime}); - /// Retrieves offsets of this consumer group from the server. - /// - /// Keys in [topicPartitions] map are topic names and values are corresponding - /// partition IDs. + /// Sends heartbeat for the member specified by [subscription]. + Future heartbeat(GroupSubscription subscription) async { + var host = await _getCoordinator(); + var request = new HeartbeatRequest( + name, subscription.generationId, subscription.memberId); + _logger.fine( + 'Sending heartbeat for member ${subscription.memberId} (generationId: ${subscription.generationId})'); + return session.send(request, host.host, host.port); + } + + /// Retrieves offsets of this consumer group for specified [partitions]. Future> fetchOffsets( - Map> topicPartitions) async { - return _fetchOffsets(topicPartitions, retries: 3); + List partitions) async { + return _fetchOffsets(partitions, retries: 3); } /// Internal method for fetching offsets with retries. - Future> _fetchOffsets( - Map> topicPartitions, - {int retries: 0, - bool refresh: false}) async { - var host = await _getCoordinator(refresh: refresh); - var request = new OffsetFetchRequest(name, topicPartitions); - var response = await session.send(host, request); - var offsets = new List.from(response.offsets); - - for (var offset in offsets) { - var error = new KafkaServerError(offset.errorCode); - if (error.isNotCoordinatorForConsumer && retries > 1) { - // Re-fetch coordinator metadata and try again - kafkaLogger?.info( - 'ConsumerGroup(${name}): encountered API error 16 (NotCoordinatorForConsumerCode) when fetching offsets. Scheduling retry with metadata refresh.'); - return _fetchOffsets(topicPartitions, - retries: retries - 1, refresh: true); - } else if (error.isOffsetsLoadInProgress && retries > 1) { - // Wait a little and try again. - kafkaLogger?.info( - 'ConsumerGroup(${name}): encountered API error 14 (OffsetsLoadInProgressCode) when fetching offsets. Scheduling retry after delay.'); + Future> _fetchOffsets(List partitions, + {int retries: 0, bool refresh: false}) async { + var broker = await _getCoordinator(refresh: refresh); + var request = new OffsetFetchRequest(name, partitions); + try { + var response = await session.send(request, broker.host, broker.port); + return new List.from(response.offsets); + } on NotCoordinatorForConsumerError { + if (retries > 1) { + _logger.info( + 'GroupMember(${name}): encountered NotCoordinatorForConsumerError(16) when fetching offsets. ' + 'Scheduling retry with metadata refresh.'); + return _fetchOffsets(partitions, retries: retries - 1, refresh: true); + } else { + rethrow; + } + } on OffsetsLoadInProgressError { + if (retries > 1) { + _logger.info( + 'GroupMember(${name}): encountered OffsetsLoadInProgressError(14) when fetching offsets. ' + 'Scheduling retry after delay.'); return new Future>.delayed( const Duration(seconds: 1), () async { - return _fetchOffsets(topicPartitions, retries: retries - 1); + return _fetchOffsets(partitions, retries: retries - 1); }); - } else if (error.isError) { - kafkaLogger?.info( - 'ConsumerGroup(${name}): fetchOffsets failed. Error code: ${offset.errorCode} for partition ${offset.partitionId} of ${offset.topicName}.'); - throw error; + } else { + rethrow; } } - - return offsets; } - /// Commits provided [offsets] to the server for this consumer group. - Future commitOffsets(List offsets, int consumerGenerationId, - String consumerId) async { - return _commitOffsets(offsets, consumerGenerationId, consumerId, - retries: 3); + /// Commits provided [partitions] to the server for this consumer group. + Future commitOffsets(List offsets, + {GroupSubscription subscription}) { + return _commitOffsets(offsets, subscription: subscription, retries: 3); } /// Internal method for commiting offsets with retries. - Future _commitOffsets( - List offsets, int consumerGenerationId, String consumerId, - {int retries: 0, bool refresh: false}) async { - var host = await _getCoordinator(refresh: refresh); - var request = new OffsetCommitRequest(name, offsets, consumerGenerationId, - consumerId, -1); // TODO: allow to customize retention time. - OffsetCommitResponse response = await session.send(host, request); - for (var offset in response.offsets) { - var error = new KafkaServerError(offset.errorCode); - if (error.isNotCoordinatorForConsumer && retries > 1) { - // Re-fetch coordinator metadata and try again - kafkaLogger?.info( - 'ConsumerGroup(${name}): encountered API error 16 (NotCoordinatorForConsumerCode) when commiting offsets. Scheduling retry with metadata refresh.'); - return _commitOffsets(offsets, consumerGenerationId, consumerId, - retries: retries - 1, refresh: true); - } else if (error.isError) { - kafkaLogger?.info( - 'ConsumerGroup(${name}): commitOffsets failed. Error code: ${offset.errorCode} for partition ${offset.partitionId} of ${offset.topicName}.'); - throw error; + Future _commitOffsets(List offsets, + {GroupSubscription subscription, + int retries: 0, + bool refresh: false}) async { + try { + var host = await _getCoordinator(refresh: refresh); + var generationId = subscription?.generationId ?? -1; + var memberId = subscription?.memberId ?? ''; + var retentionInMsecs = retentionTime?.inMilliseconds ?? -1; + var request = new OffsetCommitRequest( + name, offsets, generationId, memberId, retentionInMsecs); + + _logger.fine( + "commiting offsets: group_id: $name on $host member_id: $memberId"); + await session.send(request, host.host, host.port); + } on NotCoordinatorForConsumerError { + if (retries > 1) { + _logger.info( + 'ConsumerGroup(${name}): encountered NotCoordinatorForConsumerError(16) when commiting offsets. ' + 'Scheduling retry with metadata refresh.'); + return _commitOffsets(offsets, + subscription: subscription, retries: retries - 1, refresh: true); + } else { + rethrow; } } - - return null; } - Future resetOffsetsToEarliest(Map> topicPartitions) async { + Future resetOffsetsToEarliest(List topicPartitions, + {GroupSubscription subscription}) async { var offsetMaster = new OffsetMaster(session); var earliestOffsets = await offsetMaster.fetchEarliest(topicPartitions); var offsets = new List(); @@ -95,38 +121,145 @@ class ConsumerGroup { // message so here we need to substract 1 from earliest offset, otherwise // we'll end up in an infinite loop of "InvalidOffset" errors. var actualOffset = earliest.offset - 1; - offsets.add(new ConsumerOffset(earliest.topicName, earliest.partitionId, - actualOffset, 'resetToEarliest')); + offsets.add(new ConsumerOffset( + earliest.topic, earliest.partition, actualOffset, 'resetToEarliest')); } - return commitOffsets(offsets, -1, ''); + return commitOffsets(offsets, subscription: subscription); } - Future resetOffsetsToLatest(Map> topicPartitions) async { - var offsetMaster = new OffsetMaster(session); - var latestOffsets = await offsetMaster.fetchLatest(topicPartitions); - var offsets = new List(); - for (var latest in latestOffsets) { - var actualOffset = latest.offset - 1; - offsets.add(new ConsumerOffset(latest.topicName, latest.partitionId, - actualOffset, 'resetToEarliest')); - } - - return commitOffsets(offsets, -1, ''); - } + // Future resetOffsetsToLatest(Map> topicPartitions, + // {GroupMembershipInfo membership}) async { + // var offsetMaster = new OffsetMaster(session); + // var latestOffsets = await offsetMaster.fetchLatest(topicPartitions); + // var offsets = new List(); + // for (var latest in latestOffsets) { + // var actualOffset = latest.offset - 1; + // offsets.add(new ConsumerOffset(latest.topicName, latest.partitionId, + // actualOffset, 'resetToEarliest')); + // } + // + // return commitOffsets(offsets, membership: membership); + // } /// Returns instance of coordinator host for this consumer group. - Future _getCoordinator({bool refresh: false}) async { + Future _getCoordinator({bool refresh: false}) { if (refresh) { _coordinatorHost = null; } if (_coordinatorHost == null) { - var metadata = await session.getConsumerMetadata(name); - _coordinatorHost = new Broker(metadata.coordinatorId, - metadata.coordinatorHost, metadata.coordinatorPort); + _coordinatorHost = + session.metadata.fetchGroupCoordinator(name).catchError((error) { + _coordinatorHost = null; + _logger.severe('Error fetching consumer coordinator.', error); + throw error; + }); } return _coordinatorHost; } + + Future join( + int sessionTimeout, + int rebalanceTimeout, + String memberId, + String protocolType, + Iterable groupProtocols) async { + var broker = await _getCoordinator(); + var joinRequest = new JoinGroupRequest(name, sessionTimeout, + rebalanceTimeout, memberId, protocolType, groupProtocols); + JoinGroupResponse joinResponse = + await session.send(joinRequest, broker.host, broker.port); + var protocol = joinResponse.groupProtocol; + var isLeader = joinResponse.leaderId == joinResponse.memberId; + + var groupAssignments = new List(); + if (isLeader) { + groupAssignments = await _assignPartitions(protocol, joinResponse); + } + + var syncRequest = new SyncGroupRequest(name, joinResponse.generationId, + joinResponse.memberId, groupAssignments); + SyncGroupResponse syncResponse; + try { + // Wait before sending SyncRequest to give the server some time to respond + // to all the rest JoinRequests. + syncResponse = await new Future.delayed(new Duration(seconds: 1), () { + return session.send(syncRequest, broker.host, broker.port); + }); + + return new GroupSubscription( + joinResponse.memberId, + joinResponse.leaderId, + syncResponse.assignment, + joinResponse.generationId, + joinResponse.groupProtocol); + } on RebalanceInProgressError { + _logger.warning( + 'Received "RebalanceInProgress" error code for SyncRequest, will attempt to rejoin again now.'); + return join(sessionTimeout, rebalanceTimeout, memberId, protocolType, + groupProtocols); + } + } + + Future> _assignPartitions( + String protocol, JoinGroupResponse joinResponse) async { + var groupAssignments = new List(); + var assignor = new PartitionAssignor.forStrategy(protocol); + var topics = new Set(); + Map> subscriptions = new Map(); + joinResponse.members.forEach((m) { + var memberProtocol = new GroupProtocol.fromBytes(protocol, m.metadata); + subscriptions[m.id] = memberProtocol.topics; + }); + subscriptions.values.forEach(topics.addAll); + + var meta = await session.metadata.fetchTopics(topics.toList()); + var partitionsPerTopic = new Map.fromIterable(meta.asList, + key: (_) => _.name, value: (_) => _.partitions.length); + + Map> assignments = + assignor.assign(partitionsPerTopic, subscriptions); + for (var memberId in assignments.keys) { + var partitionAssignment = new Map>(); + assignments[memberId].forEach((topicPartition) { + partitionAssignment.putIfAbsent( + topicPartition.topic, () => new List()); + partitionAssignment[topicPartition.topic].add(topicPartition.partition); + }); + groupAssignments.add(new GroupAssignment( + memberId, new MemberAssignment(0, partitionAssignment, null))); + } + + return groupAssignments; + } + + // + // Future leave(GroupMembershipInfo membership) async { + // _logger.info('Attempting to leave group "${name}".'); + // var host = await _getCoordinator(); + // var request = new LeaveGroupRequest(name, membership.memberId); + // return session.send(host, request).catchError((error) { + // _logger.warning('Received ${error} on attempt to leave group gracefully. ' + // 'Ignoring the error to let current session timeout.'); + // }); + // } +} + +class GroupSubscription { + final String memberId; + final String leaderId; + final MemberAssignment assignment; + final int generationId; + final String groupProtocol; + + GroupSubscription(this.memberId, this.leaderId, this.assignment, + this.generationId, this.groupProtocol); + + bool get isLeader => leaderId == memberId; + + @override + String toString() => + 'GroupSubscription{memberId: $memberId, leaderId: $leaderId, assignment: $assignment, generationId: $generationId, protocol: $groupProtocol}'; } diff --git a/lib/src/consumer_metadata_api.dart b/lib/src/consumer_metadata_api.dart new file mode 100644 index 0000000..371203d --- /dev/null +++ b/lib/src/consumer_metadata_api.dart @@ -0,0 +1,68 @@ +import 'common.dart'; +import 'io.dart'; +import 'errors.dart'; + +/// Kafka GroupCoordinator request. +class GroupCoordinatorRequest extends KRequest { + @override + final int apiKey = ApiKey.groupCoordinator; + + /// The name of consumer group to fetch coordinator details for. + final String group; + + GroupCoordinatorRequest(this.group); + + @override + final ResponseDecoder decoder = + const _GroupCoordinatorResponseDecoder(); + + @override + final RequestEncoder encoder = + const _GroupCoordinatorRequestEncoder(); +} + +class GroupCoordinatorResponse { + final int error; + final int coordinatorId; + final String coordinatorHost; + final int coordinatorPort; + + GroupCoordinatorResponse(this.error, this.coordinatorId, this.coordinatorHost, + this.coordinatorPort) { + if (error != Errors.NoError) { + throw new KafkaError.fromCode(error, this); + } + } + + Broker get coordinator => + new Broker(coordinatorId, coordinatorHost, coordinatorPort); +} + +class _GroupCoordinatorRequestEncoder + implements RequestEncoder { + const _GroupCoordinatorRequestEncoder(); + + @override + List encode(GroupCoordinatorRequest request, int version) { + assert(version == 0, + 'Only v0 of GroupCoordinator request is supported by the client.'); + var builder = new KafkaBytesBuilder(); + builder.addString(request.group); + return builder.takeBytes(); + } +} + +class _GroupCoordinatorResponseDecoder + implements ResponseDecoder { + const _GroupCoordinatorResponseDecoder(); + + @override + GroupCoordinatorResponse decode(List data) { + var reader = new KafkaBytesReader.fromBytes(data); + var error = reader.readInt16(); + var id = reader.readInt32(); + var host = reader.readString(); + var port = reader.readInt32(); + return new GroupCoordinatorResponse(error, id, host, port); + } +} diff --git a/lib/src/consumer_offset_api.dart b/lib/src/consumer_offset_api.dart new file mode 100644 index 0000000..6597c83 --- /dev/null +++ b/lib/src/consumer_offset_api.dart @@ -0,0 +1,132 @@ +import 'common.dart'; +import 'errors.dart'; +import 'io.dart'; +import 'util/group_by.dart'; + +/// Consumer position in particular [topic] and [partition]. +class ConsumerOffset { + /// The name of the topic. + final String topic; + + /// The partition number; + final int partition; + + /// The offset of last message handled by a consumer. + final int offset; + + /// User-defined metadata associated with this offset. + final String metadata; + + /// The error code returned by the server. + final int error; + + ConsumerOffset(this.topic, this.partition, this.offset, this.metadata, + [this.error]); + + TopicPartition get topicPartition => new TopicPartition(topic, partition); + + /// Copies this offset and overwrites [offset] and [metadata] if provided. + ConsumerOffset copy({int offset, String metadata}) { + assert(offset != null); + return new ConsumerOffset(topic, partition, offset, metadata); + } + + @override + String toString() => + 'ConsumerOffset{partition: $topic:$partition, offset: $offset, metadata: $metadata'; +} + +/// Kafka OffsetFetchRequest. +/// +/// Throws `GroupLoadInProgressError`, `NotCoordinatorForGroupError`, +/// `IllegalGenerationError`, `UnknownMemberIdError`, `TopicAuthorizationFailedError`, +/// `GroupAuthorizationFailedError`. +class OffsetFetchRequest extends KRequest { + @override + final int apiKey = ApiKey.offsetFetch; + + /// Name of consumer group. + final String group; + + /// Map of topic names and partition IDs to fetch offsets for. + final List partitions; + + /// Creates new instance of [OffsetFetchRequest]. + OffsetFetchRequest(this.group, this.partitions); + + @override + ResponseDecoder get decoder => + const _OffsetFetchResponseDecoder(); + + @override + RequestEncoder get encoder => const _OffsetFetchRequestEncoder(); +} + +class _OffsetFetchRequestEncoder implements RequestEncoder { + const _OffsetFetchRequestEncoder(); + + @override + List encode(OffsetFetchRequest request, int version) { + assert(version == 1, + 'Only v1 of OffsetFetch request supported by the client.'); + var builder = new KafkaBytesBuilder(); + + builder.addString(request.group); + Map> grouped = groupBy( + request.partitions, (partition) => partition.topic); + + builder.addInt32(grouped.length); + grouped.forEach((topic, partitions) { + builder.addString(topic); + var partitionIds = + partitions.map((_) => _.partition).toList(growable: false); + builder.addInt32Array(partitionIds); + }); + + return builder.takeBytes(); + } +} + +/// Kafka OffsetFetchResponse. +class OffsetFetchResponse { + final List offsets; + + OffsetFetchResponse(this.offsets) { + var errors = offsets.map((_) => _.error).where((_) => _ != Errors.NoError); + if (errors.isNotEmpty) { + throw new KafkaError.fromCode(errors.first, this); + } + } +} + +class _OffsetFetchResponseDecoder + implements ResponseDecoder { + const _OffsetFetchResponseDecoder(); + + @override + OffsetFetchResponse decode(List data) { + List offsets = []; + var reader = new KafkaBytesReader.fromBytes(data); + + var count = reader.readInt32(); + while (count > 0) { + String topic = reader.readString(); + int partitionCount = reader.readInt32(); + while (partitionCount > 0) { + ConsumerOffset offset = reader.readObject((_) { + var partition = _.readInt32(); + var offset = _.readInt64(); + var metadata = _.readString(); + var error = _.readInt16(); + return new ConsumerOffset(topic, partition, offset, metadata, error); + }); + + offsets.add(offset); + partitionCount--; + } + count--; + } + + return new OffsetFetchResponse(offsets); + } +} diff --git a/lib/src/consumer_streamiterator.dart b/lib/src/consumer_streamiterator.dart new file mode 100644 index 0000000..2e0e173 --- /dev/null +++ b/lib/src/consumer_streamiterator.dart @@ -0,0 +1,210 @@ +import 'dart:async'; +import 'common.dart'; +import 'consumer.dart'; +import 'consumer_offset_api.dart'; + +/** + * Extended version of built-in [StreamIterator] implementation. + * + * Pauses the stream between calls to [moveNext]. + */ +class ConsumerStreamIterator + implements StreamIterator> { + // The stream iterator is always in one of four states. + // The value of the [_stateData] field depends on the state. + // + // When `_subscription == null` and `_stateData != null`: + // The stream iterator has been created, but [moveNext] has not been called + // yet. The [_stateData] field contains the stream to listen to on the first + // call to [moveNext] and [current] returns `null`. + // + // When `_subscription != null` and `!_isPaused`: + // The user has called [moveNext] and the iterator is waiting for the next + // event. The [_stateData] field contains the [_Future] returned by the + // [_moveNext] call and [current] returns `null.` + // + // When `_subscription != null` and `_isPaused`: + // The most recent call to [moveNext] has completed with a `true` value + // and [current] provides the value of the data event. + // The [_stateData] field contains the [current] value. + // + // When `_subscription == null` and `_stateData == null`: + // The stream has completed or been canceled using [cancel]. + // The stream completes on either a done event or an error event. + // The last call to [moveNext] has completed with `false` and [current] + // returns `null`. + + /// Subscription being listened to. + /// + /// Set to `null` when the stream subscription is done or canceled. + StreamSubscription> _subscription; + + /// Data value depending on the current state. + /// + /// Before first call to [moveNext]: The stream to listen to. + /// + /// After calling [moveNext] but before the returned future completes: + /// The returned future. + /// + /// After calling [moveNext] and the returned future has completed + /// with `true`: The value of [current]. + /// + /// After calling [moveNext] and the returned future has completed + /// with `false`, or after calling [cancel]: `null`. + Object _stateData; + + /// Whether the iterator is between calls to `moveNext`. + /// This will usually cause the [_subscription] to be paused, but as an + /// optimization, we only pause after the [moveNext] future has been + /// completed. + bool _isPaused = false; + + /// Stores accumulated consumer offsets for each topic-partition. + /// + /// Offsets are aggregated so that only latest consumed offset is stored. + /// These offsets are used by [Consumer] to commit to the server. + final Map _offsets = new Map(); + + ConsumerStreamIterator(final Stream> stream) + : _stateData = stream; + + List get offsets => _offsets.values.toList(growable: false); + + /// Removes all accumulated offsets. + /// + /// This method is called by [Consumer] after successful commit of current + /// offsets. + void clearOffsets() { + _offsets.clear(); + } + + void _updateOffsets(ConsumerRecords records) { + for (var record in records.records) { + var partition = record.topicPartition; + // TODO: handle metadata? + _offsets[partition] = + new ConsumerOffset(record.topic, record.partition, record.offset, ''); + } + } + + ConsumerRecords get current { + if (_subscription != null && _isPaused) { + return _stateData as ConsumerRecords; + } + return null; + } + + /// Attaches new stream to this iterator. + /// + /// [Consumer] calls this method in case of a rebalance event. + /// This cancels active subscription (if exists) so that no new events can be + /// delivered to the listener from the previous stream. + /// Subscription to the new [stream] is created immediately and current state + /// of the listener is preserved. + void attachStream(Stream> stream) { + Completer completer; + Object records; + if (_subscription != null) { + _subscription.cancel(); + _subscription = null; + if (!_isPaused) { + // User waits for `moveNext` to complete. + completer = _stateData as Completer; + } else { + records = _stateData; + } + } + // During rebalance offset commits are not accepted by the server and result + // in RebalanceInProgress error (or UnknownMemberId if rebalance completed). + clearOffsets(); + _stateData = stream; + _initializeOrDone(); + // Restore state after initialize. + if (_isPaused) { + _subscription.pause(); + _stateData = records; + } else { + _subscription.resume(); + _stateData = completer; + } + } + + Future moveNext() { + if (_subscription != null) { + if (_isPaused) { + var records = _stateData as ConsumerRecords; + // Acknowledge this record set. Signals to consumer to resume polling. + records.ack(); + + var completer = new Completer(); + _stateData = completer; + _isPaused = false; + _subscription.resume(); + return completer.future; + } + throw new StateError("Already waiting for next."); + } + return _initializeOrDone(); + } + + /// Called if there is no active subscription when [moveNext] is called. + /// + /// Either starts listening on the stream if this is the first call to + /// [moveNext], or returns a `false` future because the stream has already + /// ended. + Future _initializeOrDone() { + assert(_subscription == null); + var stateData = _stateData; + if (stateData != null) { + Stream> stream = + stateData as Stream>; + _subscription = stream.listen(_onData, + onError: _onError, onDone: _onDone, cancelOnError: true); + var completer = new Completer(); + _stateData = completer; + return completer.future; + } + return new Future.value(false); + } + + Future cancel() { + StreamSubscription> subscription = _subscription; + Object stateData = _stateData; + _stateData = null; + if (subscription != null) { + _subscription = null; + if (!_isPaused) { + Completer completer = stateData as Completer; + completer.complete(false); + } + return subscription.cancel(); + } + return new Future.value(null); + } + + void _onData(ConsumerRecords data) { + assert(_subscription != null && !_isPaused); + Completer moveNextFuture = _stateData as Completer; + _stateData = data; + _isPaused = true; + _updateOffsets(data); + moveNextFuture.complete(true); + if (_subscription != null && _isPaused) _subscription.pause(); + } + + void _onError(Object error, [StackTrace stackTrace]) { + assert(_subscription != null && !_isPaused); + Completer moveNextFuture = _stateData as Completer; + _subscription = null; + _stateData = null; + moveNextFuture.completeError(error, stackTrace); + } + + void _onDone() { + assert(_subscription != null && !_isPaused); + Completer moveNextFuture = _stateData as Completer; + _subscription = null; + _stateData = null; + moveNextFuture.complete(false); + } +} diff --git a/lib/src/errors.dart b/lib/src/errors.dart new file mode 100644 index 0000000..6170894 --- /dev/null +++ b/lib/src/errors.dart @@ -0,0 +1,310 @@ +import 'common.dart'; + +/// Used to indicate there is a mismatch in CRC sum of a message (message is +/// corrupted). +class MessageCrcMismatchError extends StateError { + MessageCrcMismatchError(String message) : super(message); +} + +/// List of all Kafka server error codes. +abstract class Errors { + static const int NoError = 0; + static const int Unknown = -1; + static const int OffsetOutOfRange = 1; + static const int InvalidMessage = 2; + static const int UnknownTopicOrPartition = 3; + static const int InvalidMessageSize = 4; + static const int LeaderNotAvailable = 5; + static const int NotLeaderForPartition = 6; + static const int RequestTimedOut = 7; + static const int BrokerNotAvailable = 8; + static const int ReplicaNotAvailable = 9; + static const int MessageSizeTooLarge = 10; + static const int StaleControllerEpoch = 11; + static const int OffsetMetadataTooLarge = 12; + static const int OffsetsLoadInProgress = 14; + static const int ConsumerCoordinatorNotAvailable = 15; + static const int NotCoordinatorForConsumer = 16; + static const int InvalidTopic = 17; + static const int RecordListTooLarge = 18; + static const int NotEnoughReplicas = 19; + static const int NotEnoughReplicasAfterAppend = 20; + static const int InvalidRequiredAcks = 21; + static const int IllegalGeneration = 22; + static const int InconsistentGroupProtocol = 23; + static const int InvalidGroupId = 24; + static const int UnknownMemberId = 25; + static const int InvalidSessionTimeout = 26; + static const int RebalanceInProgress = 27; + static const int InvalidCommitOffsetSize = 28; + static const int TopicAuthorizationFailed = 29; + static const int GroupAuthorizationFailed = 30; + static const int ClusterAuthorizationFailed = 31; + static const int InvalidTimestamp = 32; + static const int UnsupportedSaslMechanism = 33; + static const int IllegalSaslState = 34; + static const int UnsupportedVersion = 35; +} + +/// Represents errors returned by Kafka server. +class KafkaError { + /// Numeric code of this server error. + final int code; + + /// The response object associated containing this error. + final response; + + KafkaError._(this.code, this.response); + + factory KafkaError.fromCode(int code, response) { + switch (code) { + case Errors.Unknown: + return new UnknownError(response); + case Errors.OffsetOutOfRange: + return new OffsetOutOfRangeError(response, null); + case Errors.InvalidMessage: + return new InvalidMessageError(response); + case Errors.UnknownTopicOrPartition: + return new UnknownTopicOrPartitionError(response); + case Errors.InvalidMessageSize: + return new InvalidMessageSizeError(response); + case Errors.LeaderNotAvailable: + return new LeaderNotAvailableError(response); + case Errors.NotLeaderForPartition: + return new NotLeaderForPartitionError(response); + case Errors.RequestTimedOut: + return new RequestTimedOutError(response); + case Errors.BrokerNotAvailable: + return new BrokerNotAvailableError(response); + case Errors.ReplicaNotAvailable: + return new ReplicaNotAvailableError(response); + case Errors.MessageSizeTooLarge: + return new MessageSizeTooLargeError(response); + case Errors.StaleControllerEpoch: + return new StaleControllerEpochError(response); + case Errors.OffsetMetadataTooLarge: + return new OffsetMetadataTooLargeError(response); + case Errors.OffsetsLoadInProgress: + return new OffsetsLoadInProgressError(response); + case Errors.ConsumerCoordinatorNotAvailable: + return new ConsumerCoordinatorNotAvailableError(response); + case Errors.NotCoordinatorForConsumer: + return new NotCoordinatorForConsumerError(response); + case Errors.InvalidTopic: + return new InvalidTopicError(response); + case Errors.RecordListTooLarge: + return new RecordListTooLargeError(response); + case Errors.NotEnoughReplicas: + return new NotEnoughReplicasError(response); + case Errors.NotEnoughReplicasAfterAppend: + return new NotEnoughReplicasAfterAppendError(response); + case Errors.InvalidRequiredAcks: + return new InvalidRequiredAcksError(response); + case Errors.IllegalGeneration: + return new IllegalGenerationError(response); + case Errors.InconsistentGroupProtocol: + return new InconsistentGroupProtocolError(response); + case Errors.InvalidGroupId: + return new InvalidGroupIdError(response); + case Errors.UnknownMemberId: + return new UnknownMemberIdError(response); + case Errors.InvalidSessionTimeout: + return new InvalidSessionTimeoutError(response); + case Errors.RebalanceInProgress: + return new RebalanceInProgressError(response); + case Errors.InvalidCommitOffsetSize: + return new InvalidCommitOffsetSizeError(response); + case Errors.TopicAuthorizationFailed: + return new TopicAuthorizationFailedError(response); + case Errors.GroupAuthorizationFailed: + return new GroupAuthorizationFailedError(response); + case Errors.ClusterAuthorizationFailed: + return new ClusterAuthorizationFailedError(response); + case Errors.InvalidTimestamp: + return new InvalidTimestampError(response); + case Errors.UnsupportedSaslMechanism: + return new UnsupportedSaslMechanismError(response); + case Errors.IllegalSaslState: + return new IllegalSaslStateError(response); + case Errors.UnsupportedVersion: + return new UnsupportedVersionError(response); + default: + throw new ArgumentError('Unsupported Kafka server error code $code.'); + } + } + + @override + String toString() => '${runtimeType}(${code})'; +} + +class NoError extends KafkaError { + NoError(response) : super._(Errors.NoError, response); +} + +class UnknownError extends KafkaError { + UnknownError(response) : super._(Errors.Unknown, response); +} + +class OffsetOutOfRangeError extends KafkaError { + final List topicPartitions; + OffsetOutOfRangeError(response, this.topicPartitions) + : super._(Errors.OffsetOutOfRange, response); +} + +class InvalidMessageError extends KafkaError { + InvalidMessageError(response) : super._(Errors.InvalidMessage, response); +} + +class UnknownTopicOrPartitionError extends KafkaError { + UnknownTopicOrPartitionError(response) + : super._(Errors.UnknownTopicOrPartition, response); +} + +class InvalidMessageSizeError extends KafkaError { + InvalidMessageSizeError(response) + : super._(Errors.InvalidMessageSize, response); +} + +class LeaderNotAvailableError extends KafkaError { + LeaderNotAvailableError(response) + : super._(Errors.LeaderNotAvailable, response); +} + +class NotLeaderForPartitionError extends KafkaError { + NotLeaderForPartitionError(response) + : super._(Errors.NotLeaderForPartition, response); +} + +class RequestTimedOutError extends KafkaError { + RequestTimedOutError(response) : super._(Errors.RequestTimedOut, response); +} + +class BrokerNotAvailableError extends KafkaError { + BrokerNotAvailableError(response) + : super._(Errors.BrokerNotAvailable, response); +} + +class ReplicaNotAvailableError extends KafkaError { + ReplicaNotAvailableError(response) + : super._(Errors.ReplicaNotAvailable, response); +} + +class MessageSizeTooLargeError extends KafkaError { + MessageSizeTooLargeError(response) + : super._(Errors.MessageSizeTooLarge, response); +} + +class StaleControllerEpochError extends KafkaError { + StaleControllerEpochError(response) + : super._(Errors.StaleControllerEpoch, response); +} + +class OffsetMetadataTooLargeError extends KafkaError { + OffsetMetadataTooLargeError(response) + : super._(Errors.OffsetMetadataTooLarge, response); +} + +class OffsetsLoadInProgressError extends KafkaError { + OffsetsLoadInProgressError(response) + : super._(Errors.OffsetsLoadInProgress, response); +} + +class ConsumerCoordinatorNotAvailableError extends KafkaError { + ConsumerCoordinatorNotAvailableError(response) + : super._(Errors.ConsumerCoordinatorNotAvailable, response); +} + +class NotCoordinatorForConsumerError extends KafkaError { + NotCoordinatorForConsumerError(response) + : super._(Errors.NotCoordinatorForConsumer, response); +} + +class InvalidTopicError extends KafkaError { + InvalidTopicError(response) : super._(Errors.InvalidTopic, response); +} + +class RecordListTooLargeError extends KafkaError { + RecordListTooLargeError(response) + : super._(Errors.RecordListTooLarge, response); +} + +class NotEnoughReplicasError extends KafkaError { + NotEnoughReplicasError(response) + : super._(Errors.NotEnoughReplicas, response); +} + +class NotEnoughReplicasAfterAppendError extends KafkaError { + NotEnoughReplicasAfterAppendError(response) + : super._(Errors.NotEnoughReplicasAfterAppend, response); +} + +class InvalidRequiredAcksError extends KafkaError { + InvalidRequiredAcksError(response) + : super._(Errors.InvalidRequiredAcks, response); +} + +class IllegalGenerationError extends KafkaError { + IllegalGenerationError(response) + : super._(Errors.IllegalGeneration, response); +} + +class InconsistentGroupProtocolError extends KafkaError { + InconsistentGroupProtocolError(response) + : super._(Errors.InconsistentGroupProtocol, response); +} + +class InvalidGroupIdError extends KafkaError { + InvalidGroupIdError(response) : super._(Errors.InvalidGroupId, response); +} + +class UnknownMemberIdError extends KafkaError { + UnknownMemberIdError(response) : super._(Errors.UnknownMemberId, response); +} + +class InvalidSessionTimeoutError extends KafkaError { + InvalidSessionTimeoutError(response) + : super._(Errors.InvalidSessionTimeout, response); +} + +class RebalanceInProgressError extends KafkaError { + RebalanceInProgressError(response) + : super._(Errors.RebalanceInProgress, response); +} + +class InvalidCommitOffsetSizeError extends KafkaError { + InvalidCommitOffsetSizeError(response) + : super._(Errors.InvalidCommitOffsetSize, response); +} + +class TopicAuthorizationFailedError extends KafkaError { + TopicAuthorizationFailedError(response) + : super._(Errors.TopicAuthorizationFailed, response); +} + +class GroupAuthorizationFailedError extends KafkaError { + GroupAuthorizationFailedError(response) + : super._(Errors.GroupAuthorizationFailed, response); +} + +class ClusterAuthorizationFailedError extends KafkaError { + ClusterAuthorizationFailedError(response) + : super._(Errors.ClusterAuthorizationFailed, response); +} + +class InvalidTimestampError extends KafkaError { + InvalidTimestampError(response) : super._(Errors.InvalidTimestamp, response); +} + +class UnsupportedSaslMechanismError extends KafkaError { + UnsupportedSaslMechanismError(response) + : super._(Errors.UnsupportedSaslMechanism, response); +} + +class IllegalSaslStateError extends KafkaError { + IllegalSaslStateError(response) : super._(Errors.IllegalSaslState, response); +} + +class UnsupportedVersionError extends KafkaError { + UnsupportedVersionError(response) + : super._(Errors.UnsupportedVersion, response); +} diff --git a/lib/src/fetch_api.dart b/lib/src/fetch_api.dart new file mode 100644 index 0000000..6430514 --- /dev/null +++ b/lib/src/fetch_api.dart @@ -0,0 +1,216 @@ +import 'dart:io'; + +import 'package:logging/logging.dart'; + +import 'common.dart'; +import 'errors.dart'; +import 'io.dart'; +import 'messages.dart'; +import 'util/crc32.dart'; + +final _logger = new Logger('FetchApi'); + +/// Kafka FetchRequest. +class FetchRequest implements KRequest { + @override + final int apiKey = ApiKey.fetch; + + /// The replica id indicates the node id of the replica initiating this request. + /// Normal consumers should always specify this as -1 as they have no node id. + static const int replicaId = -1; + + /// Maximum amount of time in milliseconds to block waiting if insufficient + /// data is available at the time the request is issued. + final int maxWaitTime; + + /// Minimum number of bytes of messages that must be available + /// to give a response. + final int minBytes; + + /// Topics and partitions to fetch messages from. + final Map fetchData = new Map(); + + /// Creates new instance of FetchRequest. + FetchRequest(this.maxWaitTime, this.minBytes); + + void add(TopicPartition partition, FetchData data) { + fetchData[partition] = data; + } + + @override + toString() => 'FetchRequest{${maxWaitTime}, ${minBytes}, ${fetchData}}'; + + @override + ResponseDecoder get decoder => const _FetchResponseDecoder(); + + @override + RequestEncoder get encoder => const _FetchRequestEncoder(); + + Map> _fetchDataByTopic; + Map> get fetchDataByTopic { + if (_fetchDataByTopic == null) { + var result = new Map>(); + fetchData.keys.forEach((_) { + result.putIfAbsent(_.topic, () => new Map()); + result[_.topic][_.partition] = fetchData[_]; + }); + _fetchDataByTopic = result; + } + return _fetchDataByTopic; + } +} + +class FetchData { + final int fetchOffset; + final int maxBytes; + FetchData(this.fetchOffset, this.maxBytes); +} + +class _FetchRequestEncoder implements RequestEncoder { + const _FetchRequestEncoder(); + + @override + List encode(FetchRequest request, int version) { + assert( + version == 2, 'Only v2 of Fetch request is supported by the client.'); + var builder = new KafkaBytesBuilder(); + + builder.addInt32(FetchRequest.replicaId); + builder.addInt32(request.maxWaitTime); + builder.addInt32(request.minBytes); + + builder.addInt32(request.fetchDataByTopic.length); + request.fetchDataByTopic.forEach((topic, partitions) { + builder.addString(topic); + builder.addInt32(partitions.length); + partitions.forEach((partition, data) { + builder.addInt32(partition); + builder.addInt64(data.fetchOffset); + builder.addInt32(data.maxBytes); + }); + }); + return builder.takeBytes(); + } +} + +/// Kafka FetchResponse. +class FetchResponse { + /// Duration in milliseconds for which the request was throttled due to quota + /// violation. (Zero if the request did not violate any quota.) + final int throttleTime; + + /// List of [FetchResult]s for each topic-partition. + final List results; + + FetchResponse(this.throttleTime, this.results) { + var errors = results.map((_) => _.error).where((_) => _ != Errors.NoError); + if (errors.isNotEmpty) { + throw new KafkaError.fromCode(errors.first, this); + } + } +} + +/// Represents result of fetching messages for a particular +/// topic-partition. +class FetchResult { + final String topic; + final int partition; + final int error; + final int highwaterMarkOffset; + final Map messages; + + FetchResult(this.topic, this.partition, this.error, this.highwaterMarkOffset, + this.messages); +} + +class _FetchResponseDecoder implements ResponseDecoder { + const _FetchResponseDecoder(); + + @override + FetchResponse decode(List data) { + var reader = new KafkaBytesReader.fromBytes(data); + var throttleTime = reader.readInt32(); + var count = reader.readInt32(); + var results = new List(); + while (count > 0) { + var topic = reader.readString(); + var partitionCount = reader.readInt32(); + while (partitionCount > 0) { + var partition = reader.readInt32(); + var error = reader.readInt16(); + var highwaterMarkOffset = reader.readInt64(); + var messageSetSize = reader.readInt32(); + var data = reader.readRaw(messageSetSize); + var messageReader = new KafkaBytesReader.fromBytes(data); + var messageSet = _readMessageSet(messageReader); + + results.add(new FetchResult( + topic, partition, error, highwaterMarkOffset, messageSet)); + partitionCount--; + } + count--; + } + + return new FetchResponse(throttleTime, results); + } + + /// Reads a set of messages from FetchResponse. + Map _readMessageSet(KafkaBytesReader reader) { + int messageSize = -1; + var messages = new Map(); + while (reader.isNotEOF) { + try { + int offset = reader.readInt64(); + messageSize = reader.readInt32(); + var crc = reader.readInt32(); + + var data = reader.readRaw(messageSize - 4); + var actualCrc = Crc32.signed(data); + if (actualCrc != crc) { + _logger.warning( + 'Message CRC sum mismatch. Expected crc: ${crc}, actual: ${actualCrc}'); + throw new MessageCrcMismatchError( + 'Expected crc: ${crc}, actual: ${actualCrc}'); + } + var messageReader = new KafkaBytesReader.fromBytes(data); + var message = _readMessage(messageReader); + if (message.attributes.compression == Compression.none) { + messages[offset] = message; + } else { + if (message.attributes.compression == Compression.snappy) + throw new UnimplementedError( + 'Snappy compression is not supported yet by the client.'); + + var codec = new GZipCodec(); + var innerReader = + new KafkaBytesReader.fromBytes(codec.decode(message.value)); + var innerMessageSet = _readMessageSet(innerReader); + messages.addAll(innerMessageSet); + } + } on RangeError { + // According to the spec server is allowed to return partial + // messages, so we just ignore it here and exit the loop. + var remaining = reader.length - reader.offset; + _logger.info('Encountered partial message. ' + 'Expected message size: ${messageSize}, bytes left in ' + 'buffer: ${remaining}, total buffer size ${reader.length}'); + break; + } + } + return messages; + } + + Message _readMessage(KafkaBytesReader reader) { + final magicByte = reader.readInt8(); + assert(magicByte == 1, + 'Unsupported message format $magicByte. Only version 1 is supported by the client.'); + final attrByte = reader.readInt8(); + final attributes = new MessageAttributes.fromByte(attrByte); + final timestamp = reader.readInt64(); + + final key = reader.readBytes(); + final value = reader.readBytes(); + return new Message(value, + attributes: attributes, key: key, timestamp: timestamp); + } +} diff --git a/lib/src/fetcher.dart b/lib/src/fetcher.dart deleted file mode 100644 index 0f8ce9c..0000000 --- a/lib/src/fetcher.dart +++ /dev/null @@ -1,153 +0,0 @@ -part of kafka; - -/// Message Fetcher. -/// -/// Main difference to [Consumer] is that this class does not store it's state -/// in consumer metadata. -/// -/// It will fetch all messages starting from specified [topicOffsets]. If no -/// limit is set it will run forever consuming all incoming messages. -class Fetcher { - /// Instance of Kafka session. - final KafkaSession session; - - /// Offsets to start from. - final List topicOffsets; - - Fetcher(this.session, this.topicOffsets); - - /// Consumes messages from Kafka topics. - /// - /// It will start from specified [topicOffsets]. If no [limit] is set it will - /// run continuously consuming all incoming messages. - Stream fetch({int limit: -1}) { - var controller = new _MessageStreamController(limit); - - Future> list = _buildWorkers(controller); - list.then((workers) { - if (workers.isEmpty) { - controller.close(); - return; - } - var remaining = workers.length; - var futures = workers.map((w) => w.run()).toList(); - futures.forEach((Future f) { - f.then((_) { - remaining--; - if (remaining == 0) { - kafkaLogger - ?.info('Fetcher: All workers are done. Closing the stream.'); - controller.close(); - } - }); - }); - }); - - return controller.stream; - } - - Future> _buildWorkers( - _MessageStreamController controller) async { - var topicNames = new Set.from(topicOffsets.map((_) => _.topicName)); - var meta = await session.getMetadata(topicNames); - var offsetsByBroker = new Map>(); - - topicOffsets.forEach((offset) { - var leader = meta - .getTopicMetadata(offset.topicName) - .getPartition(offset.partitionId) - .leader; - var broker = meta.getBroker(leader); - if (offsetsByBroker.containsKey(broker) == false) { - offsetsByBroker[broker] = new List(); - } - offsetsByBroker[broker].add(offset); - }); - - var workers = new List<_FetcherWorker>(); - offsetsByBroker.forEach((host, offsets) { - workers - .add(new _FetcherWorker(session, host, controller, offsets, 100, 1)); - }); - - return workers; - } -} - -class _FetcherWorker { - final KafkaSession session; - final Broker broker; - final _MessageStreamController controller; - final List startFromOffsets; - final int maxWaitTime; - final int minBytes; - - _FetcherWorker(this.session, this.broker, this.controller, - this.startFromOffsets, this.maxWaitTime, this.minBytes); - - Future run() async { - kafkaLogger?.info( - 'Fetcher: Running worker on broker ${broker.host}:${broker.port}'); - var offsets = startFromOffsets.toList(); - - while (controller.canAdd) { - var request = await _createRequest(offsets); - FetchResponse response = await session.send(broker, request); - _checkResponseForErrors(response); - - for (var item in response.results) { - for (var offset in item.messageSet.messages.keys) { - var message = item.messageSet.messages[offset]; - var envelope = new MessageEnvelope( - item.topicName, item.partitionId, offset, message); - if (!controller.add(envelope)) { - return; - } else { - var result = await envelope.result; - if (result.status == _ProcessingStatus.cancel) { - controller.cancel(); - return; - } - } - } - if (item.messageSet.messages.isNotEmpty) { - var nextOffset = new TopicOffset(item.topicName, item.partitionId, - item.messageSet.messages.keys.last + 1); - var previousOffset = offsets.firstWhere((o) => - o.topicName == item.topicName && - o.partitionId == item.partitionId); - offsets.remove(previousOffset); - offsets.add(nextOffset); - } - } - } - } - - Future _createRequest(List offsets) async { - var offsetMaster = new OffsetMaster(session); - var request = new FetchRequest(maxWaitTime, minBytes); - for (var o in offsets) { - if (o.isEarliest) { - var result = await offsetMaster.fetchEarliest({ - o.topicName: [o.partitionId].toSet() - }); - request.add(result.first.topicName, result.first.partitionId, - result.first.offset); - } else { - request.add(o.topicName, o.partitionId, o.offset); - } - } - - return request; - } - - _checkResponseForErrors(FetchResponse response) { - if (!response.hasErrors) return; - - for (var result in response.results) { - if (result.errorCode != KafkaServerError.NoError) { - throw new KafkaServerError(result.errorCode); - } - } - } -} diff --git a/lib/src/group_membership_api.dart b/lib/src/group_membership_api.dart new file mode 100644 index 0000000..990dc1a --- /dev/null +++ b/lib/src/group_membership_api.dart @@ -0,0 +1,402 @@ +import 'io.dart'; +import 'errors.dart'; +import 'common.dart'; + +class JoinGroupRequest implements KRequest { + @override + final int apiKey = ApiKey.joinGroup; + + /// The name of consumer group to join. + final String group; + + /// The coordinator considers the consumer dead if it receives no heartbeat + /// after this timeout (in ms). + final int sessionTimeout; + + /// The maximum time that the coordinator will wait for each member to rejoin + /// when rebalancing the group. + final int rebalanceTimeout; + + /// The assigned consumer id or an empty string for a new consumer. + final String memberId; + + /// Unique name for class of protocols implemented by group. + final String protocolType; + + /// List of protocols that the member supports. + final List groupProtocols; + + /// Creates new JoinGroupRequest. + /// + /// [sessionTimeout] (in msecs) defines window after which coordinator will + /// consider group member dead if no heartbeats were received. + /// + /// When a member first joins the group, the [memberId] will be empty (""), + /// but a rejoining member should use the same memberId from the previous generation. + /// + /// For details on [protocolType] see Kafka documentation. This library implements + /// standard "consumer" embedded protocol described in Kafka docs. + /// + /// [groupProtocols] depends on `protocolType`. Each member joining member must + /// provide list of protocols it supports. See Kafka docs for more details. + JoinGroupRequest(this.group, this.sessionTimeout, this.rebalanceTimeout, + this.memberId, this.protocolType, this.groupProtocols); + + @override + ResponseDecoder get decoder => + const _JoinGroupResponseDecoder(); + + @override + RequestEncoder get encoder => const _JoinGroupRequestEncoder(); +} + +abstract class GroupProtocol { + String get protocolName; + List get protocolMetadata; + Set get topics; + + factory GroupProtocol.roundrobin(int version, Set topics) { + return new RoundRobinGroupProtocol(version, topics); + } + + factory GroupProtocol.fromBytes(String name, List data) { + switch (name) { + case 'roundrobin': + return new RoundRobinGroupProtocol.fromBytes(data); + default: + throw new StateError('Unsupported group protocol "$name"'); + } + } +} + +class RoundRobinGroupProtocol implements GroupProtocol { + @override + List get protocolMetadata { + var builder = new KafkaBytesBuilder(); + builder + ..addInt16(version) + ..addStringArray(topics.toList()) + ..addBytes(null); + return builder.takeBytes(); + } + + @override + String get protocolName => 'roundrobin'; + + final int version; + + final Set topics; + + RoundRobinGroupProtocol(this.version, this.topics); + + factory RoundRobinGroupProtocol.fromBytes(List data) { + var reader = new KafkaBytesReader.fromBytes(data); + var version = reader.readInt16(); + Set topics = reader.readStringArray().toSet(); + reader.readBytes(); // user data (unused) + + return new RoundRobinGroupProtocol(version, topics); + } +} + +class JoinGroupResponse { + final int error; + final int generationId; + final String groupProtocol; + final String leaderId; + final String memberId; + final List members; + + JoinGroupResponse(this.error, this.generationId, this.groupProtocol, + this.leaderId, this.memberId, this.members) { + if (error != Errors.NoError) throw new KafkaError.fromCode(error, this); + } +} + +class GroupMember { + final String id; + final List metadata; + + GroupMember(this.id, this.metadata); +} + +class _JoinGroupRequestEncoder implements RequestEncoder { + const _JoinGroupRequestEncoder(); + + @override + List encode(JoinGroupRequest request, int version) { + assert(version == 1, + 'Only v1 of JoinGroup request is supported by the client.'); + + var builder = new KafkaBytesBuilder(); + builder.addString(request.group); + builder.addInt32(request.sessionTimeout); + builder.addInt32(request.rebalanceTimeout); + builder.addString(request.memberId); + builder.addString(request.protocolType); + builder.addInt32(request.groupProtocols.length); + request.groupProtocols.forEach((protocol) { + builder.addString(protocol.protocolName); + builder.addBytes(protocol.protocolMetadata); + }); + return builder.takeBytes(); + } +} + +class _JoinGroupResponseDecoder implements ResponseDecoder { + const _JoinGroupResponseDecoder(); + + @override + JoinGroupResponse decode(List data) { + var reader = new KafkaBytesReader.fromBytes(data); + var error = reader.readInt16(); + var generationId = reader.readInt32(); + var groupProtocol = reader.readString(); + var leaderId = reader.readString(); + var memberId = reader.readString(); + List members = reader + .readObjectArray((_) => new GroupMember(_.readString(), _.readBytes())); + return new JoinGroupResponse( + error, generationId, groupProtocol, leaderId, memberId, members); + } +} + +class SyncGroupRequest implements KRequest { + @override + final int apiKey = ApiKey.syncGroup; + + /// The name of consumer group. + final String group; + final int generationId; + final String memberId; + final List groupAssignments; + + SyncGroupRequest( + this.group, this.generationId, this.memberId, this.groupAssignments); + + @override + ResponseDecoder get decoder => + const _SyncGroupResponseDecoder(); + + @override + RequestEncoder get encoder => const _SyncGroupRequestEncoder(); +} + +class GroupAssignment { + final String memberId; + final MemberAssignment memberAssignment; + + GroupAssignment(this.memberId, this.memberAssignment); +} + +class MemberAssignment { + final int version; + final Map> partitions; + final List userData; + + MemberAssignment(this.version, this.partitions, this.userData); + + List _partitionsList; + List get partitionsAsList { + if (_partitionsList != null) return _partitionsList; + var result = new List(); + for (var topic in partitions.keys) { + result.addAll(partitions[topic].map((p) => new TopicPartition(topic, p))); + } + _partitionsList = result.toList(growable: false); + return _partitionsList; + } + + List _topics; + + /// List of topic names in this member assignment. + List get topics { + if (_topics != null) return _topics; + _topics = partitions.keys.toList(growable: false); + return _topics; + } + + @override + String toString() => + 'MemberAssignment{version: $version, partitions: $partitions}'; +} + +class SyncGroupResponse { + final int error; + final MemberAssignment assignment; + + SyncGroupResponse(this.error, this.assignment) { + if (error != Errors.NoError) throw new KafkaError.fromCode(error, this); + } +} + +class _SyncGroupRequestEncoder implements RequestEncoder { + const _SyncGroupRequestEncoder(); + + @override + List encode(SyncGroupRequest request, int version) { + assert(version == 0, + 'Only v0 of SyncGroup request is supported by the client.'); + + var builder = new KafkaBytesBuilder(); + builder.addString(request.group); + builder.addInt32(request.generationId); + builder.addString(request.memberId); + + builder.addInt32(request.groupAssignments.length); + request.groupAssignments.forEach((member) { + builder.addString(member.memberId); + builder.addBytes(_encodeMemberAssignment(member.memberAssignment)); + }); + return builder.takeBytes(); + } + + List _encodeMemberAssignment(MemberAssignment assignment) { + var builder = new KafkaBytesBuilder(); + builder.addInt16(assignment.version); + builder.addInt32(assignment.partitions.length); + for (var topic in assignment.partitions.keys) { + builder.addString(topic); + builder.addInt32Array(assignment.partitions[topic]); + } + builder.addBytes(assignment.userData); + return builder.takeBytes(); + } +} + +class _SyncGroupResponseDecoder implements ResponseDecoder { + const _SyncGroupResponseDecoder(); + + @override + SyncGroupResponse decode(List data) { + var reader = new KafkaBytesReader.fromBytes(data); + var error = reader.readInt16(); + var assignmentData = reader.readBytes(); + MemberAssignment assignment; + if (assignmentData.isNotEmpty) { + var reader = new KafkaBytesReader.fromBytes(assignmentData); + var version = reader.readInt16(); + var length = reader.readInt32(); + Map> partitionAssignments = new Map(); + for (var i = 0; i < length; i++) { + var topic = reader.readString(); + partitionAssignments[topic] = reader.readInt32Array(); + } + var userData = reader.readBytes(); + assignment = + new MemberAssignment(version, partitionAssignments, userData); + } + + return new SyncGroupResponse(error, assignment); + } +} + +class LeaveGroupRequest implements KRequest { + @override + final int apiKey = ApiKey.leaveGroup; + + /// The name of consumer group. + final String group; + + /// The ID of the member leaving the group. + final String memberId; + + LeaveGroupRequest(this.group, this.memberId); + + @override + ResponseDecoder get decoder => + const _LeaveGroupResponseDecoder(); + + @override + RequestEncoder get encoder => const _LeaveGroupRequestEncoder(); +} + +class LeaveGroupResponse { + final int error; + + LeaveGroupResponse(this.error) { + if (error != Errors.NoError) throw new KafkaError.fromCode(error, this); + } +} + +class _LeaveGroupRequestEncoder implements RequestEncoder { + const _LeaveGroupRequestEncoder(); + + @override + List encode(LeaveGroupRequest request, int version) { + assert(version == 0, + 'Only v0 of LeaveGroup request is supported by the client.'); + var builder = new KafkaBytesBuilder(); + builder..addString(request.group)..addString(request.memberId); + return builder.takeBytes(); + } +} + +class _LeaveGroupResponseDecoder + implements ResponseDecoder { + const _LeaveGroupResponseDecoder(); + + @override + LeaveGroupResponse decode(List data) { + var reader = new KafkaBytesReader.fromBytes(data); + var error = reader.readInt16(); + return new LeaveGroupResponse(error); + } +} + +class HeartbeatRequest implements KRequest { + @override + final int apiKey = ApiKey.heartbeat; + + /// The name of consumer group. + final String group; + + /// The ID of group generation. + final int generationId; + + /// The ID of the member sending this heartbeat. + final String memberId; + + HeartbeatRequest(this.group, this.generationId, this.memberId); + + @override + ResponseDecoder get decoder => + const _HeartbeatResponseDecoder(); + + @override + RequestEncoder get encoder => const _HeartbeatRequestEncoder(); +} + +class HeartbeatResponse { + final int error; + HeartbeatResponse(this.error) { + if (error != Errors.NoError) throw new KafkaError.fromCode(error, this); + } +} + +class _HeartbeatRequestEncoder implements RequestEncoder { + const _HeartbeatRequestEncoder(); + + @override + List encode(HeartbeatRequest request, int version) { + assert(version == 0, + 'Only v0 of Heartbeat request is supported by the client.'); + var builder = new KafkaBytesBuilder(); + builder + ..addString(request.group) + ..addInt32(request.generationId) + ..addString(request.memberId); + return builder.takeBytes(); + } +} + +class _HeartbeatResponseDecoder implements ResponseDecoder { + const _HeartbeatResponseDecoder(); + + @override + HeartbeatResponse decode(List data) { + var reader = new KafkaBytesReader.fromBytes(data); + var error = reader.readInt16(); + return new HeartbeatResponse(error); + } +} diff --git a/lib/src/io.dart b/lib/src/io.dart new file mode 100644 index 0000000..5df6bc3 --- /dev/null +++ b/lib/src/io.dart @@ -0,0 +1,359 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:ffi'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:logging/logging.dart'; + +/// Bytes builder specific to Kafka protocol. +/// +/// Provides convenient methods for writing all Kafka data types (and some more): +/// int8, int16, int32, string, bytes, array. +class KafkaBytesBuilder { + BytesBuilder _builder = new BytesBuilder(); + + int get length => _builder.length; + + /// Creates new builder with empty buffer. + KafkaBytesBuilder(); + + /// Creates new builder and initializes buffer with proper request header. + KafkaBytesBuilder.withRequestHeader( + int apiKey, int apiVersion, int correlationId) { + addInt16(apiKey); + addInt16(apiVersion); + addInt32(correlationId); + addString('dart_kafka-0.10'); + } + + /// Adds 8 bit integer to this buffer. + void addInt8(int value) { + ByteData bdata = new ByteData(1); + bdata.setInt8(0, value); + _add(bdata); + } + + /// Adds 16 bit integer to this buffer. + void addInt16(int value) { + ByteData bdata = new ByteData(2); + bdata.setInt16(0, value); + _add(bdata); + } + + /// Adds 32 bit integer to this buffer. + void addInt32(int value) { + ByteData bdata = new ByteData(4); + bdata.setInt32(0, value); + _add(bdata); + } + + /// Adds 64 bit integer to this buffer. + void addInt64(int value) { + ByteData bdata = new ByteData(8); + bdata.setInt64(0, value); + _add(bdata); + } + + /// Adds Kafka string to this bytes builder. + /// + /// Kafka string type starts with int16 indicating size of the string + /// followed by the actual string value. + void addString(String value) { + List data = utf8.encode(value); + addInt16(data.length); + _builder.add(data); + } + + void _addArray(List items, addFunc(T item)) { + addInt32(items.length); + items.forEach((_) { + addFunc(_); + }); + } + + void addInt8Array(List items) { + _addArray(items, addInt8); + } + + void addInt16Array(List items) { + _addArray(items, addInt16); + } + + void addInt32Array(List items) { + _addArray(items, addInt32); + } + + void addInt64Array(List items) { + _addArray(items, addInt64); + } + + void addStringArray(List items) { + _addArray(items, addString); + } + + void addBytesArray(List> items) { + _addArray>(items, addBytes); + } + + /// Adds value of Kafka-specific Bytes type to this builder. + /// + /// Kafka Bytes type starts with int32 indicating size of the value following + /// by actual value bytes. + void addBytes(List value) { + if (value == null) { + addInt32(-1); + } else { + addInt32(value.length); + _builder.add(value); + } + } + + /// Adds arbitrary data to this buffer. + void addRaw(List data) { + _builder.add(data); + } + + void _add(ByteData data) { + _builder.add(data.buffer.asInt8List().toList(growable: false)); + } + + List takeBytes() => _builder.takeBytes(); + + List toBytes() => _builder.toBytes(); +} + +/// Provides convenience methods to read Kafka specific data types from a +/// stream of bytes. +class KafkaBytesReader { + Int8List _data; + int _offset = 0; + + /// Current position in this buffer. + int get offset => _offset; + + /// Size of this byte buffer. + int get length => _data.length; + + /// Whether this bytes buffer has been fully read. + bool get isEOF => _data.length == _offset; + + /// Whether there are still unread bytes left in this buffer. + bool get isNotEOF => !isEOF; + + /// Creates reader from a list of bytes. + KafkaBytesReader.fromBytes(List data) { + this._data = new Int8List.fromList(data); + } + + // Reads int8 from the data and returns it. + int readInt8() { + var data = new ByteData.view(_data.buffer, _offset, 1); + var value = data.getInt8(0); + _offset += 1; + + return value; + } + + /// Reads 16-bit integer from the current position of this buffer. + int readInt16() { + var data = new ByteData.view(_data.buffer, _offset, 2); + var value = data.getInt16(0); + _offset += 2; + + return value; + } + + /// Reads 32-bit integer from the current position of this buffer. + int readInt32() { + var data = new ByteData.view(_data.buffer, _offset, 4); + var value = data.getInt32(0); + _offset += 4; + + return value; + } + + /// Reads 64-bit integer from the current position of this buffer. + int readInt64() { + var data = new ByteData.view(_data.buffer, _offset, 8); + var value = data.getInt64(0); + _offset += 8; + + return value; + } + + String readString() { + var length = readInt16(); + var value = _data.buffer.asInt8List(_offset, length).toList(); + var valueAsString = utf8.decode(value); + _offset += length; + + return valueAsString; + } + + T readObject(T readFunc(KafkaBytesReader reader)) => readFunc(this); + + List readBytes() { + var length = readInt32(); + if (length == -1) { + return null; + } else { + var value = _data.buffer.asInt8List(_offset, length).toList(); + _offset += length; + return value; + } + } + + List readInt8Array() => _readArray(readInt8); + List readInt16Array() => _readArray(readInt16); + List readInt32Array() => _readArray(readInt32); + List readInt64Array() => _readArray(readInt64); + List readStringArray() => _readArray(readString); + List> readBytesArray() => _readArray(readBytes); + List readObjectArray(T readFunc(KafkaBytesReader reader)) { + return _readArray(() => readFunc(this)); + } + + List _readArray(T reader()) { + var length = readInt32(); + var items = new List(); + for (var i = 0; i < length; i++) { + items.add(reader()); + } + return items; + } + + /// Reads raw bytes from this buffer. + List readRaw(int length) { + var value = _data.buffer.asInt8List(_offset, length).toList(); + _offset += length; + + return value; + } +} + +class PacketStreamTransformer + implements StreamTransformer> { + List _data = new List(); + StreamController> _controller = new StreamController>(); + + @override + Stream> bind(Stream> stream) { + stream.listen(_onData, onError: _onError, onDone: _onDone); + return _controller.stream; + } + + void _onData(List data) { + _data.addAll(data); + + while (true) { + if (_data.length >= 4) { + var sizeBytes = new Int8List.fromList(_data.sublist(0, 4)); + var bd = new ByteData.view(sizeBytes.buffer); + var size = bd.getInt32(0); + if (_data.length >= size + 4) { + List packetData = _data.sublist(4, size + 4); + _controller.add(packetData); + _data.removeRange(0, size + 4); + } else { + break; // not enough data + } + } else { + break; // not enough data + } + } + } + + void _onError(error) { + _controller.addError(error); + } + + void _onDone() { + _controller.close(); + } + + @override + StreamTransformer cast() => StreamTransformer.castFrom(this); +} + +/// Handles Kafka channel multiplexing. +// TODO: Consider using RawSocket internally for more efficiency. +class KSocket { + static Logger _logger = new Logger('KSocket'); + + final Socket _ioSocket; + Stream> _stream; + StreamSubscription> _subscription; + + int _nextCorrelationId = 1; + int get nextCorrelationId => _nextCorrelationId++; + + Map>> _inflightRequests = new Map(); + + KSocket._(this._ioSocket) { + _ioSocket.setOption(SocketOption.tcpNoDelay, true); + _stream = _ioSocket.transform(PacketStreamTransformer()); + _subscription = _stream.listen(_onPacket); + } + + static Future connect(String host, int port) { + return Socket.connect(host, port).then((socket) => new KSocket._(socket)); + } + + Future _flushFuture = new Future.value(); + + Future> sendPacket(int apiKey, int apiVersion, List payload) { + var correlationId = nextCorrelationId; + var completer = new Completer>(); + _inflightRequests[correlationId] = completer; + + var bb = new KafkaBytesBuilder.withRequestHeader( + apiKey, apiVersion, correlationId); + var header = bb.takeBytes(); + bb.addInt32(header.length + payload.length); + var length = bb.takeBytes(); + + _flushFuture = _flushFuture.then((_) { + _ioSocket..add(length)..add(header)..add(payload); + return _ioSocket.flush(); + }).catchError((error) { + completer.completeError(error); + _inflightRequests.remove(correlationId); + }); + return completer.future; + } + + void _onPacket(List packet) { + var r = new KafkaBytesReader.fromBytes(packet.sublist(0, 4)); + var correlationId = r.readInt32(); + var completer = _inflightRequests[correlationId]; + if (completer.isCompleted) { + _logger.warning('Received packet for already completed request.'); + } else { + packet.removeRange(0, 4); // removes correlationId from the payload + completer.complete(packet); + } + _inflightRequests.remove(correlationId); + } + + Future destroy() async { + await _subscription.cancel(); + _ioSocket.destroy(); + } +} + +abstract class RequestEncoder { + List encode(T request, int version); +} + +abstract class ResponseDecoder { + T decode(List data); +} + +abstract class KRequest { + /// Unique numeric key of this API request. + int get apiKey; + + RequestEncoder get encoder; + ResponseDecoder get decoder; +} diff --git a/lib/src/list_offset_api.dart b/lib/src/list_offset_api.dart new file mode 100644 index 0000000..ff626f8 --- /dev/null +++ b/lib/src/list_offset_api.dart @@ -0,0 +1,126 @@ +import 'common.dart'; +import 'errors.dart'; +import 'io.dart'; +import 'util/group_by.dart'; +import 'util/tuple.dart'; + +/// Kafka ListOffsetRequest. +class ListOffsetRequest extends KRequest { + @override + final int apiKey = ApiKey.offsets; + + /// Unique ID assigned to the host within Kafka cluster. Regular consumers + /// should always pass `-1`. + static const int replicaId = -1; + + /// Map of topic-partitions to timestamps (in msecs) + final Map topics; + + /// Creates new instance of OffsetRequest. + /// + /// The [topics] argument is a `Map` where keys are instances of + /// [TopicPartition] and values are integer timestamps (ms). Timestamps + /// are used to ask for all entries before a certain time. Specify `-1` to + /// receive the latest offset (i.e. the offset of the next coming message) + /// and -2 to receive the earliest available offset. + ListOffsetRequest(this.topics); + + @override + ResponseDecoder get decoder => + const _ListOffsetResponseDecoder(); + + @override + RequestEncoder get encoder => const _ListOffsetRequestEncoder(); +} + +/// Kafka ListOffsetResponse. +class ListOffsetResponse { + /// List of offsets for each topic-partition. + final List offsets; + + ListOffsetResponse(this.offsets) { + var errorOffset = offsets.firstWhere((_) => _.error != Errors.NoError, + orElse: () => null); + if (errorOffset != null) { + throw new KafkaError.fromCode(errorOffset.error, this); + } + } +} + +/// Data structure representing offsets of particular topic-partition returned +/// by [ListOffsetRequest]. +class TopicOffset { + final String topic; + final int partition; + final int error; + final int timestamp; + final int offset; + + TopicOffset( + this.topic, this.partition, this.error, this.timestamp, this.offset); + + @override + toString() => + 'TopicOffset{$topic-$partition, error: $error, timestamp: $timestamp, offset: $offset}'; +} + +class _ListOffsetRequestEncoder implements RequestEncoder { + const _ListOffsetRequestEncoder(); + + @override + List encode(ListOffsetRequest request, int version) { + assert(version == 1, + 'Only v1 of ListOffset request is supported by the client.'); + var builder = new KafkaBytesBuilder(); + builder.addInt32(ListOffsetRequest.replicaId); + + // + List> items = request.topics.keys.map((_) { + return tuple3(_.topic, _.partition, request.topics[_]); + }).toList(growable: false); + + Map>> groupedByTopic = + groupBy(items, (_) => _.$1); + + builder.addInt32(groupedByTopic.length); + groupedByTopic.forEach((topic, partitions) { + builder.addString(topic); + builder.addInt32(partitions.length); + partitions.forEach((p) { + builder.addInt32(p.$2); + builder.addInt64(p.$3); + }); + }); + + return builder.takeBytes(); + } +} + +class _ListOffsetResponseDecoder + implements ResponseDecoder { + const _ListOffsetResponseDecoder(); + + @override + ListOffsetResponse decode(List data) { + var reader = new KafkaBytesReader.fromBytes(data); + + var count = reader.readInt32(); + var offsets = new List(); + while (count > 0) { + var topic = reader.readString(); + var partitionCount = reader.readInt32(); + while (partitionCount > 0) { + var partition = reader.readInt32(); + var error = reader.readInt16(); + var timestamp = reader.readInt64(); + var offset = reader.readInt64(); + offsets + .add(new TopicOffset(topic, partition, error, timestamp, offset)); + partitionCount--; + } + count--; + } + + return new ListOffsetResponse(offsets); + } +} diff --git a/lib/src/messages.dart b/lib/src/messages.dart new file mode 100644 index 0000000..7504b01 --- /dev/null +++ b/lib/src/messages.dart @@ -0,0 +1,103 @@ +import 'dart:collection'; + +/// Compression types supported by Kafka. +enum Compression { none, gzip, snappy } + +/// Types of [Message]'s timestamp supported by Kafka. +enum TimestampType { createTime, logAppendTime } + +const int _compressionMask = 0x07; + +const Map _kIntToCompression = const { + 0: Compression.none, + 1: Compression.gzip, + 2: Compression.snappy, +}; + +const Map _kIntToTimestamptype = const { + 0: TimestampType.createTime, + 1: TimestampType.logAppendTime, +}; + +/// Kafka Message Attributes. +class MessageAttributes { + /// Compression codec. + final Compression compression; + + /// The type of this message's timestamp. + final TimestampType timestampType; + + /// Creates new instance of MessageAttributes. + MessageAttributes( + {this.compression = Compression.none, + this.timestampType = TimestampType.createTime}); + + /// Creates MessageAttributes from the raw byte. + MessageAttributes.fromByte(int byte) + : compression = _kIntToCompression[byte & _compressionMask], + timestampType = _kIntToTimestamptype[(byte >> 3) & 1]; + + @override + String toString() => '{compression: $compression, tsType: $timestampType}'; +} + +/// Kafka Message as defined in the protocol. +class Message { + /// Metadata attributes about this message. + final MessageAttributes attributes; + + /// Actual message contents. + final List value; + + /// Optional message key that was used for partition assignment. + /// The key can be `null`. + final List key; + + /// The timestamp of this message, in msecs. + final int timestamp; + + /// Default internal constructor. + Message._(this.attributes, this.key, this.value, this.timestamp); + + /// Creates new [Message]. + factory Message(List value, + {MessageAttributes attributes, List key, int timestamp}) { + attributes ??= new MessageAttributes(); + timestamp ??= new DateTime.now().millisecondsSinceEpoch; + return new Message._(attributes, key, value, timestamp); + } +} + +/// Kafka MessageSet type. +class MessageSet { + /// Collection of messages. Keys in the map are message offsets. + final Map _messages; + + /// Map of message offsets to corresponding messages. + Map get messages => new UnmodifiableMapView(_messages); + + /// Number of messages in this set. + int get length => _messages.length; + + MessageSet(this._messages); + + /// Builds new message set for publishing. +// factory MessageSet.build(envelope) { +// if (envelope.compression == KafkaCompression.none) { +// return new MessageSet(envelope.messages.asMap()); +// } else { +// if (envelope.compression == KafkaCompression.snappy) +// throw new ArgumentError( +// 'Snappy compression is not supported yet by the client.'); +// +// // var codec = new GZipCodec(); +// // var innerEnvelope = new ProduceEnvelope( +// // envelope.topicName, envelope.partitionId, envelope.messages); +// // var innerMessageSet = new MessageSet.build(innerEnvelope); +// // var value = codec.encode(innerMessageSet.toBytes()); +// // var attrs = new MessageAttributes(KafkaCompression.gzip); +// +// // return new MessageSet({0: new Message(value, attributes: attrs)}); +// } +// } +} diff --git a/lib/src/metadata.dart b/lib/src/metadata.dart new file mode 100644 index 0000000..5af44a1 --- /dev/null +++ b/lib/src/metadata.dart @@ -0,0 +1,93 @@ +import 'dart:async'; + +import 'package:logging/logging.dart'; + +import 'common.dart'; +import 'consumer_metadata_api.dart'; +import 'errors.dart'; +import 'metadata_api.dart'; +import 'session.dart'; +import 'util/retry.dart'; + +final Logger _logger = new Logger('Metadata'); + +/// Provides access to Kafka cluster metadata like list of nodes +/// in the cluster, topics and coordinators for consumer groups. +abstract class Metadata { + /// Creates new instance of [Metadata] provider. + /// + /// Users shouldn't normally need to create a new instance themselves since + /// [Session] provides access to one already via `metadata` field. + /// + /// List of [bootstrapServers] is used to establish connection with + /// Kafka cluster. Each value in this list must be of format `host:port`. + factory Metadata(List bootstrapServers, Session session) { + assert(bootstrapServers != null && bootstrapServers.isNotEmpty); + + List bootstrapUris = bootstrapServers + .map((_) => Uri.parse('kafka://$_')) + .toList(growable: false); + var isValid = bootstrapUris.every((_) => _.host != null && _.port != null); + if (!isValid) + throw new ArgumentError( + 'Invalid bootstrap servers list provided: $bootstrapServers'); + + return new _Metadata(session, bootstrapUris); + } + + Future fetchTopics(List topics); + Future> listTopics(); + Future> listBrokers(); + Future fetchGroupCoordinator(String groupName); +} + +/// Default implementation of [Metadata] interface. +class _Metadata implements Metadata { + final List bootstrapUris; + final Session session; + + _Metadata(this.session, this.bootstrapUris); + + Future fetchTopics(List topics) { + Future fetch() { + var req = new MetadataRequest(topics); + var broker = bootstrapUris.first; + return session + .send(req, broker.host, broker.port) + .then((response) => response.topics); + } + + return retryAsync(fetch, 5, new Duration(milliseconds: 500), + test: (err) => err is LeaderNotAvailableError); + } + + Future> listTopics() { + var req = new MetadataRequest(); + var broker = bootstrapUris.first; + return session.send(req, broker.host, broker.port).then((response) { + return response.topics.names; + }); + } + + Future> listBrokers() { + var req = new MetadataRequest(); + var broker = bootstrapUris.first; + return session + .send(req, broker.host, broker.port) + .then((response) => response.brokers); + } + + Future fetchGroupCoordinator(String groupName) { + Future fetch() { + _logger.info('Fetching group coordinator for group $groupName.'); + var req = new GroupCoordinatorRequest(groupName); + var broker = bootstrapUris.first; + return session.send(req, broker.host, broker.port).then((res) => + new Broker( + res.coordinatorId, res.coordinatorHost, res.coordinatorPort)); + } + + return retryAsync(fetch, 5, new Duration(milliseconds: 1000), + test: (err) => err is ConsumerCoordinatorNotAvailableError); + } +} diff --git a/lib/src/metadata_api.dart b/lib/src/metadata_api.dart new file mode 100644 index 0000000..0583f44 --- /dev/null +++ b/lib/src/metadata_api.dart @@ -0,0 +1,189 @@ +import 'common.dart'; +import 'errors.dart'; +import 'io.dart'; + +/// Kafka MetadataRequest. +class MetadataRequest extends KRequest { + @override + final int apiKey = ApiKey.metadata; + + @override + final RequestEncoder encoder = const _MetadataRequestEncoder(); + + @override + final ResponseDecoder decoder = + const _MetadataResponseDecoder(); + + final List topics; + + /// Creates MetadataRequest. + /// + /// If [topics] is not set it fetches information about all existing + /// topics in the Kafka cluster. + MetadataRequest([this.topics]); +} + +class MetadataResponse { + /// List of brokers in a Kafka cluster. + final List brokers; + + /// List of topics in a Kafka cluster. + final Topics topics; + + MetadataResponse(this.brokers, this.topics) { + var errorTopic = topics._topics + .firstWhere((_) => _.error != Errors.NoError, orElse: () => null); + // TODO: also loop through partitions to find errors on a partition level. + if (errorTopic is Topic) { + throw KafkaError.fromCode(errorTopic.error, this); + } + } +} + +/// Represents a set of Kafka topics. +class Topics { + final List _topics; + + /// List of Kafka brokers. + final Brokers brokers; + + Topics(this._topics, this.brokers); + + Topic operator [](String topic) => asMap[topic]; + + List get asList => List.unmodifiable(_topics); + + Map _asMap; + + /// Returns a map where keys are topic names. + Map get asMap { + if (_asMap != null) return _asMap; + var map = Map.fromIterable( + _topics, + key: (topic) => topic.name, + ); + _asMap = Map.unmodifiable(map); + return _asMap; + } + + /// The list of topic names. + List get names { + return asMap.keys.toList(growable: false); + } + + /// The size of this topics set. + int get length => _topics.length; + + List _topicPartitions; + + /// List of topic-partitions accross all topics in this set. + List get topicPartitions { + if (_topicPartitions != null) return _topicPartitions; + _topicPartitions = _topics.expand((topic) { + return topic.partitions._partitions + .map((partition) => TopicPartition(topic.name, partition.id)); + }).toList(growable: false); + return _topicPartitions; + } +} + +/// Represents a list of Kafka brokers. +class Brokers { + final List _brokers; + + Brokers(this._brokers); + + Broker operator [](int id) => asMap[id]; + + Map _asMap; + Map get asMap { + if (_asMap != null) return _asMap; + var map = Map.fromIterable(_brokers, key: (broker) => broker.id); + _asMap = Map.unmodifiable(map); + return _asMap; + } +} + +class Topic { + final int error; + final String name; + final Partitions partitions; + + Topic(this.error, this.name, this.partitions); + + @override + toString() => 'Topic{$name, error: $error; $partitions}'; +} + +class Partitions { + final List _partitions; + + Partitions(this._partitions); + + Partition operator [](int id) => asMap[id]; + + Map _asMap; + Map get asMap { + if (_asMap != null) return _asMap; + _asMap = Map.fromIterable(_partitions, key: (partition) => partition.id); + return _asMap; + } + + /// Number of partitions. + int get length => _partitions.length; +} + +class Partition { + final int error; + final int id; + final int leader; + final List replicas; + final List inSyncReplicas; + + Partition( + this.error, this.id, this.leader, this.replicas, this.inSyncReplicas); + + @override + toString() => 'Partition#$id{error: $error, ' + 'leader: $leader, replicas: $replicas, isr: $inSyncReplicas}'; +} + +class _MetadataRequestEncoder implements RequestEncoder { + const _MetadataRequestEncoder(); + + @override + List encode(MetadataRequest request, int version) { + assert(version == 0, + 'Only v0 of Metadata request is supported by the client, $version given.'); + var builder = KafkaBytesBuilder(); + List topics = request.topics ?? List(); + builder.addStringArray(topics); + return builder.takeBytes(); + } +} + +class _MetadataResponseDecoder implements ResponseDecoder { + const _MetadataResponseDecoder(); + + @override + MetadataResponse decode(List data) { + var reader = KafkaBytesReader.fromBytes(data); + List brokers = reader.readObjectArray((r) { + return Broker(r.readInt32(), r.readString(), r.readInt32()); + }); + + var topics = reader.readObjectArray((r) { + var error = reader.readInt16(); + var topic = reader.readString(); + + List partitions = reader.readObjectArray((r) => Partition( + r.readInt16(), + r.readInt32(), + r.readInt32(), + r.readInt32Array(), + r.readInt32Array())); + return Topic(error, topic, Partitions(partitions)); + }); + return MetadataResponse(brokers, Topics(topics, Brokers(brokers))); + } +} diff --git a/lib/src/offset_commit_api.dart b/lib/src/offset_commit_api.dart new file mode 100644 index 0000000..ba47c0b --- /dev/null +++ b/lib/src/offset_commit_api.dart @@ -0,0 +1,116 @@ +import 'common.dart'; +import 'consumer_offset_api.dart'; +import 'errors.dart'; +import 'io.dart'; +import 'util/group_by.dart'; + +/// Kafka OffsetCommitRequest. +class OffsetCommitRequest extends KRequest { + @override + final int apiKey = ApiKey.offsetCommit; + + /// The name of consumer group. + final String group; + + /// The generation ID of consumer group. + final int generationId; + + /// The ID of consumer group member. + final String consumerId; + + /// Time period in msec to retain the offset. + final int retentionTime; + + /// List of consumer offsets to be committed. + final List offsets; + + /// Creates new instance of OffsetCommit request. + OffsetCommitRequest(this.group, this.offsets, this.generationId, + this.consumerId, this.retentionTime); + + @override + ResponseDecoder get decoder => + const _OffsetCommitResponseDecoder(); + + @override + RequestEncoder get encoder => const _OffsetCommitRequestEncoder(); +} + +/// OffsetCommitResponse. +class OffsetCommitResponse { + final List results; + + OffsetCommitResponse(this.results) { + var errorResult = results.firstWhere((_) => _.error != Errors.NoError, + orElse: () => null); + if (errorResult != null) + throw new KafkaError.fromCode(errorResult.error, this); + } +} + +/// Result of commiting a consumer offset. +class OffsetCommitResult { + final String topic; + final int partition; + final int error; + + OffsetCommitResult(this.topic, this.partition, this.error); +} + +class _OffsetCommitRequestEncoder + implements RequestEncoder { + const _OffsetCommitRequestEncoder(); + + @override + List encode(OffsetCommitRequest request, int version) { + assert(version == 2, + 'Only v2 of OffsetCommit request is supported by the client.'); + + var builder = new KafkaBytesBuilder(); + Map> groupedByTopic = + groupBy(request.offsets, (o) => o.topic); + + builder.addString(request.group); + builder.addInt32(request.generationId); + builder.addString(request.consumerId); + builder.addInt64(request.retentionTime); + builder.addInt32(groupedByTopic.length); + groupedByTopic.forEach((topic, partitionOffsets) { + builder.addString(topic); + builder.addInt32(partitionOffsets.length); + partitionOffsets.forEach((p) { + builder.addInt32(p.partition); + builder.addInt64(p.offset); + builder.addString(p.metadata); + }); + }); + + return builder.takeBytes(); + } +} + +class _OffsetCommitResponseDecoder + implements ResponseDecoder { + const _OffsetCommitResponseDecoder(); + + @override + OffsetCommitResponse decode(List data) { + var reader = new KafkaBytesReader.fromBytes(data); + List results = []; + var count = reader.readInt32(); + + while (count > 0) { + var topic = reader.readString(); + var partitionCount = reader.readInt32(); + while (partitionCount > 0) { + var partition = reader.readInt32(); + var error = reader.readInt16(); + results.add(new OffsetCommitResult(topic, partition, error)); + partitionCount--; + } + count--; + } + + return new OffsetCommitResponse(results); + } +} diff --git a/lib/src/offset_master.dart b/lib/src/offset_master.dart index 815abdc..3e94750 100644 --- a/lib/src/offset_master.dart +++ b/lib/src/offset_master.dart @@ -1,87 +1,55 @@ -part of kafka; +import 'dart:async'; +import 'common.dart'; +import 'session.dart'; +import 'list_offset_api.dart'; /// Master of Offsets. /// /// Encapsulates auto-discovery logic for fetching topic offsets. class OffsetMaster { - /// Instance of KafkaSession. - final KafkaSession session; + /// The session used by this OffsetMaster. + final Session session; /// Creates new OffsetMaster. OffsetMaster(this.session); /// Returns earliest offsets for specified topics and partitions. - Future> fetchEarliest( - Map> topicPartitions) { - return _fetch(topicPartitions, -2); + Future> fetchEarliest(List partitions) { + return _fetch(partitions, -2); } /// Returns latest offsets (that is the offset of next incoming message) /// for specified topics and partitions. /// - /// These offsets are also called 'highWatermark' offsets in Kafka docs. - Future> fetchLatest(Map> topicPartitions) { - return _fetch(topicPartitions, -1); + /// These offsets are also known as 'high watermark' offsets. + Future> fetchLatest(List partitions) { + return _fetch(partitions, -1); } Future> _fetch( - Map> topicPartitions, int time, - {refreshMetadata: false}) async { - var meta = await session.getMetadata(topicPartitions.keys.toSet(), - invalidateCache: refreshMetadata); - var requests = new Map(); - for (var topic in topicPartitions.keys) { - var partitions = topicPartitions[topic]; - for (var p in partitions) { - var leader = meta.getTopicMetadata(topic).getPartition(p).leader; - var host = meta.getBroker(leader); - if (!requests.containsKey(host)) { - requests[host] = new OffsetRequest(leader); - } - requests[host].addTopicPartition(topic, p, time, 1); - } + List partitions, int time) async { + var topics = partitions.map((_) => _.topic).toSet(); + var meta = + await session.metadata.fetchTopics(topics.toList(growable: false)); + var requests = new Map>(); + var brokers = meta.brokers; + for (var p in partitions) { + var leaderId = meta[p.topic].partitions[p.partition].leader; + var broker = brokers[leaderId]; + requests.putIfAbsent(broker, () => new List()); + requests[broker].add(p); } var offsets = new List(); for (var host in requests.keys) { - var request = requests[host]; - OffsetResponse response = await session.send(host, request); - for (var o in response.offsets) { - var error = new KafkaServerError(o.errorCode); - if (error.isNotLeaderForPartition && refreshMetadata == false) { - // Refresh metadata and try again. - return _fetch(topicPartitions, time, refreshMetadata: true); - } - - if (error.isError) throw error; - offsets - .add(new TopicOffset(o.topicName, o.partitionId, o.offsets.first)); - } + var fetchInfo = new Map.fromIterable(requests[host], + value: (partition) => time); + var request = new ListOffsetRequest(fetchInfo); + ListOffsetResponse response = + await session.send(request, host.host, host.port); + offsets.addAll(response.offsets); } return offsets; } } - -/// Represents an offset of particular topic and partition. -class TopicOffset { - final String topicName; - final int partitionId; - final int offset; - - TopicOffset(this.topicName, this.partitionId, this.offset); - - /// Creates pseudo-offset which refers to earliest offset in this topic - /// and partition. - TopicOffset.earliest(this.topicName, this.partitionId) : offset = -2; - - /// Creates pseudo-offset which refers to latest offset in this topic and - /// partition. - TopicOffset.latest(this.topicName, this.partitionId) : offset = -1; - - /// Indicates whether this is an earliest pseudo-offset. - bool get isEarliest => offset == -2; - - /// Indicates whether this is a latest pseudo-offset. - bool get isLatest => offset == -1; -} diff --git a/lib/src/partition_assignor.dart b/lib/src/partition_assignor.dart new file mode 100644 index 0000000..347f95b --- /dev/null +++ b/lib/src/partition_assignor.dart @@ -0,0 +1,53 @@ +import 'common.dart'; + +abstract class PartitionAssignor { + Map> assign(Map partitionsPerTopic, + Map> memberSubscriptions); + + factory PartitionAssignor.forStrategy(String assignmentStrategy) { + switch (assignmentStrategy) { + case 'roundrobin': + return new RoundRobinPartitionAssignor(); + default: + throw new ArgumentError( + 'Unsupported assignment strategy "$assignmentStrategy" for PartitionAssignor.'); + } + } +} + +/// Partition assignor implementing simple "round-robin" algorithm. +/// +/// It can only be used if the set of subscribed topics is identical for every +/// member within consumer group. +class RoundRobinPartitionAssignor implements PartitionAssignor { + @override + Map> assign(Map partitionsPerTopic, + Map> memberSubscriptions) { + var topics = new Set(); + memberSubscriptions.values.forEach(topics.addAll); + if (!memberSubscriptions.values + .every((list) => list.length == topics.length)) { + throw new StateError( + 'RoundRobinPartitionAssignor: All members must subscribe to the same topics. ' + 'Subscriptions given: $memberSubscriptions.'); + } + + Map> assignments = new Map.fromIterable( + memberSubscriptions.keys, + value: (_) => new List()); + + var offset = 0; + for (var topic in partitionsPerTopic.keys) { + List partitions = new List.generate( + partitionsPerTopic[topic], (_) => new TopicPartition(topic, _)); + for (var p in partitions) { + var k = (offset + p.partition) % memberSubscriptions.keys.length; + var memberId = memberSubscriptions.keys.elementAt(k); + assignments[memberId].add(p); + } + offset += partitions.last.partition + 1; + } + + return assignments; + } +} diff --git a/lib/src/produce_api.dart b/lib/src/produce_api.dart new file mode 100644 index 0000000..5b972db --- /dev/null +++ b/lib/src/produce_api.dart @@ -0,0 +1,202 @@ +import 'common.dart'; +import 'errors.dart'; +import 'io.dart'; +import 'messages.dart'; +import 'util/crc32.dart'; + +class ProduceRequest extends KRequest { + @override + final int apiKey = ApiKey.produce; + + /// Indicates how many acknowledgements the servers + /// should receive before responding to the request. + final int requiredAcks; + + /// Provides a maximum time in milliseconds the server + /// can await the receipt of the number of acknowledgements in [requiredAcks]. + final int timeout; + + /// Collection of messages to produce. + final Map>> messages; + + ProduceRequest(this.requiredAcks, this.timeout, this.messages); + + @override + ResponseDecoder get decoder => + const _ProduceResponseDecoder(); + + @override + RequestEncoder get encoder => const _ProduceRequestEncoder(); +} + +class ProduceResponse { + /// List of produce results for each topic-partition. + final PartitionResults results; + final int throttleTime; + + ProduceResponse(this.results, this.throttleTime) { + var errorResult = results.partitions + .firstWhere((_) => _.error != Errors.NoError, orElse: () => null); + + if (errorResult is PartitionResult) { + throw KafkaError.fromCode(errorResult.error, this); + } + } + + @override + String toString() => 'ProduceResponse{$results}'; +} + +class PartitionResults { + final List partitions; + + PartitionResults(this.partitions); + + Map _asMap; + Map get asMap { + if (_asMap != null) return _asMap; + _asMap = Map.fromIterable(partitions, key: (result) => result.partition); + return _asMap; + } + + PartitionResult operator [](TopicPartition partition) => asMap[partition]; + + Map _offsets; + Map get offsets { + if (_offsets != null) return _offsets; + _offsets = Map.fromIterable(partitions, + key: (result) => result.partition, value: (result) => result.offset); + return _offsets; + } + + int get length => partitions.length; +} + +/// Data structure representing result of producing messages with +/// [ProduceRequest]. +class PartitionResult { + /// The topic-parition of this result. + final TopicPartition partition; + + /// Error code returned by the server. + final int error; + + /// Offset of the first message. + final int offset; + + /// The creation timestamp of the message set. + /// + /// If `LogAppendTime` is used for the topic this is the timestamp assigned + /// by the broker to the message set. All the messages in the message set + /// have the same timestamp. + /// + /// If `CreateTime` is used, this field is always -1. The producer can assume + /// the timestamp of the messages in the produce request has been accepted + /// by the broker if there is no error code returned. + final int timestamp; + + PartitionResult(this.partition, this.error, this.offset, this.timestamp); + + String get topic => partition.topic; + + @override + String toString() => + 'PartitionResult{${partition}, error: ${error}, offset: ${offset}, timestamp: $timestamp}'; +} + +class _ProduceRequestEncoder implements RequestEncoder { + const _ProduceRequestEncoder(); + + @override + List encode(ProduceRequest request, int version) { + assert( + version == 2, 'Only v2 of Produce request is supported by the client.'); + + var builder = KafkaBytesBuilder(); + builder.addInt16(request.requiredAcks); + builder.addInt32(request.timeout); + + builder.addInt32(request.messages.length); + request.messages.forEach((topic, partitions) { + builder.addString(topic); + builder.addInt32(partitions.length); + partitions.forEach((partition, messages) { + builder.addInt32(partition); + builder.addRaw(_messageSetToBytes(messages)); + }); + }); + + return builder.takeBytes(); + } + + List _messageSetToBytes(List messages) { + var builder = KafkaBytesBuilder(); + messages.asMap().forEach((offset, message) { + var messageData = _messageToBytes(message); + builder.addInt64(offset); + builder.addInt32(messageData.length); + builder.addRaw(messageData); + }); + var messageData = builder.takeBytes(); + builder.addInt32(messageData.length); + builder.addRaw(messageData); + return builder.takeBytes(); + } + + List _messageToBytes(Message message) { + var builder = KafkaBytesBuilder(); + builder.addInt8(1); // magicByte + builder.addInt8(_encodeAttributes(message.attributes)); + builder.addInt64(message.timestamp); + builder.addBytes(message.key); + builder.addBytes(message.value); + + var data = builder.takeBytes(); + int crc = Crc32.signed(data); + builder.addInt32(crc); + builder.addRaw(data); + + return builder.takeBytes(); + } + + int _encodeAttributes(MessageAttributes attributes) { + switch (attributes.compression) { + case Compression.none: + return 0; + case Compression.gzip: + return 1; + case Compression.snappy: + return 2; + default: + throw ArgumentError( + 'Invalid compression value ${attributes.compression}.'); + } + } +} + +class _ProduceResponseDecoder implements ResponseDecoder { + const _ProduceResponseDecoder(); + + @override + ProduceResponse decode(List data) { + var reader = KafkaBytesReader.fromBytes(data); + var results = List(); + var topicCount = reader.readInt32(); + while (topicCount > 0) { + var topic = reader.readString(); + var partitionCount = reader.readInt32(); + while (partitionCount > 0) { + var partition = reader.readInt32(); + var error = reader.readInt16(); + var offset = reader.readInt64(); + var timestamp = reader.readInt64(); + results.add(PartitionResult( + TopicPartition(topic, partition), error, offset, timestamp)); + partitionCount--; + } + topicCount--; + } + var throttleTime = reader.readInt32(); + return ProduceResponse(PartitionResults(results), throttleTime); + } +} diff --git a/lib/src/producer.dart b/lib/src/producer.dart index e03730c..ce73609 100644 --- a/lib/src/producer.dart +++ b/lib/src/producer.dart @@ -1,160 +1,291 @@ -part of kafka; +import 'dart:async'; -/// High-level Producer for Kafka. -/// -/// Producer encapsulates logic for broker discovery when publishing messages to -/// multiple topic-partitions. It will send as many ProduceRequests as needed -/// based on leader assignment for corresponding topic-partitions. +import 'package:logging/logging.dart'; +import 'package:pool/pool.dart'; + +import 'common.dart'; +import 'messages.dart'; +import 'produce_api.dart'; +import 'serialization.dart'; +import 'session.dart'; + +final Logger _logger = new Logger('Producer'); + +/// Produces messages to Kafka cluster. /// -/// Requests will be send in parallel and results will be aggregated in -/// [ProduceResult]. -class Producer { - /// Instance of [KafkaSession] which is used to send requests to Kafka brokers. - final KafkaSession session; +/// Automatically discovers leader brokers for each topic-partition to +/// send messages to. +abstract class Producer implements StreamSink> { + factory Producer(Serializer keySerializer, Serializer valueSerializer, + ProducerConfig config) => + new _Producer(keySerializer, valueSerializer, config); +} - /// How many acknowledgements the servers should receive before responding to the request. - /// - /// * If it is 0 the server will not send any response. - /// * If it is 1, the server will wait the data is written to the local log before sending a response. - /// * If it is -1 the server will block until the message is committed by all in sync replicas before sending a response. - /// * For any number > 1 the server will block waiting for this number of acknowledgements to occur - final int requiredAcks; +class ProducerRecord { + final String topic; + final int partition; + final K key; + final V value; + final int timestamp; - /// Maximum time in milliseconds the server can await the receipt of the - /// number of acknowledgements in [requiredAcks]. - final int timeout; + final Completer _completer = new Completer(); - /// Creates new instance of [Producer]. - /// - /// [requiredAcks] specifies how many acknowledgements the servers should - /// receive before responding to the request. - /// - /// [timeout] specifies maximum time in milliseconds the server can await - /// the receipt of the number of acknowledgements in [requiredAcks]. - Producer(this.session, this.requiredAcks, this.timeout); + ProducerRecord(this.topic, this.partition, this.key, this.value, + {this.timestamp}); - /// Sends messages to Kafka with "at least once" guarantee. - /// - /// Producer will attempt to retry requests when Kafka server returns any of - /// the retriable errors. See [ProduceResult.hasRetriableErrors] for details. - /// - /// In case of such errors producer will attempt to re-send **all the messages** - /// and this may lead to duplicate records in the stream (therefore - /// "at least once" guarantee). - /// - /// If server returns errors which can not be retried then returned future will - /// be completed with [ProduceError]. One can still access `ProduceResult` from - /// it. + TopicPartition get topicPartition => new TopicPartition(topic, partition); + + /// The result of publishing this record. /// - /// In case of any non-protocol errors returned future will complete with actual - /// error that was thrown. - Future produce(List messages) { - return _produce(messages); - } - - Future _produce(List messages, - {bool refreshMetadata: false, - int retryTimes: 3, - Duration retryInterval: const Duration(seconds: 1)}) async { - var topicNames = new Set.from(messages.map((_) => _.topicName)); - var meta = - await session.getMetadata(topicNames, invalidateCache: refreshMetadata); - - var byBroker = new ListMultimap.fromIterable( - messages, key: (ProduceEnvelope _) { - var leaderId = - meta.getTopicMetadata(_.topicName).getPartition(_.partitionId).leader; - return meta.getBroker(leaderId); - }); - kafkaLogger.fine('Producer: sending ProduceRequests'); - - Iterable futures = new List.from(byBroker.keys.map( - (broker) => session.send(broker, - new ProduceRequest(requiredAcks, timeout, byBroker[broker])))); - - var result = await Future.wait(futures).then((responses) => - new ProduceResult.fromResponses( - new List.from(responses))); - - if (!result.hasErrors) return result; - if (retryTimes <= 0) return result; - - if (result.hasRetriableErrors) { - kafkaLogger.warning( - 'Producer: server returned errors which can be retried. All returned errors are: ${result.errors}'); - kafkaLogger.info( - 'Producer: will retry after ${retryInterval.inSeconds} seconds.'); - var retriesLeft = retryTimes - 1; - var newInterval = new Duration(seconds: retryInterval.inSeconds * 2); - return new Future.delayed( - retryInterval, - () => _produce(messages, - refreshMetadata: true, - retryTimes: retriesLeft, - retryInterval: newInterval)); - } else if (result.hasErrors) { - throw new ProduceError(result); - } else { - return result; - } + /// Returned `Future` is completed with [ProduceResult] on success, otherwise + /// completed with the produce error. + Future get result => _completer.future; + + void _complete(ProduceResult result) { + _completer.complete(result); + } + + void _completeError(error) { + _completer.completeError(error); } } -/// Exception thrown in case when server returned errors in response to -/// `Producer.produce()`. -class ProduceError implements Exception { - final ProduceResult result; +class ProduceResult { + final TopicPartition topicPartition; + final int offset; + final int timestamp; - ProduceError(this.result); + ProduceResult(this.topicPartition, this.offset, this.timestamp); @override - toString() => 'ProduceError: ${result.errors}'; + toString() => + 'ProduceResult{${topicPartition}, offset: $offset, timestamp: $timestamp}'; } -/// Result of producing messages with [Producer]. -class ProduceResult { - /// List of actual ProduceResponse objects returned by the server. - final List responses; - - /// Indicates whether any of server responses contain errors. - final bool hasErrors; - - /// Collection of all unique errors returned by the server. - final Iterable errors; - - /// Offsets for latest messages for each topic-partition assigned by the server. - final Map> offsets; - - ProduceResult._(this.responses, Set errors, this.offsets) - : hasErrors = errors.isNotEmpty, - errors = new UnmodifiableListView(errors); - - factory ProduceResult.fromResponses(Iterable responses) { - var errors = new Set(); - var offsets = new Map>(); - for (var r in responses) { - var er = r.results - .where((_) => _.errorCode != KafkaServerError.NoError) - .map((result) => new KafkaServerError(result.errorCode)); - errors.addAll(new Set.from(er)); - r.results.forEach((result) { - offsets.putIfAbsent(result.topicName, () => new Map()); - offsets[result.topicName][result.partitionId] = result.offset; +class _Producer implements Producer { + final ProducerConfig config; + final Serializer keySerializer; + final Serializer valueSerializer; + final Session session; + + final StreamController> _controller = + new StreamController(); + + _Producer(this.keySerializer, this.valueSerializer, this.config) + : session = new Session(config.bootstrapServers) { + _logger.info('Producer created with config:'); + _logger.info(config); + } + + Future _closeFuture; + @override + Future close() { + if (_closeFuture != null) return _closeFuture; + + /// We first close our internal stream controller so that no new records + /// can be added. Then check if producing is still in progress and wait + /// for it to complete. And last, after producing is done we close + /// the session. + _closeFuture = _controller.close().then((_) { + return _produceFuture; + }).then((_) => session.close()); + return _closeFuture; + } + + @override + void add(ProducerRecord event) { + _subscribe(); + _controller.add(event); + } + + @override + void addError(errorEvent, [StackTrace stackTrace]) { + /// TODO: Should this throw instead to not allow errors? + /// Shouldn't really need to implement this method since stream + /// listener is internal to this class (?) + _subscribe(); + _controller.addError(errorEvent, stackTrace); + } + + @override + Future addStream(Stream> stream) { + _subscribe(); + return _controller.addStream(stream); + } + + @override + Future get done => close(); + + StreamSubscription _subscription; + void _subscribe() { + if (_subscription == null) { + _subscription = _controller.stream.listen(_onData, onDone: _onDone); + } + } + + List> _buffer = new List(); + void _onData(ProducerRecord event) { + _buffer.add(event); + _resume(); + } + + void _onDone() { + _logger.fine('Done event received'); + } + + Future _produceFuture; + void _resume() { + if (_produceFuture != null) return; + _logger.fine('New records arrived. Resuming producer.'); + _produceFuture = _produce().whenComplete(() { + _logger.fine('No more new records. Pausing producer.'); + _produceFuture = null; + }); + } + + Future _produce() async { + while (_buffer.isNotEmpty) { + var records = _buffer; + _buffer = new List(); + var leaders = await _groupByLeader(records); + var pools = new Map(); + for (var leader in leaders.keys) { + pools[leader] = new Pool(config.maxInFlightRequestsPerConnection); + var requests = _buildRequests(leaders[leader]); + for (var req in requests) { + pools[leader].withResource(() => _send(leader, req, leaders[leader])); + } + } + var futures = pools.values.map((_) => _.close()); + await Future.wait(futures); + } + } + + Future _send(Broker broker, ProduceRequest request, + List> records) { + return session.send(request, broker.host, broker.port).then((response) { + Map offsets = new Map.from(response.results.offsets); + for (var rec in records) { + var p = rec.topicPartition; + rec._complete( + new ProduceResult(p, offsets[p], response.results[p].timestamp)); + offsets[p]++; + } + }).catchError((error) { + records.forEach((_) { + _._completeError(error); }); + }); + } + + List _buildRequests(List> records) { + /// TODO: Split requests by max size. + var messages = new Map>>(); + for (var rec in records) { + var key = keySerializer.serialize(rec.key); + var value = valueSerializer.serialize(rec.value); + var timestamp = + rec.timestamp ?? new DateTime.now().millisecondsSinceEpoch; + var message = new Message(value, key: key, timestamp: timestamp); + messages.putIfAbsent(rec.topic, () => new Map()); + messages[rec.topic].putIfAbsent(rec.partition, () => new List()); + messages[rec.topic][rec.partition].add(message); } + var request = new ProduceRequest(config.acks, config.timeoutMs, messages); + return [request]; + } - return new ProduceResult._(responses, errors, offsets); + Future>>> _groupByLeader( + List> records) async { + var topics = records.map((_) => _.topic).toSet().toList(growable: false); + var metadata = await session.metadata.fetchTopics(topics); + var result = new Map>>(); + for (var rec in records) { + var leader = metadata[rec.topic].partitions[rec.partition].leader; + var broker = metadata.brokers[leader]; + result.putIfAbsent(broker, () => new List()); + result[broker].add(rec); + } + return result; } +} + +/// Configuration for [Producer]. +/// +/// The only required setting which must be set is [bootstrapServers], +/// other settings are optional and have default values. Refer +/// to settings documentation for more details. +class ProducerConfig { + /// A list of host/port pairs to use for establishing the initial + /// connection to the Kafka cluster. The client will make use of + /// all servers irrespective of which servers are specified here + /// for bootstrapping - this list only impacts the initial hosts + /// used to discover the full set of servers. The values should + /// be in the form `host:port`. + /// Since these servers are just used for the initial connection + /// to discover the full cluster membership (which may change + /// dynamically), this list need not contain the full set of + /// servers (you may want more than one, though, in case a + /// server is down). + final List bootstrapServers; + + /// The number of acknowledgments the producer requires the leader to have + /// received before considering a request complete. + /// This controls the durability of records that are sent. + final int acks; + + /// Controls the maximum amount of time the server + /// will wait for acknowledgments from followers to meet the acknowledgment + /// requirements the producer has specified with the [acks] configuration. + /// If the requested number of acknowledgments are not met when the timeout + /// elapses an error is returned by the server. This timeout is measured on the + /// server side and does not include the network latency of the request. + final int timeoutMs; + + /// Setting a value greater than zero will cause the client to resend any + /// record whose send fails with a potentially transient error. + final int retries; - /// Returns `true` if this result contains server error with specified [code]. - bool hasError(int code) => errors.contains(new KafkaServerError(code)); + /// An id string to pass to the server when making requests. + /// The purpose of this is to be able to track the source of requests + /// beyond just ip/port by allowing a logical application name to be + /// included in server-side request logging. + final String clientId; - /// Returns `true` if at least one server error in this result can be retried. - bool get hasRetriableErrors { - return hasError(KafkaServerError.LeaderNotAvailable) || - hasError(KafkaServerError.NotLeaderForPartition) || - hasError(KafkaServerError.RequestTimedOut) || - hasError(KafkaServerError.NotEnoughReplicasCode) || - hasError(KafkaServerError.NotEnoughReplicasAfterAppendCode); + /// The maximum size of a request in bytes. This is also effectively a + /// cap on the maximum record size. Note that the server has its own + /// cap on record size which may be different from this. + final int maxRequestSize; + + /// The maximum number of unacknowledged requests the client will + /// send on a single connection before blocking. Note that if this + /// setting is set to be greater than 1 and there are failed sends, + /// there is a risk of message re-ordering due to retries (i.e., + /// if retries are enabled). + final int maxInFlightRequestsPerConnection; + + ProducerConfig({ + this.bootstrapServers, + this.acks = 1, + this.timeoutMs = 30000, + this.retries = 0, + this.clientId = '', + this.maxRequestSize = 1048576, + this.maxInFlightRequestsPerConnection = 5, + }) { + assert(bootstrapServers != null); } + + @override + String toString() => ''' +ProducerConfig( + bootstrapServers: $bootstrapServers, + acks: $acks, + timeoutMs: $timeoutMs, + retries: $retries, + clientId: $clientId, + maxRequestSize: $maxRequestSize, + maxInFlightRequestsPerConnection: $maxInFlightRequestsPerConnection +) +'''; } diff --git a/lib/src/protocol/bytes_builder.dart b/lib/src/protocol/bytes_builder.dart deleted file mode 100644 index 78e4d0f..0000000 --- a/lib/src/protocol/bytes_builder.dart +++ /dev/null @@ -1,122 +0,0 @@ -part of kafka.protocol; - -enum KafkaType { int8, int16, int32, int64, string, bytes, object } - -/// Bytes builder specific to Kafka protocol. -/// -/// Provides convenient methods for writing all Kafka data types (and some more): -/// int8, int16, int32, string, bytes, array. -class KafkaBytesBuilder { - BytesBuilder _builder = new BytesBuilder(); - - int get length => _builder.length; - - /// Creates new builder with empty buffer. - KafkaBytesBuilder(); - - /// Creates new builder and initializes buffer with proper request header. - KafkaBytesBuilder.withRequestHeader( - int apiKey, int apiVersion, int correlationId) { - addInt16(apiKey); - addInt16(apiVersion); - addInt32(correlationId); - addString(dartKafkaId); - } - - /// Adds 8 bit integer to this buffer. - void addInt8(int value) { - ByteData bdata = new ByteData(1); - bdata.setInt8(0, value); - _add(bdata); - } - - /// Adds 16 bit integer to this buffer. - void addInt16(int value) { - ByteData bdata = new ByteData(2); - bdata.setInt16(0, value); - _add(bdata); - } - - /// Adds 32 bit integer to this buffer. - void addInt32(int value) { - ByteData bdata = new ByteData(4); - bdata.setInt32(0, value); - _add(bdata); - } - - /// Adds 64 bit integer to this buffer. - void addInt64(int value) { - ByteData bdata = new ByteData(8); - bdata.setInt64(0, value); - _add(bdata); - } - - /// Adds Kafka string to this bytes builder. - /// - /// Kafka string type starts with int16 indicating size of the string - /// followed by the actual string value. - void addString(String value) { - List data = UTF8.encode(value); - addInt16(data.length); - _builder.add(data); - } - - /// Adds Kafka array to this bytes builder. - /// - /// Kafka array starts with int32 indicating size of the array followed by - /// the array items encoded according to their [KafkaType] - void addArray(Iterable items, KafkaType itemType) { - addInt32(items.length); - for (var item in items) { - switch (itemType) { - case KafkaType.int8: - addInt8(item); - break; - case KafkaType.int16: - addInt16(item); - break; - case KafkaType.int32: - addInt32(item); - break; - case KafkaType.int64: - addInt64(item); - break; - case KafkaType.string: - addString(item); - break; - case KafkaType.bytes: - addBytes(new List.from(item)); - break; - case KafkaType.object: - throw new StateError('Objects are not supported yet'); - break; - } - } - } - - /// Adds value of Kafka-specific Bytes type to this builder. - /// - /// Kafka Bytes type starts with int32 indicating size of the value following - /// by actual value bytes. - void addBytes(List value) { - if (value == null) { - addInt32(-1); - } else { - addInt32(value.length); - _builder.add(value); - } - } - - /// Adds arbitrary data to this buffer. - void addRaw(List data) { - _builder.add(data); - } - - void _add(ByteData data) { - _builder.add(data.buffer.asInt8List().toList(growable: false)); - } - - List takeBytes() => _builder.takeBytes(); - - List toBytes() => _builder.toBytes(); -} diff --git a/lib/src/protocol/bytes_reader.dart b/lib/src/protocol/bytes_reader.dart deleted file mode 100644 index 68288da..0000000 --- a/lib/src/protocol/bytes_reader.dart +++ /dev/null @@ -1,124 +0,0 @@ -part of kafka.protocol; - -/// Provides convenience methods read Kafka specific data types from a stream of bytes. -class KafkaBytesReader { - Int8List _data; - int _offset = 0; - - /// Current position in this buffer. - int get offset => _offset; - - /// Size of this byte buffer. - int get length => _data.length; - - /// Whether this bytes buffer has been fully read. - bool get isEOF => _data.length == _offset; - - /// Whether there are still unread bytes left in this buffer. - bool get isNotEOF => !isEOF; - - /// Creates reader from a list of bytes. - KafkaBytesReader.fromBytes(List data) { - this._data = new Int8List.fromList(data); - } - - // Reads int8 from the data and returns it. - int readInt8() { - var data = new ByteData.view(_data.buffer, _offset, 1); - var value = data.getInt8(0); - _offset += 1; - - return value; - } - - /// Reads 16-bit integer from the current position of this buffer. - int readInt16() { - var data = new ByteData.view(_data.buffer, _offset, 2); - var value = data.getInt16(0); - _offset += 2; - - return value; - } - - /// Reads 32-bit integer from the current position of this buffer. - int readInt32() { - var data = new ByteData.view(_data.buffer, _offset, 4); - var value = data.getInt32(0); - _offset += 4; - - return value; - } - - /// Reads 64-bit integer from the current position of this buffer. - int readInt64() { - var data = new ByteData.view(_data.buffer, _offset, 8); - var value = data.getInt64(0); - _offset += 8; - - return value; - } - - String readString() { - var length = readInt16(); - var value = _data.buffer.asInt8List(_offset, length).toList(); - var valueAsString = UTF8.decode(value); - _offset += length; - - return valueAsString; - } - - List readBytes() { - var length = readInt32(); - if (length == -1) { - return null; - } else { - var value = _data.buffer.asInt8List(_offset, length).toList(); - _offset += length; - return value; - } - } - - List readArray(KafkaType itemType, - [dynamic objectReadHandler(KafkaBytesReader reader)]) { - var length = readInt32(); - var items = new List(); - for (var i = 0; i < length; i++) { - switch (itemType) { - case KafkaType.int8: - items.add(readInt8()); - break; - case KafkaType.int16: - items.add(readInt16()); - break; - case KafkaType.int32: - items.add(readInt32()); - break; - case KafkaType.int64: - items.add(readInt64()); - break; - case KafkaType.string: - items.add(readString()); - break; - case KafkaType.bytes: - items.add(readBytes()); - break; - case KafkaType.object: - if (objectReadHandler == null) { - throw new StateError('ObjectReadHandler must be provided'); - } - items.add(objectReadHandler(this)); - break; - } - } - - return items; - } - - /// Reads raw bytes from this buffer. - List readRaw(int length) { - var value = _data.buffer.asInt8List(_offset, length).toList(); - _offset += length; - - return value; - } -} diff --git a/lib/src/protocol/common.dart b/lib/src/protocol/common.dart deleted file mode 100644 index 64315fc..0000000 --- a/lib/src/protocol/common.dart +++ /dev/null @@ -1,14 +0,0 @@ -part of kafka.protocol; - -/// Base interface for all Kafka API requests. -abstract class KafkaRequest { - static final _random = new Random(); - - final int correlationId; - - KafkaRequest() : correlationId = _random.nextInt(65536); - - List toBytes(); - - dynamic createResponse(List data); -} diff --git a/lib/src/protocol/consumer_metadata_api.dart b/lib/src/protocol/consumer_metadata_api.dart deleted file mode 100644 index df40599..0000000 --- a/lib/src/protocol/consumer_metadata_api.dart +++ /dev/null @@ -1,60 +0,0 @@ -part of kafka.protocol; - -/// Kafka ConsumerMetadataRequest. -class GroupCoordinatorRequest extends KafkaRequest { - final int apiKey = 10; - final int apiVersion = 0; - final String consumerGroup; - - /// Creates new instance of ConsumerMetadataRequest. - GroupCoordinatorRequest(this.consumerGroup) : super(); - - /// Converts this request into byte list - @override - List toBytes() { - var builder = new KafkaBytesBuilder.withRequestHeader( - apiKey, apiVersion, correlationId); - - builder.addString(consumerGroup); - - var body = builder.takeBytes(); - builder.addBytes(body); - - return builder.takeBytes(); - } - - @override - createResponse(List data) { - return new GroupCoordinatorResponse.fromBytes(data); - } -} - -/// Response for [GroupCoordinatorRequest]. -class GroupCoordinatorResponse { - final int errorCode; - final int coordinatorId; - final String coordinatorHost; - final int coordinatorPort; - - Broker get coordinator => - new Broker(coordinatorId, coordinatorHost, coordinatorPort); - - /// Creates new instance of ConsumerMetadataResponse. - GroupCoordinatorResponse(this.errorCode, this.coordinatorId, - this.coordinatorHost, this.coordinatorPort); - - /// Creates response from provided data. - factory GroupCoordinatorResponse.fromBytes(List data) { - var reader = new KafkaBytesReader.fromBytes(data); - var size = reader.readInt32(); - assert(size == data.length - 4); - - reader.readInt32(); // correlationId - var errorCode = reader.readInt16(); - var id = reader.readInt32(); - var host = reader.readString(); - var port = reader.readInt32(); - - return new GroupCoordinatorResponse(errorCode, id, host, port); - } -} diff --git a/lib/src/protocol/fetch_api.dart b/lib/src/protocol/fetch_api.dart deleted file mode 100644 index 739dbbd..0000000 --- a/lib/src/protocol/fetch_api.dart +++ /dev/null @@ -1,140 +0,0 @@ -part of kafka.protocol; - -/// Kafka FetchRequest. -class FetchRequest extends KafkaRequest { - /// API key of [FetchRequest] - final int apiKey = 1; - - /// API version of [FetchRequest] - final int apiVersion = 0; - - /// The replica id indicates the node id of the replica initiating this request. - /// Normal consumers should always specify this as -1 as they have no node id. - final int _replicaId = -1; - - /// Maximum amount of time in milliseconds to block waiting if insufficient - /// data is available at the time the request is issued. - final int maxWaitTime; - - /// Minimum number of bytes of messages that must be available - /// to give a response. - final int minBytes; - - Map> _topics = new Map(); - - /// Creates new instance of FetchRequest. - FetchRequest(this.maxWaitTime, this.minBytes) : super(); - - @override - toString() => 'FetchRequest(${maxWaitTime}, ${minBytes}, ${_topics})'; - - /// Adds [topicName] with [paritionId] to this FetchRequest. [fetchOffset] - /// defines the offset to begin this fetch from. - void add(String topicName, int partitionId, int fetchOffset, - [int maxBytes = 65536]) { - // - if (!_topics.containsKey(topicName)) { - _topics[topicName] = new List(); - } - _topics[topicName] - .add(new _FetchPartitionInfo(partitionId, fetchOffset, maxBytes)); - } - - @override - List toBytes() { - var builder = new KafkaBytesBuilder.withRequestHeader( - apiKey, apiVersion, correlationId); - - builder.addInt32(_replicaId); - builder.addInt32(maxWaitTime); - builder.addInt32(minBytes); - - builder.addInt32(_topics.length); - _topics.forEach((topicName, partitions) { - builder.addString(topicName); - builder.addInt32(partitions.length); - partitions.forEach((p) { - builder.addInt32(p.partitionId); - builder.addInt64(p.fetchOffset); - builder.addInt32(p.maxBytes); - }); - }); - - var body = builder.takeBytes(); - builder.addBytes(body); - - return builder.takeBytes(); - } - - @override - createResponse(List data) { - return new FetchResponse.fromBytes(data); - } -} - -class _FetchPartitionInfo { - int partitionId; - int fetchOffset; - int maxBytes; - _FetchPartitionInfo(this.partitionId, this.fetchOffset, this.maxBytes); -} - -/// Kafka FetchResponse. -class FetchResponse { - /// List of [FetchResult]s for each topic-partition. - final List results; - - /// Indicates if server returned any errors in this response. - /// - /// Actual errors can be found in the result object for particular - /// topic-partition. - final bool hasErrors; - - FetchResponse._(this.results, this.hasErrors); - - /// Creates new instance of FetchResponse from binary data. - factory FetchResponse.fromBytes(List data) { - var reader = new KafkaBytesReader.fromBytes(data); - var size = reader.readInt32(); - assert(size == data.length - 4); - - reader.readInt32(); // correlationId - var count = reader.readInt32(); - var results = new List(); - var hasErrors = false; - while (count > 0) { - var topicName = reader.readString(); - var partitionCount = reader.readInt32(); - while (partitionCount > 0) { - var partitionId = reader.readInt32(); - var errorCode = reader.readInt16(); - var highwaterMarkOffset = reader.readInt64(); - var messageSetSize = reader.readInt32(); - var data = reader.readRaw(messageSetSize); - var messageReader = new KafkaBytesReader.fromBytes(data); - var messageSet = new MessageSet.fromBytes(messageReader); - if (errorCode != KafkaServerError.NoError) hasErrors = true; - - results.add(new FetchResult(topicName, partitionId, errorCode, - highwaterMarkOffset, messageSet)); - partitionCount--; - } - count--; - } - - return new FetchResponse._(results, hasErrors); - } -} - -/// Data structure representing result of fetching messages for particular -/// topic-partition. -class FetchResult { - final String topicName; - final int partitionId; - final int errorCode; - final int highwaterMarkOffset; - final MessageSet messageSet; - - FetchResult(this.topicName, this.partitionId, this.errorCode, - this.highwaterMarkOffset, this.messageSet); -} diff --git a/lib/src/protocol/group_membership_api.dart b/lib/src/protocol/group_membership_api.dart deleted file mode 100644 index 55c6cae..0000000 --- a/lib/src/protocol/group_membership_api.dart +++ /dev/null @@ -1 +0,0 @@ -part of kafka.protocol; diff --git a/lib/src/protocol/messages.dart b/lib/src/protocol/messages.dart deleted file mode 100644 index 5da6f5c..0000000 --- a/lib/src/protocol/messages.dart +++ /dev/null @@ -1,121 +0,0 @@ -part of kafka.protocol; - -/// Kafka MessageSet type. -class MessageSet { - /// Collection of messages. Keys in the map are message offsets. - final Map _messages; - - /// Map of message offsets to corresponding messages. - Map get messages => new UnmodifiableMapView(_messages); - - /// Number of messages in this set. - int get length => _messages.length; - - MessageSet._(this._messages); - - /// Builds new message set for publishing. - factory MessageSet.build(ProduceEnvelope envelope) { - if (envelope.compression == KafkaCompression.none) { - return new MessageSet._(envelope.messages.asMap()); - } else { - if (envelope.compression == KafkaCompression.snappy) - throw new ArgumentError( - 'Snappy compression is not supported yet by the client.'); - - var codec = new GZipCodec(); - var innerEnvelope = new ProduceEnvelope( - envelope.topicName, envelope.partitionId, envelope.messages); - var innerMessageSet = new MessageSet.build(innerEnvelope); - var value = codec.encode(innerMessageSet.toBytes()); - var attrs = new MessageAttributes(KafkaCompression.gzip); - - return new MessageSet._({0: new Message(value, attributes: attrs)}); - } - } - - /// Creates new MessageSet from provided data. - factory MessageSet.fromBytes(KafkaBytesReader reader) { - int messageSize = -1; - var messages = new Map(); - while (reader.isNotEOF) { - try { - int offset = reader.readInt64(); - messageSize = reader.readInt32(); - var crc = reader.readInt32(); - - var data = reader.readRaw(messageSize - 4); - var actualCrc = Crc32.signed(data); - if (actualCrc != crc) { - kafkaLogger?.warning( - 'Message CRC sum mismatch. Expected crc: ${crc}, actual: ${actualCrc}'); - throw new MessageCrcMismatchError( - 'Expected crc: ${crc}, actual: ${actualCrc}'); - } - var messageReader = new KafkaBytesReader.fromBytes(data); - var message = _readMessage(messageReader); - if (message.attributes.compression == KafkaCompression.none) { - messages[offset] = message; - } else { - if (message.attributes.compression == KafkaCompression.snappy) - throw new ArgumentError( - 'Snappy compression is not supported yet by the client.'); - - var codec = new GZipCodec(); - var innerReader = - new KafkaBytesReader.fromBytes(codec.decode(message.value)); - var innerMessageSet = new MessageSet.fromBytes(innerReader); - for (var innerOffset in innerMessageSet.messages.keys) { - messages[innerOffset] = innerMessageSet.messages[innerOffset]; - } - } - } on RangeError { - // According to spec server is allowed to return partial - // messages, so we just ignore it here and exit the loop. - var remaining = reader.length - reader.offset; - kafkaLogger?.info( - 'Encountered partial message. Expected message size: ${messageSize}, bytes left in buffer: ${remaining}, total buffer size ${reader.length}'); - break; - } - } - - return new MessageSet._(messages); - } - - static Message _readMessage(KafkaBytesReader reader) { - reader.readInt8(); // magicByte - var attributes = new MessageAttributes.fromByte(reader.readInt8()); - var key = reader.readBytes(); - var value = reader.readBytes(); - - return new Message(value, attributes: attributes, key: key); - } - - /// Converts this MessageSet into sequence of bytes according to Kafka - /// protocol. - List toBytes() { - var builder = new KafkaBytesBuilder(); - _messages.forEach((offset, message) { - var messageData = _messageToBytes(message); - builder.addInt64(offset); - builder.addInt32(messageData.length); - builder.addRaw(messageData); - }); - - return builder.toBytes(); - } - - List _messageToBytes(Message message) { - var builder = new KafkaBytesBuilder(); - builder.addInt8(0); // magicByte - builder.addInt8(message.attributes.toInt()); - builder.addBytes(message.key); - builder.addBytes(message.value); - - var data = builder.takeBytes(); - int crc = Crc32.signed(data); - builder.addInt32(crc); - builder.addRaw(data); - - return builder.toBytes(); - } -} diff --git a/lib/src/protocol/metadata_api.dart b/lib/src/protocol/metadata_api.dart deleted file mode 100644 index 9318586..0000000 --- a/lib/src/protocol/metadata_api.dart +++ /dev/null @@ -1,120 +0,0 @@ -part of kafka.protocol; - -/// Kafka MetadataRequest. -class MetadataRequest extends KafkaRequest { - /// API key of [MetadataRequest] - final int apiKey = 3; - - /// API version of [MetadataRequest] - final int apiVersion = 0; - - /// List of topic names to fetch metadata for. If set to null or empty - /// this request will fetch metadata for all topics. - final Set topicNames; - - /// Creats new instance of Kafka MetadataRequest. - /// - /// If [topicNames] is omitted or empty then metadata for all existing topics - /// will be returned. - MetadataRequest([this.topicNames]) : super(); - - @override - List toBytes() { - var builder = new KafkaBytesBuilder.withRequestHeader( - apiKey, apiVersion, correlationId); - Set list = (this.topicNames is Set) ? this.topicNames : new Set(); - builder.addArray(list, KafkaType.string); - - var body = builder.takeBytes(); - builder.addBytes(body); - - return builder.takeBytes(); - } - - @override - createResponse(List data) { - return new MetadataResponse.fromBytes(data); - } -} - -/// Kafka MetadataResponse. -class MetadataResponse { - /// List of brokers in the cluster. - final List brokers; - - /// List with metadata for each topic. - final List topics; - - MetadataResponse._(this.brokers, this.topics); - - /// Creates response from binary data. - factory MetadataResponse.fromBytes(List data) { - var reader = new KafkaBytesReader.fromBytes(data); - var size = reader.readInt32(); - assert(size == data.length - 4); - - reader.readInt32(); // correlationId - - var brokers = reader.readArray(KafkaType.object, (reader) { - return new Broker( - reader.readInt32(), reader.readString(), reader.readInt32()); - }); - - var topicMetadata = reader.readArray( - KafkaType.object, (reader) => new TopicMetadata._readFrom(reader)); - return new MetadataResponse._(new List.from(brokers), - new List.from(topicMetadata)); - } -} - -/// Represents Kafka TopicMetadata data structure returned in MetadataResponse. -class TopicMetadata { - final int errorCode; - final String topicName; - final List partitions; - - TopicMetadata._(this.errorCode, this.topicName, this.partitions); - - factory TopicMetadata._readFrom(KafkaBytesReader reader) { - var errorCode = reader.readInt16(); - var topicName = reader.readString(); - List partitions = reader.readArray( - KafkaType.object, (reader) => new PartitionMetadata._readFrom(reader)); - // ignore: STRONG_MODE_DOWN_CAST_COMPOSITE - return new TopicMetadata._(errorCode, topicName, partitions); - } - - PartitionMetadata getPartition(int partitionId) => - partitions.firstWhere((p) => p.partitionId == partitionId); - - @override - String toString() => - "TopicMetadata(errorCode: ${errorCode}, name: ${topicName}, partitions: ${partitions.length})"; -} - -/// Data structure representing partition metadata returned in MetadataResponse. -class PartitionMetadata { - final int partitionErrorCode; - final int partitionId; - final int leader; - final List replicas; - final List inSyncReplicas; - - PartitionMetadata._(this.partitionErrorCode, this.partitionId, this.leader, - this.replicas, this.inSyncReplicas); - - factory PartitionMetadata._readFrom(KafkaBytesReader reader) { - var errorCode = reader.readInt16(); - var partitionId = reader.readInt32(); - var leader = reader.readInt32(); - var replicas = reader.readArray(KafkaType.int32); - var inSyncReplicas = reader.readArray(KafkaType.int32); - - return new PartitionMetadata._( - errorCode, - partitionId, - leader, - replicas, // ignore: STRONG_MODE_DOWN_CAST_COMPOSITE - inSyncReplicas); - } -} diff --git a/lib/src/protocol/offset_api.dart b/lib/src/protocol/offset_api.dart deleted file mode 100644 index 902c0f9..0000000 --- a/lib/src/protocol/offset_api.dart +++ /dev/null @@ -1,134 +0,0 @@ -part of kafka.protocol; - -/// Kafka OffsetRequest. -class OffsetRequest extends KafkaRequest { - /// API key of [OffsetRequest]. - final int apiKey = 2; - - /// API version of [OffsetRequest]. - final int apiVersion = 0; - - /// Unique ID assigned to the [host] within Kafka cluster. - final int replicaId; - - Map> _topics = new Map(); - - /// Creates new instance of OffsetRequest. - /// - /// The [replicaId] argument indicates unique ID assigned to the [host] within - /// Kafka cluster. One can obtain this information via [MetadataRequest]. - OffsetRequest(this.replicaId) : super(); - - /// Adds topic and partition to this requests. - /// - /// [time] is used to ask for all messages before a certain time (ms). - /// There are two special values: - /// * Specify -1 to receive the latest offset (that is the offset of the next coming message). - /// * Specify -2 to receive the earliest available offset. - /// - /// [maxNumberOfOffsets] indicates max number of offsets to return. - void addTopicPartition( - String topicName, int partitionId, int time, int maxNumberOfOffsets) { - if (_topics.containsKey(topicName) == false) { - _topics[topicName] = new List(); - } - - _topics[topicName].add( - new _PartitionOffsetRequestInfo(partitionId, time, maxNumberOfOffsets)); - } - - /// Converts this request into a binary representation according to Kafka - /// protocol. - @override - List toBytes() { - var builder = new KafkaBytesBuilder.withRequestHeader( - apiKey, apiVersion, correlationId); - builder.addInt32(replicaId); - - builder.addInt32(_topics.length); - _topics.forEach((topicName, partitions) { - builder.addString(topicName); - builder.addInt32(partitions.length); - partitions.forEach((p) { - builder.addInt32(p.partitionId); - builder.addInt64(p.time); - builder.addInt32(p.maxNumberOfOffsets); - }); - }); - - var body = builder.takeBytes(); - builder.addBytes(body); - - return builder.takeBytes(); - } - - @override - createResponse(List data) { - return new OffsetResponse.fromBytes(data); - } -} - -/// Value object holding information about partition offsets to be fetched -/// by [OffsetRequest]. -class _PartitionOffsetRequestInfo { - /// The ID of this partition. - final int partitionId; - - /// Used to ask for all messages before a certain time (ms). - /// - /// There are two special values: - /// * Specify -1 to receive the latest offset (that is the offset of the next coming message). - /// * Specify -2 to receive the earliest available offset. - final int time; - - /// How many offsets to return. - final int maxNumberOfOffsets; - _PartitionOffsetRequestInfo( - this.partitionId, this.time, this.maxNumberOfOffsets); -} - -/// Kafka OffsetResponse. -class OffsetResponse { - /// Map of topics and list of partitions with offset details. - final List offsets; - - OffsetResponse._(this.offsets); - - /// Creates OffsetResponse from the provided binary data. - factory OffsetResponse.fromBytes(List data) { - var reader = new KafkaBytesReader.fromBytes(data); - var size = reader.readInt32(); - assert(size == data.length - 4); - - reader.readInt32(); // correlationId - var count = reader.readInt32(); - var offsets = new List(); - while (count > 0) { - var topicName = reader.readString(); - var partitionCount = reader.readInt32(); - while (partitionCount > 0) { - var partitionId = reader.readInt32(); - var errorCode = reader.readInt16(); - var partitionOffsets = reader.readArray(KafkaType.int64); - offsets.add(new TopicOffsets._(topicName, partitionId, errorCode, - partitionOffsets)); // ignore: STRONG_MODE_DOWN_CAST_COMPOSITE - partitionCount--; - } - count--; - } - - return new OffsetResponse._(offsets); - } -} - -/// Data structure representing offsets of particular topic-partition returned -/// by [OffsetRequest]. -class TopicOffsets { - final String topicName; - final int partitionId; - final int errorCode; - final List offsets; - - TopicOffsets._( - this.topicName, this.partitionId, this.errorCode, this.offsets); -} diff --git a/lib/src/protocol/offset_commit_api.dart b/lib/src/protocol/offset_commit_api.dart deleted file mode 100644 index 1b23c8c..0000000 --- a/lib/src/protocol/offset_commit_api.dart +++ /dev/null @@ -1,107 +0,0 @@ -part of kafka.protocol; - -/// Kafka OffsetCommitRequest. -class OffsetCommitRequest extends KafkaRequest { - /// API key of [OffsetCommitRequest]. - final int apiKey = 8; - - /// API version of [OffsetCommitRequest]. - final int apiVersion = 1; - - /// Name of the consumer group. - final String consumerGroup; - - /// Generation ID of the consumer group. - final int consumerGroupGenerationId; - - /// ID of the consumer. - final String consumerId; - - /// Time period in msec to retain the offset. - final int retentionTime; - - /// List of consumer offsets to be committed. - final List offsets; - - /// Creates new instance of [OffsetCommitRequest]. - /// - /// [host] must be current coordinator broker for [consumerGroup]. - OffsetCommitRequest(this.consumerGroup, this.offsets, - this.consumerGroupGenerationId, this.consumerId, this.retentionTime) - : super(); - - @override - List toBytes() { - var builder = new KafkaBytesBuilder.withRequestHeader( - apiKey, apiVersion, correlationId); - - // TODO: replace groupBy with ListMultimap - // ignore: STRONG_MODE_DOWN_CAST_COMPOSITE - Map> groupedByTopic = groupBy( - offsets, (o) => o.topicName); // ignore: STRONG_MODE_DOWN_CAST_COMPOSITE - var timestamp = new DateTime.now().millisecondsSinceEpoch; - builder.addString(consumerGroup); - builder.addInt32(consumerGroupGenerationId); - builder.addString(consumerId); - builder.addInt32(groupedByTopic.length); - groupedByTopic.forEach((topicName, partitionOffsets) { - builder.addString(topicName); - builder.addInt32(partitionOffsets.length); - partitionOffsets.forEach((p) { - builder.addInt32(p.partitionId); - builder.addInt64(p.offset); - builder.addInt64(timestamp); - builder.addString(p.metadata); - }); - }); - - var body = builder.takeBytes(); - builder.addBytes(body); - - return builder.takeBytes(); - } - - @override - createResponse(List data) { - return new OffsetCommitResponse.fromData(data); - } -} - -/// Kafka OffsetCommitResponse. -class OffsetCommitResponse { - final List offsets; - - OffsetCommitResponse._(this.offsets); - - factory OffsetCommitResponse.fromData(List data) { - List offsets = []; - var reader = new KafkaBytesReader.fromBytes(data); - var size = reader.readInt32(); - assert(size == data.length - 4); - - reader.readInt32(); // correlationId - var count = reader.readInt32(); - while (count > 0) { - var topicName = reader.readString(); - var partitionCount = reader.readInt32(); - while (partitionCount > 0) { - var partitionId = reader.readInt32(); - var errorCode = reader.readInt16(); - offsets.add(new OffsetCommitResult(topicName, partitionId, errorCode)); - partitionCount--; - } - count--; - } - - return new OffsetCommitResponse._(offsets); - } -} - -/// Data structure representing result of commiting of consumer offset. -class OffsetCommitResult { - final String topicName; - final int partitionId; - final int errorCode; - - OffsetCommitResult(this.topicName, this.partitionId, this.errorCode); -} diff --git a/lib/src/protocol/offset_fetch_api.dart b/lib/src/protocol/offset_fetch_api.dart deleted file mode 100644 index 208ff45..0000000 --- a/lib/src/protocol/offset_fetch_api.dart +++ /dev/null @@ -1,79 +0,0 @@ -part of kafka.protocol; - -/// Kafka OffsetFetchRequest. -class OffsetFetchRequest extends KafkaRequest { - /// API key of [OffsetFetchRequest] - final int apiKey = 9; - - /// API version of [OffsetFetchRequest]. - final int apiVersion = 1; - - /// Name of consumer group. - final String consumerGroup; - - /// Map of topic names and partition IDs. - final Map> topics; - - /// Creates new instance of [OffsetFetchRequest]. - OffsetFetchRequest(this.consumerGroup, this.topics) : super(); - - @override - List toBytes() { - var builder = new KafkaBytesBuilder.withRequestHeader( - apiKey, apiVersion, correlationId); - - builder.addString(consumerGroup); - builder.addInt32(topics.length); - topics.forEach((topicName, partitions) { - builder.addString(topicName); - builder.addArray(partitions, KafkaType.int32); - }); - - var body = builder.takeBytes(); - builder.addBytes(body); - - return builder.takeBytes(); - } - - @override - createResponse(List data) { - return new OffsetFetchResponse.fromData(data); - } -} - -/// Kafka OffsetFetchResponse. -class OffsetFetchResponse { - final List offsets; - - OffsetFetchResponse._(this.offsets); - - factory OffsetFetchResponse.fromOffsets(List offsets) { - return new OffsetFetchResponse._(new List.from(offsets)); - } - - factory OffsetFetchResponse.fromData(List data) { - List offsets = []; - var reader = new KafkaBytesReader.fromBytes(data); - var size = reader.readInt32(); - assert(size == data.length - 4); - - reader.readInt32(); // correlationId - var count = reader.readInt32(); - while (count > 0) { - var topicName = reader.readString(); - var partitionCount = reader.readInt32(); - while (partitionCount > 0) { - var id = reader.readInt32(); - var offset = reader.readInt64(); - var metadata = reader.readString(); - var errorCode = reader.readInt16(); - offsets.add( - new ConsumerOffset(topicName, id, offset, metadata, errorCode)); - partitionCount--; - } - count--; - } - - return new OffsetFetchResponse._(offsets); - } -} diff --git a/lib/src/protocol/produce_api.dart b/lib/src/protocol/produce_api.dart deleted file mode 100644 index 832e768..0000000 --- a/lib/src/protocol/produce_api.dart +++ /dev/null @@ -1,124 +0,0 @@ -part of kafka.protocol; - -/// Kafka ProduceRequest. -class ProduceRequest extends KafkaRequest { - /// API key of [ProduceRequest] - final int apiKey = 0; - - /// API version of [ProduceRequest] - final int apiVersion = 0; - - /// Indicates how many acknowledgements the servers - /// should receive before responding to the request. - final int requiredAcks; - - /// Provides a maximum time in milliseconds the server - /// can await the receipt of the number of acknowledgements in [requiredAcks]. - final int timeout; - - /// List of produce envelopes containing messages to be published. - final List messages; - - /// Creates Kafka [ProduceRequest]. - /// - /// The [requiredAcks] field indicates how many acknowledgements the servers - /// should receive before responding to the request. - /// The [timeout] field provides a maximum time in milliseconds the server - /// can await the receipt of the number of acknowledgements in [requiredAcks]. - ProduceRequest(this.requiredAcks, this.timeout, this.messages) : super(); - - @override - List toBytes() { - var builder = new KafkaBytesBuilder.withRequestHeader( - apiKey, apiVersion, correlationId); - builder.addInt16(requiredAcks); - builder.addInt32(timeout); - - Map> messageSets = new Map(); - for (var envelope in messages) { - if (!messageSets.containsKey(envelope.topicName)) { - messageSets[envelope.topicName] = new Map(); - } - messageSets[envelope.topicName][envelope.partitionId] = - new MessageSet.build(envelope); - } - - builder.addInt32(messageSets.length); - messageSets.forEach((topicName, partitions) { - builder.addString(topicName); - builder.addInt32(partitions.length); - partitions.forEach((partitionId, messageSet) { - builder.addInt32(partitionId); - var messageData = messageSet.toBytes(); - builder.addInt32(messageData.length); - builder.addRaw(messageData); - }); - }); - - var body = builder.takeBytes(); - builder.addBytes(body); - - return builder.takeBytes(); - } - - @override - createResponse(List data) { - return new ProduceResponse.fromBytes(data); - } -} - -/// Kafka ProduceResponse. -class ProduceResponse { - /// List of produce results for each topic-partition. - final List results; - - ProduceResponse._(this.results); - - /// Creates response from the provided bytes [data]. - factory ProduceResponse.fromBytes(List data) { - var reader = new KafkaBytesReader.fromBytes(data); - var size = reader.readInt32(); - assert(size == data.length - 4); - - reader.readInt32(); // correlationId - var results = new List(); - var topicCount = reader.readInt32(); - while (topicCount > 0) { - var topicName = reader.readString(); - var partitionCount = reader.readInt32(); - while (partitionCount > 0) { - var partitionId = reader.readInt32(); - var errorCode = reader.readInt16(); - var offset = reader.readInt64(); - results.add(new TopicProduceResult._( - topicName, partitionId, errorCode, offset)); - partitionCount--; - } - topicCount--; - } - return new ProduceResponse._(results); - } -} - -/// Data structure representing result of producing messages with -/// [ProduceRequest]. -class TopicProduceResult { - /// Name of the topic. - final String topicName; - - /// ID of the partition. - final int partitionId; - - /// Error code returned by the server. - final int errorCode; - - /// Offset of the first message. - final int offset; - - TopicProduceResult._( - this.topicName, this.partitionId, this.errorCode, this.offset); - - @override - String toString() => - 'Topic: ${topicName}, partition: ${partitionId}, errorCode: ${errorCode}, offset: ${offset}'; -} diff --git a/lib/src/serialization.dart b/lib/src/serialization.dart new file mode 100644 index 0000000..36aa7c2 --- /dev/null +++ b/lib/src/serialization.dart @@ -0,0 +1,32 @@ +import 'dart:convert'; + +abstract class Serializer { + List serialize(T data); +} + +/// Serializer for `String` objects. Defaults to UTF8 encoding. +class StringSerializer implements Serializer { + @override + List serialize(String data) => utf8.encode(data); +} + +class CodecSerializer implements Serializer { + final Codec> codec; + + CodecSerializer(this.codec); + + @override + List serialize(S data) { + return codec.encode(data); + } +} + +abstract class Deserializer { + T deserialize(List data); +} + +/// Deserializer for `String` objects. Defaults to UTF8 encoding. +class StringDeserializer implements Deserializer { + @override + String deserialize(List data) => utf8.decode(data); +} diff --git a/lib/src/session.dart b/lib/src/session.dart index 0739e74..b8ea14d 100644 --- a/lib/src/session.dart +++ b/lib/src/session.dart @@ -1,292 +1,114 @@ -part of kafka; +import 'dart:async'; -/// Initial contact point with a Kafka cluster. -class ContactPoint { - final String host; - final int port; +import 'package:logging/logging.dart'; - ContactPoint(this.host, this.port); -} - -/// Session responsible for communication with Kafka cluster. -/// -/// In order to create new Session you need to pass a list of [ContactPoint]s to -/// the constructor. Each ContactPoint is defined by a host and a port of one -/// of the Kafka brokers. At least one ContactPoint is required to connect to -/// the cluster, all the rest members of the cluster will be automatically -/// detected by the Session. -/// -/// For production deployments it is recommended to provide more than one -/// ContactPoint since this will enable "failover" in case one of the instances -/// is temporarily unavailable. -class KafkaSession { - /// List of Kafka brokers which are used as initial contact points. - final Queue contactPoints; +import 'io.dart'; +import 'metadata.dart'; +import 'versions_api.dart'; +import 'versions.dart'; - Map> _sockets = new Map(); - Map _subscriptions = new Map(); - Map> _buffers = new Map(); - Map _sizes = new Map(); - Map _inflightRequests = new Map(); - Map _flushFutures = new Map(); +final Logger _logger = new Logger('Session'); - // Cluster Metadata - Future> _brokers; - Map> _topicsMetadata = new Map(); - - /// Creates new session. +/// Session responsible for handling connections to all brokers in a Kafka +/// cluster. +/// +/// Handles resolution of supported API versions by the server and the client. +/// The latest API version supported by both sides is used. +abstract class Session { + /// Creates new connection session to a Kafka cluster. /// - /// [contactPoints] will be used to fetch Kafka metadata information. At least - /// one is required. However for production consider having more than 1. - /// In case of one of the hosts is temporarily unavailable the session will - /// rotate them until sucessful response is returned. Error will be thrown - /// when all of the default hosts are unavailable. - KafkaSession(List contactPoints) - : contactPoints = new Queue.from(contactPoints); - - /// Returns names of all existing topics in the Kafka cluster. - Future> listTopics() async { - // TODO: actually rotate default hosts on failure. - var contactPoint = _getCurrentContactPoint(); - var request = new MetadataRequest(); - MetadataResponse response = - await _send(contactPoint.host, contactPoint.port, request); - - return response.topics.map((_) => _.topicName).toSet(); + /// The list of [bootstrapServers] is used initially to establish connection + /// to at least one node in the cluster. Full list of nodes is discovered + /// from metadata information fetched from the bootstrap node. + factory Session(List bootstrapServers) { + return new _SessionImpl(bootstrapServers); } - /// Fetches Kafka cluster metadata. If [topicNames] is null then metadata for - /// all topics will be returned. - /// - /// Please note that requests to fetch __all__ topics can not be cached by - /// the client, so it may not be as performant as requesting topics - /// explicitely. - /// - /// Also, if Kafka server is configured to auto-create topics you must - /// explicitely specify topic name in metadata request, otherwise topic - /// will not be created. - Future getMetadata(Set topicNames, - {bool invalidateCache: false}) async { - if (topicNames.isEmpty) - throw new ArgumentError.value( - topicNames, 'topicNames', 'List of topic names can not be empty'); - - if (invalidateCache) { - _brokers = null; - _topicsMetadata = new Map(); - } - // TODO: actually rotate default hosts on failure. - var contactPoint = _getCurrentContactPoint(); + /// Provides access to metadata about the Kafka cluster this session is + /// connected to. + Metadata get metadata; - var topicsToFetch = - topicNames.where((t) => !_topicsMetadata.keys.contains(t)); - if (topicsToFetch.length > 0) { - Future responseFuture = _sendMetadataRequest( - topicsToFetch.toSet(), contactPoint.host, contactPoint.port); - for (var name in topicsToFetch) { - _topicsMetadata[name] = responseFuture.then((response) { - return response.topics.firstWhere((_) => _.topicName == name); - }); - } + /// Sends [request] to a broker specified by [host] and [port]. + Future send(KRequest request, String host, int port); - _brokers = responseFuture.then((response) => response.brokers); - } - List allMetadata = await Future.wait(_topicsMetadata.values); - var metadata = allMetadata.where((_) => topicNames.contains(_.topicName)); - var brokers = await _brokers; - - return new ClusterMetadata(brokers, new List.unmodifiable(metadata)); - } - - Future _sendMetadataRequest( - Set topics, String host, int port) async { - var request = new MetadataRequest(topics); - MetadataResponse response = await _send(host, port, request); - - var topicWithError = response.topics.firstWhere( - (_) => _.errorCode != KafkaServerError.NoError, - orElse: () => null); - - if (topicWithError is TopicMetadata) { - var retries = 1; - var error = new KafkaServerError(topicWithError.errorCode); - while (error.isLeaderNotAvailable && retries < 5) { - var future = new Future.delayed( - new Duration(seconds: retries), () => _send(host, port, request)); - - response = await future; - topicWithError = response.topics.firstWhere( - (_) => _.errorCode != KafkaServerError.NoError, - orElse: () => null); - var errorCode = - (topicWithError is TopicMetadata) ? topicWithError.errorCode : 0; - error = new KafkaServerError(errorCode); - retries++; - } - - if (error.isError) throw error; - } - - return response; - } - - /// Fetches metadata for specified [consumerGroup]. + /// Closes all open connections to Kafka brokers. /// - /// It handles `ConsumerCoordinatorNotAvailableCode(15)` API error which Kafka - /// returns in case [GroupCoordinatorRequest] is sent for the very first time - /// to this particular broker (when special topic to store consumer offsets - /// does not exist yet). - /// - /// It will attempt up to 5 retries (with linear delay) in order to fetch - /// metadata. - Future getConsumerMetadata( - String consumerGroup) async { - // TODO: rotate default hosts. - var contactPoint = _getCurrentContactPoint(); - var request = new GroupCoordinatorRequest(consumerGroup); - - GroupCoordinatorResponse response = - await _send(contactPoint.host, contactPoint.port, request); - var retries = 1; - var error = new KafkaServerError(response.errorCode); - while (error.isConsumerCoordinatorNotAvailable && retries < 5) { - var future = new Future.delayed(new Duration(seconds: retries), - () => _send(contactPoint.host, contactPoint.port, request)); - - response = await future; - error = new KafkaServerError(response.errorCode); - retries++; - } - - if (error.isError) throw error; - - return response; - } - - /// Sends request to specified [Broker]. - Future send(Broker broker, KafkaRequest request) { - return _send(broker.host, broker.port, request); - } + /// Waits for any in-flight requests to complete before closing connections. + /// After calling `close()` no new requests are accepted by this session. + Future close(); +} - Future _send(String host, int port, KafkaRequest request) async { - kafkaLogger.finer('Session: Sending request ${request} to ${host}:${port}'); - var socket = await _getSocket(host, port); - Completer completer = new Completer(); - _inflightRequests[request] = completer; +class _SessionImpl implements Session { + final Map> _sockets = new Map(); + Metadata _metadata; + Metadata get metadata => _metadata; - /// Writing to socket is synchronous, so we need to remember future - /// returned by last call to `flush` and only write this request after - /// previous one has been flushed. - var flushFuture = _flushFutures[socket]; - _flushFutures[socket] = flushFuture.then((_) { - socket.add(request.toBytes()); - return socket.flush().catchError((error) { - _inflightRequests.remove(request); - completer.completeError(error); - return new Future.value(); - }); - }); + /// Resolved API versions supported by both server and client. + Map _apiVersions; + Completer _apiResolution; - return completer.future; + _SessionImpl(List bootstrapServers) { + _metadata = new Metadata(bootstrapServers, this); } - /// Closes this session and terminates all open socket connections. - /// - /// After session has been closed it can't be used or re-opened. - Future close() async { - for (var h in _sockets.keys) { - await _subscriptions[h].cancel(); - (await _sockets[h]).destroy(); - } - _sockets.clear(); - } - - void _handleData(String hostPort, List d) { - var buffer = _buffers[hostPort]; - - buffer.addAll(d); - if (buffer.length >= 4 && _sizes[hostPort] == -1) { - var sizeBytes = buffer.sublist(0, 4); - var reader = new KafkaBytesReader.fromBytes(sizeBytes); - _sizes[hostPort] = reader.readInt32(); - } - - List extra; - if (buffer.length > _sizes[hostPort] + 4) { - extra = buffer.sublist(_sizes[hostPort] + 4); - buffer.removeRange(_sizes[hostPort] + 4, buffer.length); + Future send(KRequest request, String host, int port) { + /// TODO: Find a way to perform `socket.sendPacket()` without async gap + Future result; + if (_apiVersions == null) { + /// TODO: resolve versions for each node separately + /// It may not always be true that all nodes in Kafka cluster + /// support exactly the same versions, e.g. during server upgrades. + /// Might have to move this logic to [KSocket] (?). + result = + _resolveApiVersions(host, port).then((_) => _getSocket(host, port)); + } else { + result = _getSocket(host, port); } - if (buffer.length == _sizes[hostPort] + 4) { - var header = buffer.sublist(4, 8); - var reader = new KafkaBytesReader.fromBytes(header); - var correlationId = reader.readInt32(); - var request = _inflightRequests.keys - .firstWhere((r) => r.correlationId == correlationId); - var completer = _inflightRequests[request]; - var response = request.createResponse(buffer); - _inflightRequests.remove(request); - buffer.clear(); - _sizes[hostPort] = -1; + return result.then((socket) { + var version = _apiVersions[request.apiKey]; + _logger.finest('Sending $request (v$version) to $host:$port'); + var payload = request.encoder.encode(request, version); + return socket.sendPacket(request.apiKey, version, payload); + }).then((responseData) { + // var version = _apiVersions[request.apiKey]; - completer.complete(response); - if (extra is List && extra.isNotEmpty) { - _handleData(hostPort, extra); - } - } + /// TODO: supply api version to response decoders. + return request.decoder.decode(responseData); + }); } - ContactPoint _getCurrentContactPoint() { - return contactPoints.first; + Future _resolveApiVersions(String host, int port) { + if (_apiResolution != null) return _apiResolution.future; + _apiResolution = new Completer(); + var request = new ApiVersionsRequest(); + _getSocket(host, port).then((socket) { + var payload = request.encoder.encode(request, 0); + return socket.sendPacket(request.apiKey, 0, payload); + }).then((data) { + var response = request.decoder.decode(data); + _apiVersions = resolveApiVersions(response.versions, supportedVersions); + }).whenComplete(() { + _apiResolution.complete(); + }); + return _apiResolution.future; } - // void _rotateDefaultHosts() { - // var current = defaultHosts.removeFirst(); - // defaultHosts.addLast(current); - // } - - Future _getSocket(String host, int port) { + Future _getSocket(String host, int port) { var key = '${host}:${port}'; if (!_sockets.containsKey(key)) { - _sockets[key] = Socket.connect(host, port); - _sockets[key].then((socket) { - socket.setOption(SocketOption.TCP_NODELAY, true); - _buffers[key] = new List(); - _sizes[key] = -1; - _subscriptions[key] = socket.listen((d) => _handleData(key, d)); - _flushFutures[socket] = new Future.value(); - }, onError: (error) { - _sockets.remove(key); - }); + _sockets[key] = KSocket.connect(host, port); } return _sockets[key]; } -} - -/// Stores metadata information about cluster including available brokers -/// and topics. -class ClusterMetadata { - /// List of brokers in the cluster. - final List brokers; - - /// List with metadata for each topic. - final List topics; - - /// Creates new instance of cluster metadata. - ClusterMetadata(this.brokers, this.topics); - - /// Returns [Broker] by specified [nodeId]. - Broker getBroker(int nodeId) { - return brokers.firstWhere((b) => b.id == nodeId); - } - /// Returns [TopicMetadata] for specified [topicName]. - /// - /// If no topic is found will throw `StateError`. - TopicMetadata getTopicMetadata(String topicName) { - return topics.firstWhere((topic) => topic.topicName == topicName, - orElse: () => - throw new StateError('No topic ${topicName} found in metadata.')); + Future close() async { + /// TODO: wait for complition of any in-flight request. + /// TODO: don't allow any new requests to be send. + for (Future s in _sockets.values) { + await (await s).destroy(); + } + _sockets.clear(); } } diff --git a/lib/src/util/crc32.dart b/lib/src/util/crc32.dart index 5d110d8..f7b17b7 100644 --- a/lib/src/util/crc32.dart +++ b/lib/src/util/crc32.dart @@ -1,5 +1,3 @@ -part of kafka.protocol; - /// CRC32 checksum calculator. /// // TODO: extract in it's own package (?) diff --git a/lib/src/util/group_by.dart b/lib/src/util/group_by.dart new file mode 100644 index 0000000..674be2b --- /dev/null +++ b/lib/src/util/group_by.dart @@ -0,0 +1,12 @@ +Map> groupBy(List list, K func(V element)) { + var grouped = new Map>(); + for (var element in list) { + K key = func(element); + if (!grouped.containsKey(key)) { + grouped[key] = new List(); + } + grouped[key].add(element); + } + + return grouped; +} diff --git a/lib/src/util/retry.dart b/lib/src/util/retry.dart new file mode 100644 index 0000000..cd694d6 --- /dev/null +++ b/lib/src/util/retry.dart @@ -0,0 +1,16 @@ +import 'dart:async'; + +Future retryAsync(Future func(), int retries, Duration delay, + {bool test(error)}) { + return func().catchError((error) { + if (retries == 0) { + return new Future.error(error); + } else { + if (test is Function && !test(error)) { + return new Future.error(error); + } + return new Future.delayed( + delay, () => retryAsync(func, retries - 1, delay)); + } + }); +} diff --git a/lib/src/util/tuple.dart b/lib/src/util/tuple.dart new file mode 100644 index 0000000..8016b7c --- /dev/null +++ b/lib/src/util/tuple.dart @@ -0,0 +1,42 @@ +import 'package:quiver/core.dart'; + +/// Creates tuple of two values. +Tuple2 tuple2(T1 $1, T2 $2) { + return new Tuple2($1, $2); +} + +/// Creates tuple of three values. +Tuple3 tuple3(T1 $1, T2 $2, T3 $3) { + return new Tuple3($1, $2, $3); +} + +class Tuple2 { + final T1 $1; + final T2 $2; + + Tuple2(this.$1, this.$2); + + @override + int get hashCode => hash2($1, $2); + + bool operator ==(o) => o is Tuple3 && o.$1 == $1 && o.$2 == $2; + + @override + toString() => "[${$1}, ${$2}]"; +} + +class Tuple3 { + final T1 $1; + final T2 $2; + final T3 $3; + + Tuple3(this.$1, this.$2, this.$3); + + @override + int get hashCode => hash3($1, $2, $3); + + bool operator ==(o) => o is Tuple3 && o.$1 == $1 && o.$2 == $2 && o.$3 == $3; + + @override + toString() => "[${$1}, ${$2}, ${$3}]"; +} diff --git a/lib/src/versions.dart b/lib/src/versions.dart new file mode 100644 index 0000000..3bc37eb --- /dev/null +++ b/lib/src/versions.dart @@ -0,0 +1,61 @@ +import 'common.dart'; +import 'versions_api.dart'; + +export 'common.dart' show ApiKey; +export 'versions_api.dart' show ApiVersion; + +const List supportedVersions = const [ + const ApiVersion(ApiKey.produce, 2, 2), + const ApiVersion(ApiKey.fetch, 2, 2), + const ApiVersion(ApiKey.offsets, 1, 1), + const ApiVersion(ApiKey.metadata, 0, 0), + const ApiVersion(ApiKey.offsetCommit, 2, 2), + const ApiVersion(ApiKey.offsetFetch, 1, 1), + const ApiVersion(ApiKey.groupCoordinator, 0, 0), + const ApiVersion(ApiKey.joinGroup, 1, 1), + const ApiVersion(ApiKey.heartbeat, 0, 0), + const ApiVersion(ApiKey.leaveGroup, 0, 0), + const ApiVersion(ApiKey.syncGroup, 0, 0), + const ApiVersion(ApiKey.apiVersions, 0, 0), +]; + +/// Returns a map where keys are one of [ApiKey] constants and values +/// contain version number of corresponding Kafka API, which: +/// +/// - supported by both server and client +/// - is highest possible. +/// +/// This function throws `UnsupportedError` if any of API versions +/// between server and client don't have an overlap. +Map resolveApiVersions( + List serverVersions, List clientVersions) { + Map serverMap = + new Map.fromIterable(serverVersions, key: (v) => v.key); + Map clientMap = + new Map.fromIterable(clientVersions, key: (v) => v.key); + Map result = new Map(); + for (var key in clientMap.keys) { + var client = clientMap[key]; + var server = serverMap[key]; + + /// Check if version ranges overlap. If they don't then client + /// can't communicate with this Kafka broker. + if (!overlap(client.min, client.max, server.min, server.max)) { + throw new UnsupportedError( + 'Unsupported API version: $server. Client supported versions are: $client.'); + } + + /// Find latest possible version supported by both client and server. + var version = client.max; + while (version > server.max) { + version--; + } + result[key] = version; + } + return result; +} + +/// Checks if two integer ranges overlap. +bool overlap(int x1, int x2, int y1, y2) { + return x2 >= y1 && y2 >= x1; +} diff --git a/lib/src/versions_api.dart b/lib/src/versions_api.dart new file mode 100644 index 0000000..f3d6c95 --- /dev/null +++ b/lib/src/versions_api.dart @@ -0,0 +1,63 @@ +import 'common.dart'; +import 'errors.dart'; +import 'io.dart'; + +class ApiVersionsRequest implements KRequest { + @override + final int apiKey = ApiKey.apiVersions; + + @override + ResponseDecoder get decoder => + const _ApiVersionsResponseDecoder(); + + @override + RequestEncoder get encoder => const _ApiVersionsRequestEncoder(); +} + +class ApiVersionsResponse { + final int error; + final List versions; + + ApiVersionsResponse(this.error, this.versions) { + if (error != Errors.NoError) throw new KafkaError.fromCode(error, this); + } + + @override + String toString() => 'ApiVersionsResponse{$versions}'; +} + +class ApiVersion { + final int key; + final int min; + final int max; + + const ApiVersion(this.key, this.min, this.max); + + @override + String toString() => 'ApiVersion{$key, min: $min, max: $max}'; +} + +class _ApiVersionsRequestEncoder implements RequestEncoder { + const _ApiVersionsRequestEncoder(); + + @override + List encode(ApiVersionsRequest request, int version) { + assert(version == 0, + 'Only v0 of ApiVersions request is supported by the client.'); + return []; + } +} + +class _ApiVersionsResponseDecoder + implements ResponseDecoder { + const _ApiVersionsResponseDecoder(); + + @override + ApiVersionsResponse decode(List data) { + var reader = new KafkaBytesReader.fromBytes(data); + var error = reader.readInt16(); + var versions = reader.readObjectArray( + (_) => new ApiVersion(_.readInt16(), _.readInt16(), _.readInt16())); + return new ApiVersionsResponse(error, versions); + } +} diff --git a/lib/testing.dart b/lib/testing.dart new file mode 100644 index 0000000..a76b567 --- /dev/null +++ b/lib/testing.dart @@ -0,0 +1,150 @@ +/// Provides testing utilities for Kafka-driven apps. +/// +/// Includes implementations of `MockSession`, `MockProducer`, +/// `MockConsumer` which conform to corresponding interfaces. +library kafka.testing; + +// import 'package:kafka/ng.dart'; +// import 'dart:async'; +// import 'dart:collection'; + +// class MockKafkaSession implements Session { +// @override +// Future close() { +// return null; +// // no-op +// } + +// @override +// Future getConsumerMetadata(String consumerGroup) { +// return new Future.value( +// new GroupCoordinatorResponse(0, 1, '127.0.0.1', 9092)); +// } + +// @override +// Future getMetadata(Set topicNames, +// {bool invalidateCache: false}) { +// var brokers = [new Broker(1, '127.0.0.1', 9092)]; +// var topics = topicNames.map((name) { +// return new TopicMetadata(0, name, [ +// new PartitionMetadata(0, 0, 1, [1], [1]) +// ]); +// }).toList(); +// return new Future.value(new ClusterMetadata(brokers, topics)); +// } + +// @override +// Future> listTopics() { +// throw new UnsupportedError('Unsupported by MockKafkaSession.'); +// } + +// Map>> _data = +// new Map>>(); + +// @override +// Future send(Broker broker, KafkaRequest request) { +// switch (request.runtimeType) { +// case ProduceRequest: +// return new Future.value(_produce(request)); +// case JoinGroupRequest: +// return new Future.value(_joinGroup(request)); +// case SyncGroupRequest: +// return new Future.value(_syncGroup(request)); +// case OffsetFetchRequest: +// return new Future.value(_offsetFetch(request)); +// case OffsetCommitRequest: +// return new Future.value(_offsetCommit(request)); +// case FetchRequest: +// return new Future.value(_fetch(request)); +// case HeartbeatRequest: +// return new Future.value( +// new HeartbeatResponse(KafkaServerError.NoError_)); +// default: +// return null; +// } +// } + +// ProduceResponse _produce(ProduceRequest request) { +// var results = new List(); +// for (var envelope in request.messages) { +// _data.putIfAbsent(envelope.topicName, () => new Map()); +// _data[envelope.topicName] +// .putIfAbsent(envelope.partitionId, () => new List()); +// var offset = _data[envelope.topicName][envelope.partitionId].length; +// results.add(new TopicProduceResult( +// envelope.topicName, envelope.partitionId, 0, offset)); +// _data[envelope.topicName][envelope.partitionId].addAll(envelope.messages); +// } + +// return new ProduceResponse(results); +// } + +// JoinGroupResponse _joinGroup(JoinGroupRequest request) { +// var id = +// 'dart_kafka-' + new DateTime.now().millisecondsSinceEpoch.toString(); +// var meta = request.groupProtocols.first.protocolMetadata; +// return new JoinGroupResponse( +// 0, +// 1, +// request.groupProtocols.first.protocolName, +// id, +// id, +// [new GroupMember(id, meta)]); +// } + +// SyncGroupResponse _syncGroup(SyncGroupRequest request) { +// return new SyncGroupResponse( +// 0, request.groupAssignments.first.memberAssignment); +// } + +// Map>> consumerOffsets = new Map(); + +// OffsetFetchResponse _offsetFetch(OffsetFetchRequest request) { +// consumerOffsets.putIfAbsent( +// request.consumerGroup, () => new Map>()); +// Map> groupOffsets = +// consumerOffsets[request.consumerGroup]; +// var offsets = new List(); +// for (var topic in request.topics.keys) { +// groupOffsets.putIfAbsent(topic, () => new Map()); +// for (var partition in request.topics[topic]) { +// groupOffsets[topic].putIfAbsent(partition, () => -1); +// offsets.add(new ConsumerOffset( +// topic, partition, groupOffsets[topic][partition], '')); +// } +// } + +// return new OffsetFetchResponse.fromOffsets(offsets); +// } + +// OffsetCommitResponse _offsetCommit(OffsetCommitRequest request) { +// var groupOffsets = consumerOffsets[request.consumerGroup]; +// List results = []; +// for (var offset in request.offsets) { +// groupOffsets[offset.topicName][offset.partitionId] = offset.offset; +// results.add(new OffsetCommitResult( +// offset.topicName, offset.partitionId, KafkaServerError.NoError_)); +// } + +// return new OffsetCommitResponse(results); +// } + +// FetchResponse _fetch(FetchRequest request) { +// List results = new List(); +// for (var topic in request.topics.keys) { +// for (var p in request.topics[topic]) { +// var messages = _data[topic][p.partitionId].asMap(); +// var requestedMessages = new Map(); +// for (var o in messages.keys) { +// if (o >= p.fetchOffset) { +// requestedMessages[o] = messages[o]; +// } +// } +// var messageSet = new MessageSet(requestedMessages); +// results.add(new FetchResult(topic, p.partitionId, +// KafkaServerError.NoError_, messages.length, messageSet)); +// } +// } +// return new FetchResponse(results); +// } +// } diff --git a/pubspec.yaml b/pubspec.yaml index 29b6f92..29af522 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,17 +1,21 @@ name: 'kafka' version: 0.1.0 -description: Kafka Client library for Dartlang -homepage: https://github.com/pulyaevskiy/dart-kafka +description: Kafka client library for Dartlang +homepage: https://github.com/dart-drivers/kafka author: Anatoly Pulyaevskiy environment: - sdk: '>=1.12.0 <2.0.0' + sdk: '>=2.1.0 <3.0.0' dependencies: - quiver: ^0.22.0 - logging: "^0.11.1+1" + async: ^2.4.1 + quiver: ^2.1.3 + logging: ^0.11.4 + pool: ^1.4.0 dev_dependencies: - test: "^0.12.4+7" - mockito: "^0.11.0" - dart_coveralls: "^0.4.0" + test: ^1.8.0 + mockito: ^4.1.1 + dart_coveralls: ^0.6.0+4 + string_scanner: ^1.0.5 + source_span: ^1.7.0 diff --git a/test/all.dart b/test/all.dart index 2831a37..1253cbf 100644 --- a/test/all.dart +++ b/test/all.dart @@ -1,35 +1,45 @@ -library kafka.all_tests; - -import 'common/errors_test.dart' as errors_test; -import 'common/messages_test.dart' as messages_test; -import 'util/crc32_test.dart' as crc32_test; -import 'protocol/bytes_builder_test.dart' as bytes_builder_test; -import 'protocol/bytes_reader_test.dart' as bytes_reader_test; -import 'protocol/fetch_test.dart' as fetch_test; -import 'protocol/offset_commit_test.dart' as offset_commit_test; -import 'protocol/offset_fetch_test.dart' as offset_fetch_test; -import 'protocol/offset_test.dart' as offset_test; -import 'protocol/produce_test.dart' as produce_test; -import 'session_test.dart' as session_test; +import 'async_test.dart' as async_test; import 'consumer_group_test.dart' as consumer_group_test; -import 'producer_test.dart' as producer_test; +import 'consumer_metadata_api_test.dart' as consumer_metadata_api_test; +import 'consumer_offset_api_test.dart' as consumer_offset_api_test; import 'consumer_test.dart' as consumer_test; -import 'fetcher_test.dart' as fetcher_test; +import 'errors_test.dart' as errors_test; +import 'fetch_api_test.dart' as fetch_api_test; +import 'group_membership_api_test.dart' as group_membership_api_test; +import 'io_test.dart' as io_test; +import 'list_offset_api_test.dart' as list_offset_api_test; +import 'messages_test.dart' as messages_test; +import 'metadata_api_test.dart' as metadata_api_test; +import 'metadata_test.dart' as metadata_test; +import 'offset_commit_test.dart' as offset_commit_test; +import 'partition_assignor_test.dart' as partition_assignor_test; +import 'produce_api_test.dart' as produce_api_test; +import 'producer_test.dart' as producer_test; +import 'util/crc32_test.dart' as crc32_test; +import 'versions_api_test.dart' as versions_api_test; +import 'versions_test.dart' as versions_test; +// import 'testing_test.dart' as testing_test; void main() { + async_test.main(); + consumer_group_test.main(); + consumer_offset_api_test.main(); + offset_commit_test.main(); errors_test.main(); - messages_test.main(); - bytes_builder_test.main(); - bytes_reader_test.main(); crc32_test.main(); - session_test.main(); - fetch_test.main(); - offset_commit_test.main(); - offset_fetch_test.main(); - offset_test.main(); - produce_test.main(); - consumer_group_test.main(); - producer_test.main(); + metadata_api_test.main(); + fetch_api_test.main(); + consumer_metadata_api_test.main(); + metadata_test.main(); + produce_api_test.main(); + group_membership_api_test.main(); + io_test.main(); + messages_test.main(); consumer_test.main(); - fetcher_test.main(); + producer_test.main(); + partition_assignor_test.main(); + list_offset_api_test.main(); + versions_api_test.main(); + versions_test.main(); + // testing_test.main(); } diff --git a/test/async_test.dart b/test/async_test.dart new file mode 100644 index 0000000..3403f31 --- /dev/null +++ b/test/async_test.dart @@ -0,0 +1,119 @@ +// Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import "dart:async"; +import "package:test/test.dart"; +import "package:kafka/src/consumer_streamiterator.dart"; +import "package:kafka/src/consumer.dart"; + +main() { + test("stream iterator basic", () async { + var stream = createStream(); + ConsumerStreamIterator iterator = + new ConsumerStreamIterator(stream); + expect(iterator.current, isNull); + expect(await iterator.moveNext(), isTrue); + expect(iterator.current.records.first.key, 'k1'); + expect(await iterator.moveNext(), isTrue); + expect(iterator.current.records.first.key, 'k2'); + expect(await iterator.moveNext(), isFalse); + expect(iterator.current, isNull); + expect(await iterator.moveNext(), isFalse); + }); + + test("stream iterator prefilled", () async { + var stream = createStream(); + ConsumerStreamIterator iterator = + new ConsumerStreamIterator(stream); + await new Future.delayed(Duration.zero); + expect(iterator.current, isNull); + expect(await iterator.moveNext(), isTrue); + expect(iterator.current.records.first.key, 'k1'); + expect(await iterator.moveNext(), isTrue); + expect(iterator.current.records.first.key, 'k2'); + expect(await iterator.moveNext(), isFalse); + expect(iterator.current, isNull); + expect(await iterator.moveNext(), isFalse); + }); + + test("stream iterator error", () async { + var stream = createErrorStream(); + ConsumerStreamIterator iterator = + new ConsumerStreamIterator(stream); + expect(await iterator.moveNext(), isTrue); + expect(iterator.current.records.first.key, 'k1'); + var hasNext = iterator.moveNext(); + expect(hasNext, throwsA("BAD")); // This is an async expectation, + await hasNext.catchError((_) {}); // so we have to wait for the future too. + expect(iterator.current, isNull); + expect(await iterator.moveNext(), isFalse); + expect(iterator.current, isNull); + }); + + test("stream iterator current/moveNext during move", () async { + var stream = createStream(); + ConsumerStreamIterator iterator = + new ConsumerStreamIterator(stream); + var hasNext = iterator.moveNext(); + expect(iterator.moveNext, throwsA(isStateError)); + expect(await hasNext, isTrue); + expect(iterator.current.records.first.key, 'k1'); + iterator.cancel(); + }); + + test("stream iterator error during cancel", () async { + var stream = createCancelErrorStream(); + ConsumerStreamIterator iterator = + new ConsumerStreamIterator(stream); + for (int i = 0; i < 10; i++) { + expect(await iterator.moveNext(), isTrue); + expect(iterator.current.records.first.offset, i); + } + var hasNext = iterator.moveNext(); // active moveNext will be completed. + var cancel = iterator.cancel(); + expect(cancel, throwsA("BAD")); + expect(await hasNext, isFalse); + expect(await iterator.moveNext(), isFalse); + }); + + test("stream iterator collects offsets of consumed records", () async { + var stream = createStream(); + ConsumerStreamIterator iterator = + new ConsumerStreamIterator(stream); + await iterator.moveNext(); + expect(iterator.offsets, hasLength(1)); + expect(iterator.offsets.first.offset, 100); + await iterator.moveNext(); + expect(iterator.offsets, hasLength(2)); + expect(iterator.offsets.last.offset, 200); + iterator.clearOffsets(); + expect(iterator.offsets, isEmpty); + }); +} + +Stream> createStream() async* { + yield new ConsumerRecords( + [new ConsumerRecord('test', 0, 100, 'k1', 'v1', 0)]); + yield new ConsumerRecords( + [new ConsumerRecord('test', 1, 200, 'k2', 'v2', 0)]); +} + +Stream> createErrorStream() async* { + yield new ConsumerRecords([new ConsumerRecord('test', 0, 1, 'k1', 'v1', 0)]); + // Emit an error without stopping the generator. + yield* (new Future>.error("BAD").asStream()); + yield new ConsumerRecords([new ConsumerRecord('test', 1, 1, 'k2', 'v2', 0)]); +} + +/// Create a stream that throws when cancelled. +Stream> createCancelErrorStream() async* { + int i = 0; + try { + while (true) + yield new ConsumerRecords( + [new ConsumerRecord('test', 0, i++, 'k1', 'v1', 0)]); + } finally { + throw "BAD"; + } +} diff --git a/test/common/errors_test.dart b/test/common/errors_test.dart deleted file mode 100644 index 5f794f9..0000000 --- a/test/common/errors_test.dart +++ /dev/null @@ -1,39 +0,0 @@ -library kafka.common.errors.test; - -import 'package:test/test.dart'; -import 'package:kafka/common.dart'; - -void main() { - group('KafkaServerError:', () { - test('it handles error codes correctly', () { - expect(new KafkaServerError(0).isNoError, isTrue); - expect(new KafkaServerError(-1).isUnknown, isTrue); - expect(new KafkaServerError(-1).isError, isTrue); - expect(new KafkaServerError(1).isOffsetOutOfRange, isTrue); - expect(new KafkaServerError(2).isInvalidMessage, isTrue); - expect(new KafkaServerError(3).isUnknownTopicOrPartition, isTrue); - expect(new KafkaServerError(4).isInvalidMessageSize, isTrue); - expect(new KafkaServerError(5).isLeaderNotAvailable, isTrue); - expect(new KafkaServerError(6).isNotLeaderForPartition, isTrue); - expect(new KafkaServerError(7).isRequestTimedOut, isTrue); - expect(new KafkaServerError(8).isBrokerNotAvailable, isTrue); - expect(new KafkaServerError(9).isReplicaNotAvailable, isTrue); - expect(new KafkaServerError(10).isMessageSizeTooLarge, isTrue); - expect(new KafkaServerError(11).isStaleControllerEpoch, isTrue); - expect(new KafkaServerError(12).isOffsetMetadataTooLarge, isTrue); - expect(new KafkaServerError(14).isOffsetsLoadInProgress, isTrue); - expect( - new KafkaServerError(15).isConsumerCoordinatorNotAvailable, isTrue); - expect(new KafkaServerError(16).isNotCoordinatorForConsumer, isTrue); - }); - - test('it can be converted to string', () { - expect( - new KafkaServerError(0).toString(), 'KafkaServerError: NoError(0)'); - }); - - test('it provides error message', () { - expect(new KafkaServerError(1).message, 'OffsetOutOfRange'); - }); - }); -} diff --git a/test/common/messages_test.dart b/test/common/messages_test.dart deleted file mode 100644 index ba82e1f..0000000 --- a/test/common/messages_test.dart +++ /dev/null @@ -1,33 +0,0 @@ -library kafka.common.messages.test; - -import 'package:test/test.dart'; -import 'package:kafka/common.dart'; - -void main() { - group('Messages:', () { - test( - 'compression can not be set on individual messages in produce envelope', - () { - expect(() { - new ProduceEnvelope('test', 0, [ - new Message([1], - attributes: new MessageAttributes(KafkaCompression.gzip)) - ]); - }, throwsStateError); - }); - }); - - group('MessageAttributes:', () { - test('get compression from int', () { - expect(KafkaCompression.none, MessageAttributes.getCompression(0)); - expect(KafkaCompression.gzip, MessageAttributes.getCompression(1)); - expect(KafkaCompression.snappy, MessageAttributes.getCompression(2)); - }); - - test('convert to int', () { - expect(new MessageAttributes(KafkaCompression.none).toInt(), equals(0)); - expect(new MessageAttributes(KafkaCompression.gzip).toInt(), equals(1)); - expect(new MessageAttributes(KafkaCompression.snappy).toInt(), equals(2)); - }); - }); -} diff --git a/test/consumer_group_test.dart b/test/consumer_group_test.dart index c782d59..b077f38 100644 --- a/test/consumer_group_test.dart +++ b/test/consumer_group_test.dart @@ -1,139 +1,132 @@ -library kafka.test.consumer_group; - -import 'package:test/test.dart'; -import 'package:mockito/mockito.dart'; import 'package:kafka/kafka.dart'; -import 'package:kafka/protocol.dart'; -import 'setup.dart'; +import 'package:mockito/mockito.dart'; +import 'package:test/test.dart'; + +class KSessionMock extends Mock implements Session {} void main() { + // TODO: move these tests to new Consumer implementation. group('ConsumerGroup:', () { - KafkaSession _session; - String _topicName = 'dartKafkaTest'; - Broker _coordinator; - Broker _badCoordinator; - - setUp(() async { - var host = await getDefaultHost(); - var session = new KafkaSession([new ContactPoint(host, 9092)]); - var brokersMetadata = await session.getMetadata([_topicName].toSet()); - - var metadata = await session.getConsumerMetadata('testGroup'); - _coordinator = metadata.coordinator; - _badCoordinator = - brokersMetadata.brokers.firstWhere((b) => b.id != _coordinator.id); - _session = spy(new KafkaSessionMock(), session); - }); - - tearDown(() async { - await _session.close(); - }); - - test('it fetches offsets', () async { - var group = new ConsumerGroup(_session, 'testGroup'); - var offsets = await group.fetchOffsets({ - _topicName: [0, 1, 2].toSet() - }); - expect(offsets.length, equals(3)); - offsets.forEach((o) { - expect(o.errorCode, 0); - }); - }); - - test('it tries to refresh coordinator host 3 times on fetchOffsets', - () async { - when(_session.getConsumerMetadata('testGroup')).thenReturn( - new GroupCoordinatorResponse(0, _badCoordinator.id, - _badCoordinator.host, _badCoordinator.port)); - - var group = new ConsumerGroup(_session, 'testGroup'); - // Can't use expect(throws) here since it's async, so `verify` check below - // fails. - try { - await group.fetchOffsets({ - _topicName: [0, 1, 2].toSet() - }); - } catch (e) { - expect(e, new isInstanceOf()); - expect(e.code, equals(16)); - } - verify(_session.getConsumerMetadata('testGroup')).called(3); - }); - - test( - 'it retries to fetchOffsets 3 times if it gets OffsetLoadInProgress error', - () async { - var badOffsets = [ - new ConsumerOffset(_topicName, 0, -1, '', 14), - new ConsumerOffset(_topicName, 1, -1, '', 14), - new ConsumerOffset(_topicName, 2, -1, '', 14) - ]; - when(_session.send(argThat(new isInstanceOf()), - argThat(new isInstanceOf()))) - .thenReturn(new OffsetFetchResponse.fromOffsets(badOffsets)); - - var group = new ConsumerGroup(_session, 'testGroup'); - // Can't use expect(throws) here since it's async, so `verify` check below - // fails. - var now = new DateTime.now(); - try { - await group.fetchOffsets({ - _topicName: [0, 1, 2].toSet() - }); - fail('fetchOffsets must throw an error.'); - } catch (e) { - var diff = now.difference(new DateTime.now()); - expect(diff.abs().inSeconds, greaterThanOrEqualTo(2)); - - expect(e, new isInstanceOf()); - expect(e.code, equals(14)); - } - verify(_session.send(argThat(new isInstanceOf()), - argThat(new isInstanceOf()))) - .called(3); - }); - - test('it tries to refresh coordinator host 3 times on commitOffsets', - () async { - when(_session.getConsumerMetadata('testGroup')).thenReturn( - new GroupCoordinatorResponse(0, _badCoordinator.id, - _badCoordinator.host, _badCoordinator.port)); - - var group = new ConsumerGroup(_session, 'testGroup'); - var offsets = [new ConsumerOffset(_topicName, 0, 3, '')]; - - try { - await group.commitOffsets(offsets, -1, ''); - } catch (e) { - expect(e, new isInstanceOf()); - expect(e.code, equals(16)); - } - verify(_session.getConsumerMetadata('testGroup')).called(3); - }); - - test('it can reset offsets to earliest', () async { - var offsetMaster = new OffsetMaster(_session); - var earliestOffsets = await offsetMaster.fetchEarliest({ - _topicName: [0, 1, 2].toSet() - }); - - var group = new ConsumerGroup(_session, 'testGroup'); - await group.resetOffsetsToEarliest({ - _topicName: [0, 1, 2].toSet() - }); - - var offsets = await group.fetchOffsets({ - _topicName: [0, 1, 2].toSet() - }); - expect(offsets, hasLength(3)); - - for (var o in offsets) { - var earliest = - earliestOffsets.firstWhere((to) => to.partitionId == o.partitionId); - expect(o.offset, equals(earliest.offset - 1)); - } - }); + // test('it fetches offsets', () async { + // var group = new ConsumerGroup(_session, 'testGroup'); + // var offsets = await group.fetchOffsets({ + // _topicName: [0, 1, 2].toSet() + // }); + // expect(offsets.length, equals(3)); + // offsets.forEach((o) { + // expect(o.errorCode, 0); + // }); + // }); + // + // test('it tries to refresh coordinator host 3 times on fetchOffsets', + // () async { + // when(_session.getConsumerMetadata('testGroup')).thenReturn( + // new Future.value(new GroupCoordinatorResponse(0, _badCoordinator.id, + // _badCoordinator.host, _badCoordinator.port))); + // + // var group = new ConsumerGroup(_session, 'testGroup'); + // // Can't use expect(throws) here since it's async, so `verify` check below + // // fails. + // try { + // await group.fetchOffsets({ + // _topicName: [0, 1, 2].toSet() + // }); + // } catch (e) { + // expect(e, new isInstanceOf()); + // expect(e.code, equals(16)); + // } + // verify(_session.getConsumerMetadata('testGroup')).called(3); + // }); + // + // test( + // 'it retries to fetchOffsets 3 times if it gets OffsetLoadInProgress error', + // () async { + // var badOffsets = [ + // new ConsumerOffset(_topicName, 0, -1, '', 14), + // new ConsumerOffset(_topicName, 1, -1, '', 14), + // new ConsumerOffset(_topicName, 2, -1, '', 14) + // ]; + // when(_session.send(argThat(new isInstanceOf()), + // argThat(new isInstanceOf()))) + // .thenAnswer((invocation) { + // throw new KafkaServerError.fromCode( + // KafkaServerError.OffsetsLoadInProgress, + // new OffsetFetchResponse.fromOffsets(badOffsets)); + // }); + // + // var group = new ConsumerGroup(_session, 'testGroup'); + // // Can't use expect(throws) here since it's async, so `verify` check below + // // fails. + // var now = new DateTime.now(); + // try { + // await group.fetchOffsets({ + // _topicName: [0, 1, 2].toSet() + // }); + // fail('fetchOffsets must throw an error.'); + // } catch (e) { + // var diff = now.difference(new DateTime.now()); + // expect(diff.abs().inSeconds, greaterThanOrEqualTo(2)); + // + // expect(e, new isInstanceOf()); + // expect(e.code, equals(14)); + // } + // verify(_session.send(argThat(new isInstanceOf()), + // argThat(new isInstanceOf()))) + // .called(3); + // }); + // + // test('it tries to refresh coordinator host 3 times on commitOffsets', + // () async { + // when(_session.getConsumerMetadata('testGroup')).thenReturn( + // new Future.value(new GroupCoordinatorResponse(0, _badCoordinator.id, + // _badCoordinator.host, _badCoordinator.port))); + // + // var group = new ConsumerGroup(_session, 'testGroup'); + // var offsets = [new ConsumerOffset(_topicName, 0, 3, '')]; + // + // try { + // await group.commitOffsets(offsets); + // } catch (e) { + // expect(e, new isInstanceOf()); + // expect(e.code, equals(16)); + // } + // verify(_session.getConsumerMetadata('testGroup')).called(3); + // }); + // + // test('it can reset offsets to earliest', () async { + // var offsetMaster = new OffsetMaster(_session); + // var earliestOffsets = await offsetMaster.fetchEarliest({ + // _topicName: [0, 1, 2].toSet() + // }); + // + // var group = new ConsumerGroup(_session, 'testGroup'); + // await group.resetOffsetsToEarliest({ + // _topicName: [0, 1, 2].toSet() + // }); + // + // var offsets = await group.fetchOffsets({ + // _topicName: [0, 1, 2].toSet() + // }); + // expect(offsets, hasLength(3)); + // + // for (var o in offsets) { + // var earliest = + // earliestOffsets.firstWhere((to) => to.partitionId == o.partitionId); + // expect(o.offset, equals(earliest.offset - 1)); + // } + // }); + // + // test('members can join consumer group', () async { + // var group = new ConsumerGroup(_session, 'newGroup'); + // var membership = await group.join(15000, '', 'consumer', [ + // new GroupProtocol.roundrobin(0, ['foo'].toSet()) + // ]); + // expect(membership, new isInstanceOf()); + // expect(membership.memberId, isNotEmpty); + // expect(membership.memberId, membership.leaderId); + // expect(membership.groupProtocol, 'roundrobin'); + // expect(membership.assignment.partitionAssignment, + // containsPair('foo', [0, 1, 2])); + // }); }); } - -class KafkaSessionMock extends Mock implements KafkaSession {} diff --git a/test/consumer_metadata_api_test.dart b/test/consumer_metadata_api_test.dart new file mode 100644 index 0000000..050f755 --- /dev/null +++ b/test/consumer_metadata_api_test.dart @@ -0,0 +1,40 @@ +import 'dart:async'; + +import 'package:kafka/kafka.dart'; +import 'package:test/test.dart'; + +void main() { + group('Consumer Metadata API: ', () { + Session session; + + setUpAll(() async { + try { + session = new Session(['127.0.0.1:9092']); + var request = new GroupCoordinatorRequest('testGroup'); + await session.send(request, '127.0.0.1', 9092); + } catch (error) { + await new Future.delayed(new Duration(milliseconds: 1000)); + } + }); + + tearDownAll(() async { + await session.close(); + }); + + test('we can send group coordinator requests to Kafka broker', () async { + var request = new GroupCoordinatorRequest('testGroup'); + var response = await session.send(request, '127.0.0.1', 9092); + expect(response, isA()); + expect(response.coordinatorId, greaterThanOrEqualTo(0)); + expect(response.coordinatorHost, '127.0.0.1'); + expect(response.coordinatorPort, isIn([9092, 9093])); + }); + + test('group coordinator response throws server error if present', () { + expect(() { + new GroupCoordinatorResponse( + Errors.ConsumerCoordinatorNotAvailable, null, null, null); + }, throwsA(isA())); + }); + }); +} diff --git a/test/consumer_offset_api_test.dart b/test/consumer_offset_api_test.dart new file mode 100644 index 0000000..533a061 --- /dev/null +++ b/test/consumer_offset_api_test.dart @@ -0,0 +1,32 @@ +import 'package:test/test.dart'; +import 'package:kafka/kafka.dart'; + +void main() { + group('OffsetFetchApi:', () { + Session session = new Session(['127.0.0.1:9092']); + OffsetFetchRequest _request; + Broker _coordinator; + String _testGroup; + + setUp(() async { + var now = new DateTime.now(); + _testGroup = 'group:' + now.millisecondsSinceEpoch.toString(); + _coordinator = await session.metadata.fetchGroupCoordinator(_testGroup); + _request = new OffsetFetchRequest( + _testGroup, [new TopicPartition('dartKafkaTest', 0)]); + }); + + tearDownAll(() async { + await session.close(); + }); + + test('it fetches consumer offsets', () async { + OffsetFetchResponse response = + await session.send(_request, _coordinator.host, _coordinator.port); + expect(response.offsets, hasLength(equals(1))); + expect(response.offsets.first.topic, equals('dartKafkaTest')); + expect(response.offsets.first.partition, equals(0)); + expect(response.offsets.first.error, equals(0)); + }); + }); +} diff --git a/test/consumer_test.dart b/test/consumer_test.dart index f07f750..eaf4c78 100644 --- a/test/consumer_test.dart +++ b/test/consumer_test.dart @@ -1,148 +1,149 @@ -library kafka.test.consumer; - -import 'package:test/test.dart'; import 'package:kafka/kafka.dart'; -import 'setup.dart'; +import 'package:test/test.dart'; void main() { group('Consumer:', () { - KafkaSession _session; - String _topicName = 'dartKafkaTest'; - Map _expectedOffsets = new Map(); + Session session = new Session(['127.0.0.1:9092']); + var date = new DateTime.now().millisecondsSinceEpoch; + String topic = 'testTopic-${date}'; + Map expectedOffsets = new Map(); + String group = 'cg:${date}'; setUp(() async { - var date = new DateTime.now().millisecondsSinceEpoch; - _topicName = 'testTopic-${date}'; - var host = await getDefaultHost(); - _session = new KafkaSession([new ContactPoint(host, 9092)]); - var producer = new Producer(_session, 1, 100); - var result = await producer.produce([ - new ProduceEnvelope(_topicName, 0, [new Message('msg1'.codeUnits)]), - new ProduceEnvelope(_topicName, 1, [new Message('msg2'.codeUnits)]), - new ProduceEnvelope(_topicName, 2, [new Message('msg3'.codeUnits)]), - ]); - if (result.hasErrors) { - throw new StateError( - 'Consumer test: setUp failed to produce messages.'); - } - _expectedOffsets = result.offsets[_topicName]; - }); - - tearDown(() async { - await _session.close(); - }); - - test('it can consume messages from multiple brokers and commit offsets', - () async { - var topics = { - _topicName: [0, 1, 2].toSet() - }; - var consumer = new Consumer( - _session, new ConsumerGroup(_session, 'cg'), topics, 100, 1); - var consumedOffsets = new Map(); - await for (MessageEnvelope envelope in consumer.consume(limit: 3)) { - consumedOffsets[envelope.partitionId] = envelope.offset; - expect(envelope.offset, _expectedOffsets[envelope.partitionId]); - envelope.commit(''); - } - expect(consumedOffsets.length, _expectedOffsets.length); + var producer = new Producer( + new StringSerializer(), + new StringSerializer(), + new ProducerConfig(bootstrapServers: ['127.0.0.1:9092'])); + var rec1 = new ProducerRecord(topic, 0, 'akey', 'avalue'); + var rec2 = new ProducerRecord(topic, 1, 'bkey', 'bvalue'); + var rec3 = new ProducerRecord(topic, 2, 'ckey', 'cvalue'); + producer..add(rec1)..add(rec2)..add(rec3); + var res1 = await rec1.result; + expectedOffsets[res1.topicPartition.partition] = res1.offset; + var res2 = await rec2.result; + expectedOffsets[res2.topicPartition.partition] = res2.offset; + var res3 = await rec3.result; + expectedOffsets[res3.topicPartition.partition] = res3.offset; + await producer.close(); }); - test( - 'it can consume messages from multiple brokers without commiting offsets', - () async { - var topics = { - _topicName: [0, 1, 2].toSet() - }; - var consumer = new Consumer( - _session, new ConsumerGroup(_session, 'cg'), topics, 100, 1); - var consumedOffsets = new Map(); - await for (MessageEnvelope envelope in consumer.consume(limit: 3)) { - consumedOffsets[envelope.partitionId] = envelope.offset; - expect(envelope.offset, _expectedOffsets[envelope.partitionId]); - envelope.ack(); - } - expect(consumedOffsets, _expectedOffsets); - - var group = new ConsumerGroup(_session, 'cg'); - var offsets = await group.fetchOffsets(topics); - expect(offsets, hasLength(3)); - for (var o in offsets) { - expect(-1, o.offset); - } + tearDownAll(() async { + await session.close(); }); - test('it can handle cancelation request', () async { - var topics = { - _topicName: [0, 1, 2].toSet() - }; - var consumer = new Consumer( - _session, new ConsumerGroup(_session, 'cg'), topics, 100, 1); + test('it can consume messages from multiple brokers', () async { + var consumer = new Consumer( + group, new StringDeserializer(), new StringDeserializer(), session); + await consumer.subscribe([topic]); + var iterator = consumer.poll(); + int i = 0; var consumedOffsets = new Map(); - await for (MessageEnvelope envelope in consumer.consume(limit: 3)) { - consumedOffsets[envelope.partitionId] = envelope.offset; - expect(envelope.offset, _expectedOffsets[envelope.partitionId]); - envelope.cancel(); + while (await iterator.moveNext()) { + var records = iterator.current; + records.records.forEach((record) { + consumedOffsets[record.partition] = record.offset; + print("Record: [${record.key}, ${record.value}]"); + }); + i += records.records.length; + if (i >= 3) break; } - expect(consumedOffsets.length, equals(1)); + expect(consumedOffsets, expectedOffsets); + +// var group = new ConsumerGroup(_session, 'cg'); +// var offsets = await group.fetchOffsets(topics); +// expect(offsets, hasLength(3)); +// for (var o in offsets) { +// expect(-1, o.offset); +// } }); - test('it propagates worker errors via stream controller', () async { - var topics = { - 'someTopic': - [0, 1, 2, 3].toSet() // request partition which does not exist. - }; - - var consume = () async { - try { - var consumer = new Consumer( - _session, new ConsumerGroup(_session, 'cg'), topics, 100, 1); - var consumedOffsets = new Map(); - await for (MessageEnvelope envelope in consumer.consume(limit: 3)) { - envelope.ack(); - } - return false; - } catch (e) { - return true; - } - }; - - var result = await consume(); - - expect(result, isTrue); - }); - - test('it can consume batches of messages from multiple brokers', () async { - var topics = { - _topicName: [0, 1, 2].toSet() - }; - var consumer = new Consumer( - _session, new ConsumerGroup(_session, 'cg'), topics, 100, 1); - var consumedOffsets = new Map(); - - var first, last; - await for (var batch in consumer.batchConsume(3)) { - if (first == null) { - first = batch; - first.ack(); - } else if (last == null) { - last = batch; - last.cancel(); - } - } - - expect(first.items.length + last.items.length, 3); - - for (var i in first.items) { - consumedOffsets[i.partitionId] = i.offset; - expect(i.offset, _expectedOffsets[i.partitionId]); - } - for (var i in last.items) { - consumedOffsets[i.partitionId] = i.offset; - expect(i.offset, _expectedOffsets[i.partitionId]); - } - - expect(consumedOffsets.length, _expectedOffsets.length); - }); + // test('it can consume messages from multiple brokers and commit offsets', + // () async { + // var topics = { + // _topicName: [0, 1, 2].toSet() + // }; + // var consumer = new KConsumer( + // _session, new ConsumerGroup(_session, 'cg'), topics, 100, 1); + // var consumedOffsets = new Map(); + // await for (MessageEnvelope envelope in consumer.consume(limit: 3)) { + // consumedOffsets[envelope.partitionId] = envelope.offset; + // expect(envelope.offset, _expectedOffsets[envelope.partitionId]); + // envelope.commit(''); + // } + // expect(consumedOffsets.length, _expectedOffsets.length); + // }); + + // test('it can handle cancelation request', () async { + // var topics = { + // _topicName: [0, 1, 2].toSet() + // }; + // var consumer = new Consumer( + // _session, new ConsumerGroup(_session, 'cg'), topics, 100, 1); + // var consumedOffsets = new Map(); + // await for (MessageEnvelope envelope in consumer.consume(limit: 3)) { + // consumedOffsets[envelope.partitionId] = envelope.offset; + // expect(envelope.offset, _expectedOffsets[envelope.partitionId]); + // envelope.cancel(); + // } + // expect(consumedOffsets.length, equals(1)); + // }); + + // test('it propagates worker errors via stream controller', () async { + // var topics = { + // 'someTopic': + // [0, 1, 2, 3].toSet() // request partition which does not exist. + // }; + // + // var consume = () async { + // try { + // var consumer = new Consumer( + // _session, new ConsumerGroup(_session, 'cg'), topics, 100, 1); + // var consumedOffsets = new Map(); + // await for (MessageEnvelope envelope in consumer.consume(limit: 3)) { + // envelope.ack(); + // } + // return false; + // } catch (e) { + // return true; + // } + // }; + // + // var result = await consume(); + // + // expect(result, isTrue); + // }); + + // test('it can consume batches of messages from multiple brokers', () async { + // var topics = { + // _topicName: [0, 1, 2].toSet() + // }; + // var consumer = new Consumer( + // _session, new ConsumerGroup(_session, 'cg'), topics, 100, 1); + // var consumedOffsets = new Map(); + // + // var first, last; + // await for (var batch in consumer.batchConsume(3)) { + // if (first == null) { + // first = batch; + // first.ack(); + // } else if (last == null) { + // last = batch; + // last.cancel(); + // } + // } + // + // expect(first.items.length + last.items.length, 3); + // + // for (var i in first.items) { + // consumedOffsets[i.partitionId] = i.offset; + // expect(i.offset, _expectedOffsets[i.partitionId]); + // } + // for (var i in last.items) { + // consumedOffsets[i.partitionId] = i.offset; + // expect(i.offset, _expectedOffsets[i.partitionId]); + // } + // + // expect(consumedOffsets.length, _expectedOffsets.length); + // }); }); } diff --git a/test/errors_test.dart b/test/errors_test.dart new file mode 100644 index 0000000..9c1a258 --- /dev/null +++ b/test/errors_test.dart @@ -0,0 +1,11 @@ +import 'package:test/test.dart'; +import 'package:kafka/kafka.dart'; + +void main() { + group('Errors:', () { + test('it can be converted to string', () { + expect(new KafkaError.fromCode(1, null).toString(), + 'OffsetOutOfRangeError(1)'); + }); + }); +} diff --git a/test/fetch_api_test.dart b/test/fetch_api_test.dart new file mode 100644 index 0000000..9509d20 --- /dev/null +++ b/test/fetch_api_test.dart @@ -0,0 +1,81 @@ +import 'package:kafka/kafka.dart'; +import 'package:test/test.dart'; + +void main() { + group('FetchApi:', () { + String topic = 'dartKafkaTest'; + Broker host; + Session session = new Session(['127.0.0.1:9092']); + String message; + + setUp(() async { + var meta = await session.metadata.fetchTopics([topic]); + var leaderId = meta[topic].partitions[0].leader; + host = meta.brokers[leaderId]; + }); + + tearDownAll(() async { + await session.close(); + }); + + test('it fetches messages from Kafka topic', () async { + var now = new DateTime.now(); + message = 'test:' + now.toIso8601String(); + var producer = new Producer( + new StringSerializer(), + new StringSerializer(), + new ProducerConfig(bootstrapServers: ['127.0.0.1:9092'])); + var rec = new ProducerRecord(topic, 0, 'key', message); + producer.add(rec); + var result = await rec.result; + var offset = result.offset; + await producer.close(); + + FetchRequest request = new FetchRequest(100, 1); + request.add(new TopicPartition(topic, 0), new FetchData(offset, 35656)); + var response = await session.send(request, host.host, host.port); + + expect(response.results, hasLength(1)); + expect( + response.results.first.messages, hasLength(greaterThanOrEqualTo(1))); + var keyData = response.results.first.messages[offset].key; + var valueData = response.results.first.messages[offset].value; + var deser = new StringDeserializer(); + var value = deser.deserialize(valueData); + expect(value, equals(message)); + var key = deser.deserialize(keyData); + expect(key, equals('key')); + }); + + // test('it fetches GZip encoded messages from Kafka topic', () async { + // var now = new DateTime.now(); + // _message = 'test:' + now.toIso8601String(); + // ProduceRequest produce = new ProduceRequest(1, 1000, [ + // new ProduceEnvelope( + // _topicName, + // 0, + // [ + // new Message('hello world'.codeUnits), + // new Message('peace and love'.codeUnits) + // ], + // compression: KafkaCompression.gzip) + // ]); + // + // ProduceResponse produceResponse = await _session.send(_host, produce); + // _offset = produceResponse.results.first.offset; + // _request = new FetchRequest(100, 1); + // _request.add(_topicName, 0, _offset); + // FetchResponse response = await _session.send(_host, _request); + // + // expect(response.results, hasLength(1)); + // expect(response.results.first.messageSet, hasLength(equals(2))); + // var value = response.results.first.messageSet.messages[_offset].value; + // var text = new String.fromCharCodes(value); + // expect(text, equals('hello world')); + // + // value = response.results.first.messageSet.messages[_offset + 1].value; + // text = new String.fromCharCodes(value); + // expect(text, equals('peace and love')); + // }); + }); +} diff --git a/test/fetcher_test.dart b/test/fetcher_test.dart deleted file mode 100644 index a59c7f6..0000000 --- a/test/fetcher_test.dart +++ /dev/null @@ -1,65 +0,0 @@ -library kafka.test.fetcher; - -import 'package:test/test.dart'; -import 'package:kafka/kafka.dart'; -import 'setup.dart'; - -void main() { - group('Fetcher:', () { - KafkaSession _session; - String _topicName = 'dartKafkaTest'; - Map _expectedOffsets = new Map(); - List _initialOffsets = new List(); - - setUp(() async { - var host = await getDefaultHost(); - _session = new KafkaSession([new ContactPoint(host, 9092)]); - var producer = new Producer(_session, 1, 100); - var result = await producer.produce([ - new ProduceEnvelope(_topicName, 0, [new Message('msg1'.codeUnits)]), - new ProduceEnvelope(_topicName, 1, [new Message('msg2'.codeUnits)]), - new ProduceEnvelope(_topicName, 2, [new Message('msg3'.codeUnits)]), - ]); - _expectedOffsets = result.offsets[_topicName]; - result.offsets[_topicName].forEach((p, o) { - _initialOffsets.add(new TopicOffset(_topicName, p, o)); - }); - }); - - tearDown(() async { - await _session.close(); - }); - - test('it can consume exact number of messages from multiple brokers', - () async { - var fetcher = new Fetcher(_session, _initialOffsets); - var fetchedCount = 0; - await for (MessageEnvelope envelope in fetcher.fetch(limit: 3)) { - expect(envelope.offset, _expectedOffsets[envelope.partitionId]); - envelope.commit(''); - fetchedCount++; - } - expect(fetchedCount, equals(3)); - }); - - test('it can handle cancelation request', () async { - var fetcher = new Fetcher(_session, _initialOffsets); - var fetchedCount = 0; - await for (MessageEnvelope envelope in fetcher.fetch(limit: 3)) { - expect(envelope.offset, _expectedOffsets[envelope.partitionId]); - envelope.cancel(); - fetchedCount++; - } - expect(fetchedCount, equals(1)); - }); - - test('it can resolve earliest offset', () async { - var startOffsets = [new TopicOffset.earliest(_topicName, 0)]; - var fetcher = new Fetcher(_session, startOffsets); - - await for (MessageEnvelope envelope in fetcher.fetch(limit: 1)) { - envelope.ack(); - } - }); - }); -} diff --git a/test/group_membership_api_test.dart b/test/group_membership_api_test.dart new file mode 100644 index 0000000..f6f376e --- /dev/null +++ b/test/group_membership_api_test.dart @@ -0,0 +1,62 @@ +import 'package:kafka/kafka.dart'; +import 'package:test/test.dart'; + +void main() { + group('GroupMembershipApi:', () { + String group; + String _topic = 'dartKafkaTest'; + Broker _broker; + Session _session = new Session(['127.0.0.1:9092']); + + setUp(() async { + var now = new DateTime.now().millisecondsSinceEpoch.toString(); + group = 'test-group-' + now; + _broker = await _session.metadata.fetchGroupCoordinator(group); + }); + + tearDownAll(() async { + await _session.close(); + }); + + test('we can join and leave a consumer group', () async { + var protocols = [ + new GroupProtocol.roundrobin(0, [_topic].toSet()) + ]; + var joinRequest = + new JoinGroupRequest(group, 15000, 1000, '', 'consumer', protocols); + JoinGroupResponse joinResponse = + await _session.send(joinRequest, _broker.host, _broker.port); + expect(joinResponse, isA()); + expect(joinResponse.error, 0); + expect(joinResponse.generationId, greaterThanOrEqualTo(1)); + expect(joinResponse.groupProtocol, 'roundrobin'); + expect(joinResponse.leaderId, joinResponse.memberId); + expect(joinResponse.members, hasLength(1)); + + var topics = { + _topic: [0, 1, 2] + }; + var memberAssignment = new MemberAssignment(0, topics, null); + var assignments = [ + new GroupAssignment(joinResponse.memberId, memberAssignment) + ]; + var syncRequest = new SyncGroupRequest( + group, joinResponse.generationId, joinResponse.memberId, assignments); + SyncGroupResponse syncResponse = + await _session.send(syncRequest, _broker.host, _broker.port); + expect(syncResponse.error, Errors.NoError); + expect(syncResponse.assignment.partitions, topics); + + var heartbeatRequest = new HeartbeatRequest( + group, joinResponse.generationId, joinResponse.memberId); + HeartbeatResponse heartbeatResponse = + await _session.send(heartbeatRequest, _broker.host, _broker.port); + expect(heartbeatResponse.error, Errors.NoError); + + var leaveRequest = new LeaveGroupRequest(group, joinResponse.memberId); + LeaveGroupResponse leaveResponse = + await _session.send(leaveRequest, _broker.host, _broker.port); + expect(leaveResponse.error, Errors.NoError); + }); + }); +} diff --git a/test/io_test.dart b/test/io_test.dart new file mode 100644 index 0000000..caa43af --- /dev/null +++ b/test/io_test.dart @@ -0,0 +1,158 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:kafka/src/io.dart'; +import 'package:test/test.dart'; + +void main() { + group('PacketStreamTransformer: ', () { + var _controller = new StreamController>(); + test('it transforms incoming byte stream into stream of Kafka packets', + () async { + var t = new PacketStreamTransformer(); + var packetStream = _controller.stream.transform(t); + _controller.add(_createPacket([1, 2, 3, 4])); + _controller.add(_createPacket([5, 6, 7])); + _controller.close(); + + var result = await packetStream.toList(); + expect(result.length, 2); + expect(result.first, [1, 2, 3, 4]); + expect(result.last, [5, 6, 7]); + }); + }); + + group('BytesBuilder:', () { + KafkaBytesBuilder _builder; + + setUp(() { + _builder = new KafkaBytesBuilder(); + }); + + test('it adds Int8 values', () { + _builder.addInt8(35); + expect(_builder.length, equals(1)); + List result = _builder.toBytes(); + expect(result, hasLength(equals(1))); + expect(result[0], equals(35)); + }); + + test('it adds Int16 values', () { + _builder.addInt16(341); + expect(_builder.length, equals(2)); + List result = _builder.toBytes(); + expect(result, hasLength(equals(2))); + expect(result, equals([1, 85])); + }); + + test('it adds Int32 values', () { + _builder.addInt32(1635765); + expect(_builder.length, equals(4)); + var result = _builder.toBytes(); + expect(result, hasLength(equals(4))); + expect(result, equals([0, 24, 245, 181])); + }); + + test('it adds string values', () { + _builder.addString('dart-kafka'); + var result = _builder.toBytes(); + expect(result, hasLength(equals(12))); // 2 bytes = size, 10 bytes = value + var encodedString = result.getRange(2, 12).toList(); + var value = utf8.decode(encodedString); + expect(value, equals('dart-kafka')); + }); + + test('it adds array values of Int8', () { + _builder.addInt8Array([34, 45, 12]); + var result = _builder.toBytes(); + expect(result, hasLength(equals(7))); // 4 bytes = size, 3 bytes = data + }); + + test('it adds array values of Int16', () { + _builder.addInt16Array([234, 523, 332]); + var result = _builder.toBytes(); + expect(result, hasLength(equals(10))); // 4 bytes = size, 6 bytes = data + }); + + test('it adds array of Int32', () { + _builder.addInt32Array([234, 523, 332]); + var result = _builder.toBytes(); + expect(result, hasLength(equals(16))); // 4 bytes = size, 12 bytes = data + }); + + test('it adds array values of Int64', () { + _builder.addInt64Array([234, 523, 332]); + var result = _builder.toBytes(); + expect(result, hasLength(equals(28))); // 4 bytes = size, 24 bytes = data + }); + + test('it adds array values of bytes', () { + _builder.addBytesArray([ + [123], + [32] + ]); + var result = _builder.toBytes(); + expect(result, hasLength(equals(14))); // 4 + 4 + 1 + 4 + 1 + }); + + test('it supports null for bytes type', () { + _builder.addBytes(null); + var result = _builder.toBytes(); + expect(result, hasLength(4)); + expect(result, equals([255, 255, 255, 255])); + }); + }); + + group('BytesReader:', () { + KafkaBytesReader _reader; + List _data; + + setUp(() { + var builder = new KafkaBytesBuilder(); + builder + ..addInt8(53) + ..addInt16(3541) + ..addInt32(162534612) + ..addString('dart-kafka') + ..addBytes([12, 43, 83]) + ..addStringArray(['one', 'two']); + _data = builder.takeBytes(); + _reader = new KafkaBytesReader.fromBytes(_data); + }); + + test('it indicates end of buffer', () { + var builder = new KafkaBytesBuilder(); + builder.addInt8(53); + _reader = new KafkaBytesReader.fromBytes(builder.takeBytes()); + expect(_reader.length, equals(1)); + expect(_reader.isEOF, isFalse); + expect(_reader.isNotEOF, isTrue); + _reader.readInt8(); + expect(_reader.isEOF, isTrue); + expect(_reader.isNotEOF, isFalse); + }); + + test('it reads all Kafka types', () { + expect(_reader.readInt8(), equals(53)); + expect(_reader.readInt16(), equals(3541)); + expect(_reader.readInt32(), equals(162534612)); + expect(_reader.readString(), equals('dart-kafka')); + expect(_reader.readBytes(), equals([12, 43, 83])); + expect(_reader.readStringArray(), equals(['one', 'two'])); + }); + + test('it supports null for bytes type', () { + var builder = new KafkaBytesBuilder(); + builder.addBytes(null); + var reader = new KafkaBytesReader.fromBytes(builder.takeBytes()); + expect(reader.readBytes(), equals(null)); + }); + }); +} + +List _createPacket(List payload) { + ByteData bdata = new ByteData(4); + bdata.setInt32(0, payload.length); + return bdata.buffer.asInt8List().toList()..addAll(payload); +} diff --git a/test/list_offset_api_test.dart b/test/list_offset_api_test.dart new file mode 100644 index 0000000..0b34029 --- /dev/null +++ b/test/list_offset_api_test.dart @@ -0,0 +1,46 @@ +import 'package:kafka/kafka.dart'; +import 'package:test/test.dart'; + +void main() { + group('OffsetApi:', () { + String topic = 'dartKafkaTest'; + int partition; + Broker broker; + Session session = new Session(['127.0.0.1:9092']); + int _offset; + + setUp(() async { + var meta = await session.metadata.fetchTopics([topic]); + var p = meta[topic].partitions[0]; + partition = p.id; + var leaderId = p.leader; + broker = meta.brokers[leaderId]; + + var producer = new Producer( + new StringSerializer(), + new StringSerializer(), + new ProducerConfig(bootstrapServers: ['127.0.0.1:9092'])); + var rec = new ProducerRecord(topic, partition, 'key', 'value'); + producer.add(rec); + var result = await rec.result; + _offset = result.offset; + await producer.close(); + }); + + tearDownAll(() async { + await session.close(); + }); + + test('it fetches offset info', () async { + var request = + new ListOffsetRequest({new TopicPartition(topic, partition): -1}); + ListOffsetResponse response = + await session.send(request, broker.host, broker.port); + + expect(response.offsets, hasLength(1)); + var offset = response.offsets.first; + expect(offset.error, equals(0)); + expect(offset.offset, equals(_offset + 1)); + }); + }); +} diff --git a/test/messages_test.dart b/test/messages_test.dart new file mode 100644 index 0000000..5aa973b --- /dev/null +++ b/test/messages_test.dart @@ -0,0 +1,19 @@ +import 'package:kafka/kafka.dart'; +import 'package:test/test.dart'; + +void main() { + group('MessageAttributes:', () { + test('get compression from byte', () { + expect(Compression.none, new MessageAttributes.fromByte(0).compression); + expect(Compression.gzip, new MessageAttributes.fromByte(1).compression); + expect(Compression.snappy, new MessageAttributes.fromByte(2).compression); + }); + + test('get timestamp type from byte', () { + expect(TimestampType.createTime, + new MessageAttributes.fromByte(0).timestampType); + expect(TimestampType.logAppendTime, + new MessageAttributes.fromByte(8).timestampType); + }); + }); +} diff --git a/test/metadata_api_test.dart b/test/metadata_api_test.dart new file mode 100644 index 0000000..0d0df13 --- /dev/null +++ b/test/metadata_api_test.dart @@ -0,0 +1,27 @@ +import 'package:kafka/kafka.dart'; +import 'package:test/test.dart'; + +void main() { + group('Metadata API: ', () { + Session session = new Session(['127.0.0.1:9092']); + + tearDownAll(() async { + await session.close(); + }); + + test('we can send metadata requests to Kafka broker', () async { + var request = new MetadataRequest(); + var response = await session.send(request, '127.0.0.1', 9092); + expect(response, isA()); + expect(response.brokers, hasLength(2)); + expect(response.topics, isA()); + }); + + test('metadata response throws server error if present', () { + var metadata = new Topic(Errors.InvalidTopic, 'test', new Partitions([])); + expect(() { + new MetadataResponse([], new Topics([metadata], new Brokers([]))); + }, throwsA(isA())); + }); + }); +} diff --git a/test/metadata_test.dart b/test/metadata_test.dart new file mode 100644 index 0000000..2c22c56 --- /dev/null +++ b/test/metadata_test.dart @@ -0,0 +1,64 @@ +import 'package:kafka/kafka.dart'; +import 'package:test/test.dart'; + +void main() { + group('Metadata:', () { + Session session = new Session(['127.0.0.1:9092']); + Metadata metadata = session.metadata; + + tearDownAll(() async { + await session.close(); + }); + + test('it can fetch specific topic metadata', () async { + var topics = await metadata.fetchTopics(['testTopic']); + expect(topics, isA()); + expect(topics, hasLength(1)); + expect(topics['testTopic'], isNotNull); + expect(topics['testTopic'].toString(), + contains('Topic{testTopic, error: 0;')); + }); + + test('it can list existing topics', () async { + var topics = await metadata.listTopics(); + expect(topics, isList); + expect(topics, isNotEmpty); + }); + + test('it can list Kafka brokers within cluster', () async { + var brokers = await metadata.listBrokers(); + expect(brokers, isList); + expect(brokers, hasLength(2)); + }); + + test('it can fetch group coordinator', () async { + var group = + 'testGroup' + (new DateTime.now()).millisecondsSinceEpoch.toString(); + var broker = await metadata.fetchGroupCoordinator(group); + expect(broker, isA()); + expect(broker.id, isNotNull); + expect(broker.host, isNotNull); + expect(broker.port, isNotNull); + }); + + // test('it can fetch topic metadata', () async { + // var response = await _session.getMetadata([_topicName].toSet()); + // expect(response, isA()); + // expect(response.brokers, isNotEmpty); + // var topic = response.getTopicMetadata(_topicName); + // expect(topic, isA()); + // response = await _session.getMetadata([_topicName].toSet()); + // var newTopic = response.getTopicMetadata(_topicName); + // expect(newTopic, same(topic)); + // }); + // + // test('it fetches topic metadata for auto-created topics', () async { + // var date = new DateTime.now().millisecondsSinceEpoch; + // var topicName = 'testTopic-${date}'; + // var response = await _session.getMetadata([topicName].toSet()); + // var topic = response.getTopicMetadata(topicName); + // expect(topic.errorCode, equals(KafkaServerError.NoError_)); + // expect(topic.partitions, isNotEmpty); + // }); + }); +} diff --git a/test/offset_commit_test.dart b/test/offset_commit_test.dart new file mode 100644 index 0000000..a661729 --- /dev/null +++ b/test/offset_commit_test.dart @@ -0,0 +1,60 @@ +// TODO: move to consumer_offset_api_test.dart +import 'package:kafka/kafka.dart'; +import 'package:test/test.dart'; + +void main() { + group('OffsetCommitApi:', () { + String _topic = 'dartKafkaTest'; + Session session = new Session(['127.0.0.1:9092']); + Broker coordinator; + int _offset; + String testGroup; + + tearDownAll(() async { + await session.close(); + }); + + setUp(() async { + var producer = new Producer( + new StringSerializer(), + new StringSerializer(), + new ProducerConfig(bootstrapServers: ['127.0.0.1:9092'])); + var rec = new ProducerRecord(_topic, 0, 'a', 'b'); + producer.add(rec); + var result = await rec.result; + + _offset = result.offset; + var date = new DateTime.now(); + testGroup = 'group:' + date.millisecondsSinceEpoch.toString(); + coordinator = await session.metadata.fetchGroupCoordinator(testGroup); + await producer.close(); + }); + + tearDownAll(() async { + await session.close(); + }); + + test('it commits consumer offsets', () async { + var offsets = [new ConsumerOffset(_topic, 0, _offset, 'helloworld')]; + + OffsetCommitRequest request = + new OffsetCommitRequest(testGroup, offsets, -1, '', -1); + + OffsetCommitResponse response = + await session.send(request, coordinator.host, coordinator.port); + expect(response.results, hasLength(equals(1))); + expect(response.results.first.topic, equals(_topic)); + expect(response.results.first.error, equals(0)); + + var fetch = + new OffsetFetchRequest(testGroup, [new TopicPartition(_topic, 0)]); + + OffsetFetchResponse fetchResponse = + await session.send(fetch, coordinator.host, coordinator.port); + var offset = fetchResponse.offsets.first; + expect(offset.error, equals(0)); + expect(offset.offset, equals(_offset)); + expect(offset.metadata, equals('helloworld')); + }); + }); +} diff --git a/test/partition_assignor_test.dart b/test/partition_assignor_test.dart new file mode 100644 index 0000000..e9e0cf2 --- /dev/null +++ b/test/partition_assignor_test.dart @@ -0,0 +1,39 @@ +import 'package:kafka/kafka.dart'; +import 'package:test/test.dart'; + +void main() { + group('RoundRobinPartitionAssignor:', () { + test('it assignes partitions on one topic', () async { + var assignor = new PartitionAssignor.forStrategy('roundrobin'); + var partitionsPerTopic = {"foo": 5}; + var memberSubscriptions = { + "m1": ["foo"].toSet(), + "m2": ["foo"].toSet() + }; + + var assignments = + assignor.assign(partitionsPerTopic, memberSubscriptions); + + expect(assignments['m1'], hasLength(3)); + expect(assignments['m1'], contains(new TopicPartition('foo', 0))); + expect(assignments['m1'], contains(new TopicPartition('foo', 2))); + expect(assignments['m1'], contains(new TopicPartition('foo', 4))); + expect(assignments['m2'], hasLength(2)); + expect(assignments['m2'], contains(new TopicPartition('foo', 1))); + expect(assignments['m2'], contains(new TopicPartition('foo', 3))); + }); + + test('it validates member subscriptions are identical', () { + var assignor = new PartitionAssignor.forStrategy('roundrobin'); + var partitionsPerTopic = {"foo": 5}; + var memberSubscriptions = { + "m1": ["foo"].toSet(), + "m2": ["foo", "bar"].toSet() + }; + + expect(() { + assignor.assign(partitionsPerTopic, memberSubscriptions); + }, throwsStateError); + }); + }); +} diff --git a/test/produce_api_test.dart b/test/produce_api_test.dart new file mode 100644 index 0000000..d271d49 --- /dev/null +++ b/test/produce_api_test.dart @@ -0,0 +1,56 @@ +import 'package:kafka/kafka.dart'; +import 'package:test/test.dart'; + +void main() { + group('Produce API: ', () { + String _topic = 'dartKafkaTest' + + (new DateTime.now()).millisecondsSinceEpoch.toString(); + Broker broker; + Session session = new Session(['127.0.0.1:9092']); + int partition; + + setUp(() async { + var data = await session.metadata.fetchTopics([_topic]); + partition = data[_topic].partitions[0].id; + var leaderId = data[_topic].partitions[0].leader; + broker = data.brokers[leaderId]; + }); + + tearDownAll(() async { + await session.close(); + }); + + test('it publishes messages to Kafka topic', () async { + var req = new ProduceRequest(1, 1000, { + _topic: { + partition: [new Message('hello world'.codeUnits)] + } + }); + + var res = await session.send(req, broker.host, broker.port); + var p = new TopicPartition(_topic, partition); + expect(res.results, hasLength(1)); + expect(res.results[p].topic, equals(_topic)); + expect(res.results[p].error, equals(0)); + expect(res.results[p].offset, greaterThanOrEqualTo(0)); + }); + + // test('it publishes GZip encoded messages to Kafka topic', () async { + // var request = new ProduceRequest(1, 1000, [ + // new ProduceEnvelope( + // _topicName, + // 0, + // [ + // new Message('hello world'.codeUnits), + // new Message('peace and love'.codeUnits) + // ], + // compression: KafkaCompression.gzip) + // ]); + // ProduceResponse response = await _session.send(_broker, request); + // expect(response.results, hasLength(1)); + // expect(response.results.first.topicName, equals(_topicName)); + // expect(response.results.first.errorCode, equals(0)); + // expect(response.results.first.offset, greaterThanOrEqualTo(0)); + // }); + }); +} diff --git a/test/producer_test.dart b/test/producer_test.dart index 96881c2..c19bf64 100644 --- a/test/producer_test.dart +++ b/test/producer_test.dart @@ -1,34 +1,36 @@ -library kafka.test.producer; - import 'package:test/test.dart'; import 'package:kafka/kafka.dart'; -import 'setup.dart'; -main() { - group('Producer:', () { - KafkaSession _session; - String _topicName = 'dartKafkaTest'; +void main() { + group('KProducer:', () { + var producer = new Producer( + new StringSerializer(), + new StringSerializer(), + new ProducerConfig(bootstrapServers: ['127.0.0.1:9092'])); - setUp(() async { - var host = await getDefaultHost(); - _session = new KafkaSession([new ContactPoint(host, 9092)]); + tearDownAll(() async { + await producer.close(); }); - tearDown(() async { - await _session.close(); + test('it can produce messages to Kafka', () async { + var rec = new ProducerRecord('testProduce', 0, 'key', 'value'); + producer.add(rec); + var result = await rec.result; + expect(result, isA()); + expect(result.topicPartition, new TopicPartition('testProduce', 0)); + expect(result.offset, greaterThanOrEqualTo(0)); }); - test('it can produce messages to multiple brokers', () async { - var producer = new Producer(_session, 1, 100); - var result = await producer.produce([ - new ProduceEnvelope(_topicName, 0, [new Message('test1'.codeUnits)]), - new ProduceEnvelope(_topicName, 1, [new Message('test2'.codeUnits)]), - new ProduceEnvelope(_topicName, 2, [new Message('test3'.codeUnits)]), - ]); - expect(result.hasErrors, isFalse); - expect(result.offsets[_topicName][0], greaterThanOrEqualTo(0)); - expect(result.offsets[_topicName][1], greaterThanOrEqualTo(0)); - expect(result.offsets[_topicName][2], greaterThanOrEqualTo(0)); - }); + // test('it can produce messages to multiple brokers', () async { + // var producer = new Producer(_session, 1, 100); + // var result = await producer.produce([ + // new ProduceEnvelope(_topicName, 0, [new Message('test1'.codeUnits)]), + // new ProduceEnvelope(_topicName, 1, [new Message('test2'.codeUnits)]), + // new ProduceEnvelope(_topicName, 2, [new Message('test3'.codeUnits)]), + // ]); + // expect(result.offsets[_topicName][0], greaterThanOrEqualTo(0)); + // expect(result.offsets[_topicName][1], greaterThanOrEqualTo(0)); + // expect(result.offsets[_topicName][2], greaterThanOrEqualTo(0)); + // }); }); } diff --git a/test/protocol/bytes_builder_test.dart b/test/protocol/bytes_builder_test.dart deleted file mode 100644 index c27deb8..0000000 --- a/test/protocol/bytes_builder_test.dart +++ /dev/null @@ -1,92 +0,0 @@ -library kafka.protocol.test.bytes_builder; - -import 'dart:async'; -import 'dart:convert'; -import 'package:test/test.dart'; -import 'package:kafka/protocol.dart'; - -void main() { - group('BytesBuilder:', () { - KafkaBytesBuilder _builder; - - setUp(() { - _builder = new KafkaBytesBuilder(); - }); - - test('it adds Int8 values', () { - _builder.addInt8(35); - expect(_builder.length, equals(1)); - List result = _builder.toBytes(); - expect(result, hasLength(equals(1))); - expect(result[0], equals(35)); - }); - - test('it adds Int16 values', () { - _builder.addInt16(341); - expect(_builder.length, equals(2)); - List result = _builder.toBytes(); - expect(result, hasLength(equals(2))); - expect(result, equals([1, 85])); - }); - - test('it adds Int32 values', () { - _builder.addInt32(1635765); - expect(_builder.length, equals(4)); - var result = _builder.toBytes(); - expect(result, hasLength(equals(4))); - expect(result, equals([0, 24, 245, 181])); - }); - - test('it adds string values', () { - _builder.addString('dart-kafka'); - var result = _builder.toBytes(); - expect(result, hasLength(equals(12))); // 2 bytes = size, 10 bytes = value - var encodedString = result.getRange(2, 12).toList(); - var value = UTF8.decode(encodedString); - expect(value, equals('dart-kafka')); - }); - - test('it adds array values of Int8', () { - _builder.addArray([34, 45, 12], KafkaType.int8); - var result = _builder.toBytes(); - expect(result, hasLength(equals(7))); // 4 bytes = size, 3 bytes = values - }); - - test('it adds array values of Int16', () { - _builder.addArray([234, 523, 332], KafkaType.int16); - var result = _builder.toBytes(); - expect(result, hasLength(equals(10))); // 4 bytes = size, 6 bytes = values - }); - - test('it adds array values of Int64', () { - _builder.addArray([234, 523, 332], KafkaType.int64); - var result = _builder.toBytes(); - expect( - result, hasLength(equals(28))); // 4 bytes = size, 24 bytes = values - }); - - test('it adds array values of bytes', () { - _builder.addArray([ - [123], - [32] - ], KafkaType.bytes); - var result = _builder.toBytes(); - expect(result, hasLength(equals(14))); // 4 + 4 + 1 + 4 + 1 - }); - - test('it does not support objects in array values', () { - expect( - new Future(() { - _builder.addArray(['foo'], KafkaType.object); - }), - throwsStateError); - }); - - test('it supports null for bytes type', () { - _builder.addBytes(null); - var result = _builder.toBytes(); - expect(result, hasLength(4)); - expect(result, equals([255, 255, 255, 255])); - }); - }); -} diff --git a/test/protocol/bytes_reader_test.dart b/test/protocol/bytes_reader_test.dart deleted file mode 100644 index 29f009f..0000000 --- a/test/protocol/bytes_reader_test.dart +++ /dev/null @@ -1,52 +0,0 @@ -library kafka.protocol.test.bytes_reader; - -import 'package:test/test.dart'; -import 'package:kafka/protocol.dart'; - -void main() { - group('BytesReader:', () { - KafkaBytesReader _reader; - List _data; - - setUp(() { - var builder = new KafkaBytesBuilder(); - builder - ..addInt8(53) - ..addInt16(3541) - ..addInt32(162534612) - ..addString('dart-kafka') - ..addBytes([12, 43, 83]) - ..addArray(['one', 'two'], KafkaType.string); - _data = builder.takeBytes(); - _reader = new KafkaBytesReader.fromBytes(_data); - }); - - test('it indicates end of buffer', () { - var builder = new KafkaBytesBuilder(); - builder.addInt8(53); - _reader = new KafkaBytesReader.fromBytes(builder.takeBytes()); - expect(_reader.length, equals(1)); - expect(_reader.isEOF, isFalse); - expect(_reader.isNotEOF, isTrue); - _reader.readInt8(); - expect(_reader.isEOF, isTrue); - expect(_reader.isNotEOF, isFalse); - }); - - test('it reads all Kafka types', () { - expect(_reader.readInt8(), equals(53)); - expect(_reader.readInt16(), equals(3541)); - expect(_reader.readInt32(), equals(162534612)); - expect(_reader.readString(), equals('dart-kafka')); - expect(_reader.readBytes(), equals([12, 43, 83])); - expect(_reader.readArray(KafkaType.string), equals(['one', 'two'])); - }); - - test('it supports null for bytes type', () { - var builder = new KafkaBytesBuilder(); - builder.addBytes(null); - var reader = new KafkaBytesReader.fromBytes(builder.takeBytes()); - expect(reader.readBytes(), equals(null)); - }); - }); -} diff --git a/test/protocol/fetch_test.dart b/test/protocol/fetch_test.dart deleted file mode 100644 index c2a5f5a..0000000 --- a/test/protocol/fetch_test.dart +++ /dev/null @@ -1,82 +0,0 @@ -library kafka.test.api.fetch; - -import 'package:test/test.dart'; -import 'package:kafka/kafka.dart'; -import 'package:kafka/protocol.dart'; -import '../setup.dart'; - -void main() { - group('FetchApi:', () { - String _topicName = 'dartKafkaTest'; - Broker _host; - KafkaSession _session; - FetchRequest _request; - String _message; - int _offset; - - setUp(() async { - var ip = await getDefaultHost(); - _session = new KafkaSession([new ContactPoint(ip, 9092)]); - var metadata = await _session.getMetadata([_topicName].toSet()); - var leaderId = - metadata.getTopicMetadata(_topicName).getPartition(0).leader; - _host = metadata.getBroker(leaderId); - }); - - tearDown(() async { - await _session.close(); - }); - - test('it fetches messages from Kafka topic', () async { - var now = new DateTime.now(); - _message = 'test:' + now.toIso8601String(); - ProduceRequest produce = new ProduceRequest(1, 1000, [ - new ProduceEnvelope(_topicName, 0, [new Message(_message.codeUnits)]) - ]); - - ProduceResponse produceResponse = await _session.send(_host, produce); - _offset = produceResponse.results.first.offset; - _request = new FetchRequest(100, 1); - _request.add(_topicName, 0, _offset); - FetchResponse response = await _session.send(_host, _request); - - expect(response.results, hasLength(1)); - expect(response.results.first.messageSet, - hasLength(greaterThanOrEqualTo(1))); - var value = response.results.first.messageSet.messages[_offset].value; - var text = new String.fromCharCodes(value); - expect(text, equals(_message)); - }); - - test('it fetches GZip encoded messages from Kafka topic', () async { - var now = new DateTime.now(); - _message = 'test:' + now.toIso8601String(); - ProduceRequest produce = new ProduceRequest(1, 1000, [ - new ProduceEnvelope( - _topicName, - 0, - [ - new Message('hello world'.codeUnits), - new Message('peace and love'.codeUnits) - ], - compression: KafkaCompression.gzip) - ]); - - ProduceResponse produceResponse = await _session.send(_host, produce); - _offset = produceResponse.results.first.offset; - _request = new FetchRequest(100, 1); - _request.add(_topicName, 0, _offset); - FetchResponse response = await _session.send(_host, _request); - - expect(response.results, hasLength(1)); - expect(response.results.first.messageSet, hasLength(equals(2))); - var value = response.results.first.messageSet.messages[_offset].value; - var text = new String.fromCharCodes(value); - expect(text, equals('hello world')); - - value = response.results.first.messageSet.messages[_offset + 1].value; - text = new String.fromCharCodes(value); - expect(text, equals('peace and love')); - }); - }); -} diff --git a/test/protocol/offset_commit_test.dart b/test/protocol/offset_commit_test.dart deleted file mode 100644 index 7a7cbaf..0000000 --- a/test/protocol/offset_commit_test.dart +++ /dev/null @@ -1,66 +0,0 @@ -library kafka.test.api.offset_commit; - -import 'package:test/test.dart'; -import 'package:kafka/kafka.dart'; -import 'package:kafka/protocol.dart'; -import '../setup.dart'; - -void main() { - group('OffsetCommitApi:', () { - String _topicName = 'dartKafkaTest'; - KafkaSession _session; - Broker _host; - Broker _coordinator; - int _offset; - String _testGroup; - - setUp(() async { - var ip = await getDefaultHost(); - _session = new KafkaSession([new ContactPoint(ip, 9092)]); - var meta = await _session.getMetadata([_topicName].toSet()); - var leaderId = meta.getTopicMetadata(_topicName).getPartition(0).leader; - _host = meta.getBroker(leaderId); - - var now = new DateTime.now(); - var message = 'test:' + now.toIso8601String(); - ProduceRequest produce = new ProduceRequest(1, 1000, [ - new ProduceEnvelope(_topicName, 0, [new Message(message.codeUnits)]) - ]); - ProduceResponse response = await _session.send(_host, produce); - _offset = response.results.first.offset; - - _testGroup = 'group:' + now.millisecondsSinceEpoch.toString(); - var metadata = await _session.getConsumerMetadata(_testGroup); - _coordinator = metadata.coordinator; - }); - - tearDown(() async { - await _session.close(); - }); - - test('it commits consumer offsets', () async { - var offsets = [ - new ConsumerOffset('dartKafkaTest', 0, _offset, 'helloworld') - ]; - - var request = new OffsetCommitRequest(_testGroup, offsets, -1, '', -1); - - OffsetCommitResponse response = - await _session.send(_coordinator, request); - expect(response.offsets, hasLength(equals(1))); - expect(response.offsets.first.topicName, equals('dartKafkaTest')); - expect(response.offsets.first.errorCode, equals(0)); - - var fetch = new OffsetFetchRequest(_testGroup, { - _topicName: new Set.from([0]) - }); - - OffsetFetchResponse fetchResponse = - await _session.send(_coordinator, fetch); - var offset = fetchResponse.offsets.first; - expect(offset.errorCode, equals(0)); - expect(offset.offset, equals(_offset)); - expect(offset.metadata, equals('helloworld')); - }); - }); -} diff --git a/test/protocol/offset_fetch_test.dart b/test/protocol/offset_fetch_test.dart deleted file mode 100644 index dff3945..0000000 --- a/test/protocol/offset_fetch_test.dart +++ /dev/null @@ -1,40 +0,0 @@ -library kafka.test.api.offset_fetch; - -import 'package:test/test.dart'; -import 'package:kafka/kafka.dart'; -import 'package:kafka/protocol.dart'; -import '../setup.dart'; - -void main() { - group('OffsetFetchApi:', () { - KafkaSession _session; - OffsetFetchRequest _request; - Broker _coordinator; - String _testGroup; - - setUp(() async { - var ip = await getDefaultHost(); - _session = new KafkaSession([new ContactPoint(ip, 9092)]); - var now = new DateTime.now(); - _testGroup = 'group:' + now.millisecondsSinceEpoch.toString(); - var metadata = await _session.getConsumerMetadata(_testGroup); - _coordinator = metadata.coordinator; - _request = new OffsetFetchRequest(_testGroup, { - 'dartKafkaTest': new Set.from([0]) - }); - }); - - tearDown(() async { - await _session.close(); - }); - - test('it fetches consumer offsets', () async { - OffsetFetchResponse response = - await _session.send(_coordinator, _request); - expect(response.offsets, hasLength(equals(1))); - expect(response.offsets.first.topicName, equals('dartKafkaTest')); - expect(response.offsets.first.partitionId, equals(0)); - expect(response.offsets.first.errorCode, equals(0)); - }); - }); -} diff --git a/test/protocol/offset_test.dart b/test/protocol/offset_test.dart deleted file mode 100644 index e1d99bd..0000000 --- a/test/protocol/offset_test.dart +++ /dev/null @@ -1,50 +0,0 @@ -library kafka.test.api.offset; - -import 'package:test/test.dart'; -import 'package:kafka/kafka.dart'; -import 'package:kafka/protocol.dart'; -import '../setup.dart'; - -void main() { - group('OffsetApi:', () { - String _topicName = 'dartKafkaTest'; - Broker _broker; - KafkaSession _session; - OffsetRequest _request; - int _offset; - - setUp(() async { - var ip = await getDefaultHost(); - _session = new KafkaSession([new ContactPoint(ip, 9092)]); - var metadata = await _session.getMetadata([_topicName].toSet()); - var leaderId = - metadata.getTopicMetadata(_topicName).getPartition(0).leader; - _broker = metadata.getBroker(leaderId); - - var now = new DateTime.now(); - var _message = 'test:' + now.toIso8601String(); - ProduceRequest produce = new ProduceRequest(1, 1000, [ - new ProduceEnvelope(_topicName, 0, [new Message(_message.codeUnits)]) - ]); - - ProduceResponse response = await _session.send(_broker, produce); - _offset = response.results.first.offset; - _request = new OffsetRequest(leaderId); - }); - - tearDown(() async { - await _session.close(); - }); - - test('it fetches offset info', () async { - _request.addTopicPartition(_topicName, 0, -1, 1); - OffsetResponse response = await _session.send(_broker, _request); - - expect(response.offsets, hasLength(1)); - var offset = response.offsets.first; - expect(offset.errorCode, equals(0)); - expect(offset.offsets, hasLength(1)); - expect(offset.offsets.first, equals(_offset + 1)); - }); - }); -} diff --git a/test/protocol/produce_test.dart b/test/protocol/produce_test.dart deleted file mode 100644 index f956132..0000000 --- a/test/protocol/produce_test.dart +++ /dev/null @@ -1,57 +0,0 @@ -library kafka.test.api.produce; - -import 'package:test/test.dart'; -import 'package:kafka/kafka.dart'; -import 'package:kafka/protocol.dart'; -import '../setup.dart'; - -void main() { - group('ProduceApi:', () { - String _topicName = 'dartKafkaTest'; - Broker _broker; - KafkaSession _session; - - setUp(() async { - var ip = await getDefaultHost(); - _session = new KafkaSession([new ContactPoint(ip, 9092)]); - var metadata = await _session.getMetadata([_topicName].toSet()); - var leaderId = - metadata.getTopicMetadata(_topicName).getPartition(0).leader; - _broker = metadata.getBroker(leaderId); - }); - - tearDown(() async { - await _session.close(); - }); - - test('it publishes messages to Kafka topic', () async { - var request = new ProduceRequest(1, 1000, [ - new ProduceEnvelope( - _topicName, 0, [new Message('hello world'.codeUnits)]) - ]); - ProduceResponse response = await _session.send(_broker, request); - expect(response.results, hasLength(1)); - expect(response.results.first.topicName, equals(_topicName)); - expect(response.results.first.errorCode, equals(0)); - expect(response.results.first.offset, greaterThanOrEqualTo(0)); - }); - - test('it publishes GZip encoded messages to Kafka topic', () async { - var request = new ProduceRequest(1, 1000, [ - new ProduceEnvelope( - _topicName, - 0, - [ - new Message('hello world'.codeUnits), - new Message('peace and love'.codeUnits) - ], - compression: KafkaCompression.gzip) - ]); - ProduceResponse response = await _session.send(_broker, request); - expect(response.results, hasLength(1)); - expect(response.results.first.topicName, equals(_topicName)); - expect(response.results.first.errorCode, equals(0)); - expect(response.results.first.offset, greaterThanOrEqualTo(0)); - }); - }); -} diff --git a/test/session_test.dart b/test/session_test.dart deleted file mode 100644 index 464ba4d..0000000 --- a/test/session_test.dart +++ /dev/null @@ -1,65 +0,0 @@ -library kafka.test.session; - -import 'package:test/test.dart'; -import 'package:kafka/kafka.dart'; -import 'setup.dart'; - -void main() { - group('Session:', () { - KafkaSession _session; - String _topicName = 'dartKafkaTest'; - - setUp(() async { - var host = await getDefaultHost(); - _session = new KafkaSession([new ContactPoint(host, 9092)]); - }); - - tearDown(() async { - await _session.close(); - }); - - test('it can list existing topics', () async { - var topics = await _session.listTopics(); - expect(topics, new isInstanceOf()); - expect(topics, isNotEmpty); - expect(topics, contains(_topicName)); - }); - - test('it can fetch topic metadata', () async { - var response = await _session.getMetadata([_topicName].toSet()); - expect(response, new isInstanceOf()); - expect(response.brokers, isNotEmpty); - var topic = response.getTopicMetadata(_topicName); - expect(topic, new isInstanceOf()); - response = await _session.getMetadata([_topicName].toSet()); - var newTopic = response.getTopicMetadata(_topicName); - expect(newTopic, same(topic)); - }); - - test('it invalidates topic metadata', () async { - var response = await _session.getMetadata([_topicName].toSet()); - var topic = response.getTopicMetadata(_topicName); - response = await _session.getMetadata([_topicName].toSet(), - invalidateCache: true); - var newTopic = response.getTopicMetadata(_topicName); - expect(newTopic, isNot(same(topic))); - }); - - test('it fetches topic metadata for auto-created topics', () async { - var date = new DateTime.now().millisecondsSinceEpoch; - var topicName = 'testTopic-${date}'; - var response = await _session.getMetadata([topicName].toSet()); - var topic = response.getTopicMetadata(topicName); - expect(topic.errorCode, equals(KafkaServerError.NoError)); - expect(topic.partitions, isNotEmpty); - }); - - test('it can fetch consumer metadata', () async { - var response = await _session.getConsumerMetadata('testGroup'); - expect(response.errorCode, equals(0)); - expect(response.coordinatorId, isNotNull); - expect(response.coordinatorHost, isNotNull); - expect(response.coordinatorPort, isNotNull); - }); - }); -} diff --git a/test/setup.dart b/test/setup.dart index 6f92ac6..51e71f7 100644 --- a/test/setup.dart +++ b/test/setup.dart @@ -1,21 +1,11 @@ -import 'dart:io'; import 'dart:async'; import 'package:logging/logging.dart'; -/// Returns default host's IP address depending on current environment. -/// -/// For running tests locally on developer machine we assume you're using -/// Docker Toolbox and OS X (sorry). The IP of `default` docker-machine will -/// be used. Future getDefaultHost() async { - if (Platform.environment.containsKey('TRAVIS')) { - return '127.0.0.1'; - } else { - var res = await Process.run('docker-machine', ['ip', 'default']); - return res.stdout.toString().trim(); - } + return '127.0.0.1'; } -void enableLogs() { +void enableLogs({Level level: Level.INFO}) { + Logger.root.level = level; Logger.root.onRecord.listen(print); } diff --git a/test/testing_test.dart b/test/testing_test.dart new file mode 100644 index 0000000..cacdd75 --- /dev/null +++ b/test/testing_test.dart @@ -0,0 +1,90 @@ +void main() { + // group('MockKafkaSession: ', () { + // test('it returns cluster and topic metadata', () async { + // var session = new MockKafkaSession(); + // var meta = await session.getMetadata(['test'].toSet()); + // expect(meta.brokers, hasLength(1)); + // expect(meta.getBroker(1), new isInstanceOf()); + // expect(meta.getBroker(1).host, '127.0.0.1'); + // expect(meta.getBroker(1).port, 9092); + // expect(meta.topics, hasLength(1)); + // expect(meta.getTopicMetadata('test'), new isInstanceOf()); + // expect(meta.getTopicMetadata('test').partitions, hasLength(1)); + // }); + + // test('it returns consumer metadata', () async { + // var session = new MockKafkaSession(); + // var meta = await session.getConsumerMetadata('myGroup'); + // expect(meta, new isInstanceOf()); + // expect(meta.errorCode, 0); + // expect(meta.coordinatorId, 1); + // expect(meta.coordinatorHost, '127.0.0.1'); + // expect(meta.coordinatorPort, 9092); + // }); + // }); + + // group('Producer Mock: ', () { + // test('it produces messages to MockSession', () async { + // var session = new MockKafkaSession(); + // var producer = new Producer(session, -1, 1000); + // var message = 'helloworld'; + // var messages = [ + // new ProduceEnvelope('test', 0, [new Message(message.codeUnits)]) + // ]; + // var result = await producer.produce(messages); + // expect(result, new isInstanceOf()); + // expect(result.offsets, { + // "test": {0: 0} + // }); + + // result = await producer.produce(messages); + // expect(result, new isInstanceOf()); + // expect(result.offsets, { + // "test": {0: 1} + // }); + // }); + // }); + + // group('ConsumerGroup Mock: ', () { + // test('it joins a group', () async { + // var session = new MockKafkaSession(); + // var group = new ConsumerGroup(session, 'myGroup'); + // var result = await group.join(10000, '', 'consumer', [ + // new GroupProtocol.roundrobin(0, ['test'].toSet()) + // ]); + // expect(result, new isInstanceOf()); + // }); + // }); + // + // group('HighLevelConsumer Mock: ', () { + // KafkaSession session; + // ConsumerGroup group; + // HighLevelConsumer consumer; + // + // setUp(() async { + // var now = new DateTime.now().millisecondsSinceEpoch; + // var groupName = 'group-' + now.toString(); + // var topic = 'test-topic-' + now.toString(); + // session = new MockKafkaSession(); + // group = new ConsumerGroup(session, groupName); + // consumer = new HighLevelConsumer(session, [topic].toSet(), group); + // + // var producer = new Producer(session, -1, 1000); + // var message = 'helloworld'; + // var messages = [ + // new ProduceEnvelope(topic, 0, [new Message(message.codeUnits)]) + // ]; + // await producer.produce(messages); + // }); + // + // test('it consumes messages', () async { + // var message; + // await for (var env in consumer.stream) { + // message = new String.fromCharCodes(env.message.value); + // env.commit(''); + // break; + // } + // expect(message, 'helloworld'); + // }); + // }); +} diff --git a/test/util/crc32_test.dart b/test/util/crc32_test.dart index cfa0274..2462d71 100644 --- a/test/util/crc32_test.dart +++ b/test/util/crc32_test.dart @@ -1,7 +1,5 @@ -library kafka.protocol.crc32.test; - import 'package:test/test.dart'; -import 'package:kafka/protocol.dart'; +import 'package:kafka/src/util/crc32.dart'; void main() { group('Crc32:', () { @@ -37,5 +35,7 @@ Map, int> _dataProvider() { /// Test cases generated using: http://www.lammertbies.nl/comm/info/crc-calculation.html Map, int> _stringDataProvider() { - return {'Lammert'.codeUnits: 0x71FC2734,}; + return { + 'Lammert'.codeUnits: 0x71FC2734, + }; } diff --git a/test/versions_api_test.dart b/test/versions_api_test.dart new file mode 100644 index 0000000..63791fd --- /dev/null +++ b/test/versions_api_test.dart @@ -0,0 +1,20 @@ +import 'package:test/test.dart'; +import 'package:kafka/kafka.dart'; + +void main() { + group('Versions API', () { + Session session = new Session(['127.0.0.1:9092']); + + tearDownAll(() async { + await session.close(); + }); + + test('can obtain supported api versions from Kafka cluster', () async { + var request = new ApiVersionsRequest(); + var response = await session.send(request, '127.0.0.1', 9092); + expect(response, isA()); + expect(response.error, 0); + expect(response.versions, isNotEmpty); + }); + }); +} diff --git a/test/versions_test.dart b/test/versions_test.dart new file mode 100644 index 0000000..0c53ac5 --- /dev/null +++ b/test/versions_test.dart @@ -0,0 +1,29 @@ +import 'package:test/test.dart'; +import 'package:kafka/src/versions.dart'; + +void main() { + group('Versions', () { + test('fail if there is no overlap with server', () async { + var server = [new ApiVersion(0, 3, 5)]; + var client = [new ApiVersion(0, 2, 2)]; + expect(overlap(0, 0, 1, 1), isFalse); + expect(() { + resolveApiVersions(server, client); + }, throwsUnsupportedError); + }); + + test('pick max client version if server max is higher', () { + var server = [new ApiVersion(0, 0, 5)]; + var client = [new ApiVersion(0, 0, 2)]; + var result = resolveApiVersions(server, client); + expect(result[0], 2); + }); + + test('pick max server version if client max is higher', () { + var server = [new ApiVersion(0, 0, 1)]; + var client = [new ApiVersion(0, 0, 3)]; + var result = resolveApiVersions(server, client); + expect(result[0], 1); + }); + }); +} diff --git a/tool/kafka/Dockerfile b/tool/kafka/Dockerfile new file mode 100644 index 0000000..06befbf --- /dev/null +++ b/tool/kafka/Dockerfile @@ -0,0 +1,21 @@ +FROM larryaasen/kafka:latest + +USER root + +ADD config /kafka/config +ADD start.sh /start.sh + +RUN mkdir /data1 /data2 + +RUN chown -R kafka:kafka /kafka /data1 /data2 /logs +RUN chown -R kafka:kafka /start.sh +RUN chmod 755 /start.sh + +USER kafka +ENV PATH /kafka/bin:$PATH +WORKDIR /kafka + +EXPOSE 9092 9093 +VOLUME [ "/data1", "/data2", "/logs" ] + +CMD ["/start.sh"] diff --git a/tool/kafka/config/server.properties.template b/tool/kafka/config/server.properties.template new file mode 100644 index 0000000..169cc69 --- /dev/null +++ b/tool/kafka/config/server.properties.template @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# see kafka.server.KafkaConfig for additional details and defaults + +############################# Server Basics ############################# + +# The id of the broker. This must be set to a unique integer for each broker. +broker.id={{KAFKA_BROKER_ID}} +auto.leader.rebalance.enable=true + +# Replication +auto.create.topics.enable=true +default.replication.factor=1 + +# Hostname the broker will advertise to consumers. If not set, kafka will use the value returned +# from InetAddress.getLocalHost(). If there are multiple interfaces getLocalHost +# may not be what you want. +advertised.host.name={{KAFKA_ADVERTISED_HOST_NAME}} + +# Enable topic deletion +delete.topic.enable={{KAFKA_DELETE_TOPIC_ENABLE}} + +# The offsets.topic.replication.factor broker config is now enforced upon auto topic creation. +# Internal auto topic creation will fail with a GROUP_COORDINATOR_NOT_AVAILABLE error until the +# cluster size meets this replication factor requirement. +offsets.topic.replication.factor=1 + +############################# Socket Server Settings ############################# + +# The port the socket server listens on +port={{KAFKA_PORT}} +advertised.port={{KAFKA_ADVERTISED_PORT}} + +############################# Log Basics ############################# + +# The directory under which to store log files +log.dir={{KAFKA_DATA_DIR}} +log.dirs={{KAFKA_DATA_DIRS}} + +# The number of logical partitions per topic per server. More partitions allow greater parallelism +# for consumption, but also mean more files. +num.partitions=3 + +############################# Log Retention Policy ############################# + +# The following configurations control the disposal of log segments. The policy can +# be set to delete segments after a period of time, or after a given size has accumulated. +# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens +# from the end of the log. + +# The minimum age of a log file to be eligible for deletion +log.retention.hours=1 + +############################# Zookeeper ######################################## + +# Zk connection string (see zk docs for details). +# This is a comma separated host:port pairs, each corresponding to a zk +# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". +# You can also append an optional chroot string to the urls to specify the +# root directory for all kafka znodes. +zookeeper.connect={{ZOOKEEPER_CONNECTION_STRING}}{{ZOOKEEPER_CHROOT}} +zookeeper.connection.timeout.ms=10000 +controlled.shutdown.enable=true +zookeeper.session.timeout.ms=10000 + +# vim:set filetype=jproperties + +############################# Message Format ################################### + +message.format.version=0.10.1 +log.message.timestamp.type=LogAppendTime diff --git a/tool/kafka/start.sh b/tool/kafka/start.sh new file mode 100644 index 0000000..ec984a7 --- /dev/null +++ b/tool/kafka/start.sh @@ -0,0 +1,67 @@ +#!/bin/bash -x + +# If a ZooKeeper container is linked with the alias `zookeeper`, use it. +# You MUST set ZOOKEEPER_IP in env otherwise. +[ -n "$ZOOKEEPER_PORT_2181_TCP_ADDR" ] && ZOOKEEPER_IP=$ZOOKEEPER_PORT_2181_TCP_ADDR +[ -n "$ZOOKEEPER_PORT_2181_TCP_PORT" ] && ZOOKEEPER_PORT=$ZOOKEEPER_PORT_2181_TCP_PORT + +IP=$(grep ${HOSTNAME} /etc/hosts | awk '{print $1}') + +# Concatenate the IP:PORT for ZooKeeper to allow setting a full connection +# string with multiple ZooKeeper hosts +[ -z "$ZOOKEEPER_CONNECTION_STRING" ] && ZOOKEEPER_CONNECTION_STRING="${ZOOKEEPER_IP}:${ZOOKEEPER_PORT:-2181}" + +cat /kafka/config/server.properties.template | sed \ + -e "s|{{ZOOKEEPER_CONNECTION_STRING}}|${ZOOKEEPER_CONNECTION_STRING}|g" \ + -e "s|{{ZOOKEEPER_CHROOT}}|${ZOOKEEPER_CHROOT:-}|g" \ + -e "s|{{KAFKA_BROKER_ID}}|${KAFKA_BROKER_ID:-1}|g" \ + -e "s|{{KAFKA_ADVERTISED_HOST_NAME}}|${KAFKA_ADVERTISED_HOST_NAME:-$IP}|g" \ + -e "s|{{KAFKA_PORT}}|${KAFKA_PORT:-9092}|g" \ + -e "s|{{KAFKA_ADVERTISED_PORT}}|${KAFKA_ADVERTISED_PORT:-9092}|g" \ + -e "s|{{KAFKA_DELETE_TOPIC_ENABLE}}|${KAFKA_DELETE_TOPIC_ENABLE:-false}|g" \ + -e "s|{{KAFKA_DATA_DIR}}|${KAFKA_DATA_DIR:-/data1}|g" \ + -e "s|{{KAFKA_DATA_DIRS}}|${KAFKA_DATA_DIRS:-/data1}|g" \ + > /kafka/config/server1.properties + +cat /kafka/config/server.properties.template | sed \ + -e "s|{{ZOOKEEPER_CONNECTION_STRING}}|${ZOOKEEPER_CONNECTION_STRING}|g" \ + -e "s|{{ZOOKEEPER_CHROOT}}|${ZOOKEEPER_CHROOT:-}|g" \ + -e "s|{{KAFKA_BROKER_ID}}|${KAFKA_BROKER_ID:-2}|g" \ + -e "s|{{KAFKA_ADVERTISED_HOST_NAME}}|${KAFKA_ADVERTISED_HOST_NAME:-$IP}|g" \ + -e "s|{{KAFKA_PORT}}|${KAFKA_PORT:-9093}|g" \ + -e "s|{{KAFKA_ADVERTISED_PORT}}|${KAFKA_ADVERTISED_PORT:-9093}|g" \ + -e "s|{{KAFKA_DELETE_TOPIC_ENABLE}}|${KAFKA_DELETE_TOPIC_ENABLE:-false}|g" \ + -e "s|{{KAFKA_DATA_DIR}}|${KAFKA_DATA_DIR:-/data2}|g" \ + -e "s|{{KAFKA_DATA_DIRS}}|${KAFKA_DATA_DIRS:-/data2}|g" \ + > /kafka/config/server2.properties + +# Kafka's built-in start scripts set the first three system properties here, but +# we add two more to make remote JMX easier/possible to access in a Docker +# environment: +# +# 1. RMI port - pinning this makes the JVM use a stable one instead of +# selecting random high ports each time it starts up. +# 2. RMI hostname - normally set automatically by heuristics that may have +# hard-to-predict results across environments. +# +# These allow saner configuration for firewalls, EC2 security groups, Docker +# hosts running in a VM with Docker Machine, etc. See: +# +# https://issues.apache.org/jira/browse/CASSANDRA-7087 +# if [ -z $KAFKA_JMX_OPTS ]; then +# KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true" +# KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.authenticate=false" +# KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.ssl=false" +# KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT" +# KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=${JAVA_RMI_SERVER_HOSTNAME:-$KAFKA_ADVERTISED_HOST_NAME} " +# export KAFKA_JMX_OPTS +# fi + +export JMX_PORT=7203 +echo "Starting kafka1" +nohup bash -c "/kafka/bin/kafka-server-start.sh /kafka/config/server1.properties 2>&1 &" +sleep 2 + +export JMX_PORT=7204 +echo "Starting kafka2" +exec /kafka/bin/kafka-server-start.sh /kafka/config/server2.properties diff --git a/tool/rebuild.sh b/tool/rebuild.sh new file mode 100755 index 0000000..050f47a --- /dev/null +++ b/tool/rebuild.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +docker kill kafka zookeeper +docker rm kafka zookeeper + +docker run -d --name zookeeper --publish 2181:2181 zookeeper:3.6.0 + +docker build -t kafka tool/kafka/ + +ZK_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' zookeeper) + +docker run -d --name kafka --publish 9092:9092 --publish 9093:9093 \ + --env KAFKA_ADVERTISED_HOST_NAME=127.0.0.1 \ + --env ZOOKEEPER_IP=$ZK_IP \ + kafka + +sleep 5 \ No newline at end of file