message: Remove function retrieve_metadate_with_version

diff:
@@ -100,9 +100,6 @@ defmodule KafkaEx.Protocol.Metadata do
def create_request(correlation_id, client_id, ""), do: create_request(correlation_id, client_id, "", @default_api_version)
def create_request(correlation_id, client_id, topics) when is_list(topics), do: create_request(correlation_id, client_id, topics, @default_api_version)
- def create_request(correlation_id, client_id, "", nil), do: create_request(correlation_id, client_id, "", @default_api_version)
- def create_request(correlation_id, client_id, topics, nil) when is_list(topics), do: create_request(correlation_id, client_id, topics, @default_api_version)
-
def create_request(correlation_id, client_id, "", api_version) do
topic_count = if 0 == api_version, do: 0, else: -1
KafkaEx.Protocol.create_request(:metadata, correlation_id, client_id, api_version) <> << topic_count :: 32-signed >>

diff:
@@ -383,10 +383,10 @@ defmodule KafkaEx.Server do
{:noreply, update_metadata(state)}
end
- def update_metadata(state), do: update_metadata(state, nil)
+ def update_metadata(state), do: update_metadata(state, 0)
def update_metadata(state, api_version) do
- {correlation_id, metadata} = retrieve_metadata_with_version(state.brokers, state.correlation_id, config_sync_timeout(), api_version)
+ {correlation_id, metadata} = retrieve_metadata(state.brokers, state.correlation_id, config_sync_timeout(), [], api_version)
metadata_brokers = metadata.brokers |> Enum.map(&(%{&1 | is_controller: &1.node_id == metadata.controller_id}))
brokers = state.brokers
|> remove_stale_brokers(metadata_brokers)
@@ -395,24 +395,20 @@ defmodule KafkaEx.Server do
end
# credo:disable-for-next-line Credo.Check.Refactor.FunctionArity
- def retrieve_metadata(brokers, correlation_id, sync_timeout, topic \\ []), do: retrieve_metadata(brokers, correlation_id, sync_timeout, topic, @retry_count, 0)
+ def retrieve_metadata(brokers, correlation_id, sync_timeout, topic \\ [], api_version \\ 0) do
+ retrieve_metadata(brokers, correlation_id, sync_timeout, topic, @retry_count, 0, api_version)
+ end
- # credo:disable-for-next-line Credo.Check.Refactor.FunctionArity
- def retrieve_metadata_with_version(brokers, correlation_id, sync_timeout, api_version, topic \\ []), do: retrieve_metadata(brokers, correlation_id, sync_timeout, topic, @retry_count, api_version, 0)
+ def retrieve_metadata(brokers, correlation_id, sync_timeout, topic, retry, error_code, api_version \\ 0)
# credo:disable-for-next-line Credo.Check.Refactor.FunctionArity
- def retrieve_metadata(_, correlation_id, _sync_timeout, topic, 0, error_code) do
+ def retrieve_metadata(_, correlation_id, _sync_timeout, topic, 0, error_code, api_version) do
Logger.log(:error, "Metadata request for topic #{inspect topic} failed with error_code #{inspect error_code}")
{correlation_id, %Metadata.Response{}}
end
# credo:disable-for-next-line Credo.Check.Refactor.FunctionArity
- def retrieve_metadata(brokers, correlation_id, sync_timeout, topic, retry, error_code) do
- retrieve_metadata(brokers, correlation_id, sync_timeout, topic, retry, nil, error_code)
- end
-
- # credo:disable-for-next-line Credo.Check.Refactor.FunctionArity
- def retrieve_metadata(brokers, correlation_id, sync_timeout, topic, retry, api_version, _error_code) do
+ def retrieve_metadata(brokers, correlation_id, sync_timeout, topic, retry, _error_code, api_version) do
metadata_request = Metadata.create_request(correlation_id, @client_id, topic, api_version)
data = first_broker_response(metadata_request, brokers, sync_timeout)
if data do
@@ -422,7 +418,7 @@ defmodule KafkaEx.Server do
nil -> {correlation_id + 1, response}
topic_metadata ->
:timer.sleep(300)
- retrieve_metadata(brokers, correlation_id + 1, sync_timeout, topic, retry - 1, topic_metadata.error_code)
+ retrieve_metadata(brokers, correlation_id + 1, sync_timeout, topic, retry - 1, topic_metadata.error_code, api_version)
end
else
message = "Unable to fetch metadata from any brokers. Timeout is #{sync_timeout}."

diff:
@@ -65,7 +65,7 @@ defmodule KafkaEx.Server0P10P1 do
brokers = Enum.map(uris, fn({host, port}) -> %Broker{host: host, port: port, socket: NetworkClient.create_socket(host, port, ssl_options, use_ssl)} end)
- {correlation_id, metadata} = retrieve_metadata_with_version(brokers, 0, config_sync_timeout(), @metadata_api_version)
+ {correlation_id, metadata} = retrieve_metadata(brokers, 0, config_sync_timeout(), [], @metadata_api_version)
state = %State{metadata: metadata, brokers: brokers, correlation_id: correlation_id, consumer_group: consumer_group, metadata_update_interval: metadata_update_interval, consumer_group_update_interval: consumer_group_update_interval, worker_name: name, ssl_options: ssl_options, use_ssl: use_ssl}
# Get the initial "real" broker list and start a regular refresh cycle.
state = update_metadata(state, @metadata_api_version)
@@ -85,7 +85,7 @@ defmodule KafkaEx.Server0P10P1 do
end
def kafka_server_metadata(topic, state) do
- {correlation_id, metadata} = retrieve_metadata_with_version(state.brokers, state.correlation_id, config_sync_timeout(), @metadata_api_version, topic)
+ {correlation_id, metadata} = retrieve_metadata(state.brokers, state.correlation_id, config_sync_timeout(), topic, @metadata_api_version)
updated_state = %{state | metadata: metadata, correlation_id: correlation_id}
{:reply, metadata, updated_state}
end


message: Consolidate `create_request` signatures in metadata.ex

change_type: MODIFY

diff:
@@ -97,8 +97,7 @@ defmodule KafkaEx.Protocol.Metadata do
}
end
- def create_request(correlation_id, client_id, ""), do: create_request(correlation_id, client_id, "", @default_api_version)
- def create_request(correlation_id, client_id, topics) when is_list(topics), do: create_request(correlation_id, client_id, topics, @default_api_version)
+ def create_request(correlation_id, client_id, topics, api_version \\ @default_api_version)
def create_request(correlation_id, client_id, "", api_version) do
topic_count = if 0 == api_version, do: 0, else: -1

license: MIT License


message: Change type of TopicError.error_code from integer to atom

diff:
@@ -106,7 +106,20 @@ defmodule KafkaEx.Protocol do
28 => :invalid_commit_offset,
29 => :topic_authorization_failed,
30 => :group_authorization_failed,
- 31 => :cluster_authorization_failed
+ 31 => :cluster_authorization_failed,
+ 32 => :invalid_timestamp,
+ 33 => :unsupported_sasl_mechanism,
+ 34 => :illegal_sasl_state,
+ 35 => :unsupported_version,
+ 36 => :topic_already_exists,
+ 37 => :invalid_partitions,
+ 38 => :invalid_replication_factor,
+ 39 => :invalid_replica_assignment,
+ 40 => :invalid_config,
+ 41 => :not_controller,
+ 42 => :invalid_request,
+ 43 => :unsupported_for_message_format,
+
}
@spec error(integer) :: atom | integer

diff:
@@ -52,7 +52,7 @@ defmodule KafkaEx.Protocol.CreateTopics do
defmodule TopicError do
defstruct topic_name: nil, error_code: nil
- @type t :: %TopicError{ topic_name: binary, error_code: integer }
+ @type t :: %TopicError{ topic_name: binary, error_code: atom }
end
defmodule Response do
@@ -137,7 +137,7 @@ defmodule KafkaEx.Protocol.CreateTopics do
@spec parse_topic_errors(integer, binary) :: [TopicError.t]
defp parse_topic_errors(topic_errors_count,
<< topic_name_size :: 16-signed, topic_name :: size(topic_name_size)-binary, error_code :: 16-signed, rest :: binary >>) do
- [%TopicError{topic_name: topic_name, error_code: error_code} | parse_topic_errors(topic_errors_count - 1, rest)]
+ [%TopicError{topic_name: topic_name, error_code: Protocol.error(error_code)} | parse_topic_errors(topic_errors_count - 1, rest)]
end
end

diff:
@@ -19,12 +19,10 @@ defmodule KafkaEx.Server0P10P1.Test do
]}
resp = KafkaEx.create_topics([request], timeout: 2000)
- # error = NONE
- assert {0, name} == parse_create_topic_resp(resp)
+ assert {:no_error, name} == parse_create_topic_resp(resp)
resp = KafkaEx.create_topics([request], timeout: 2000)
- # error = TOPIC_ALREADY_EXISTS
- assert {36, name} == parse_create_topic_resp(resp)
+ assert {:topic_already_exists, name} == parse_create_topic_resp(resp)
wait_for(fn ->
topics = KafkaEx.metadata.topic_metadatas |> Enum.map(&(&1.topic))


message: Implement ApiVersions message in protocol

diff:
@@ -529,15 +529,19 @@ defmodule KafkaEx do
def valid_consumer_group?(b) when is_binary(b), do: byte_size(b) > 0
def valid_consumer_group?(_), do: false
-
@doc """
+ Retrieve supported api versions for each api key.
+ """
+ @spec api_versions(Keyword.t) :: CreateTopicsResponse.t
+ def api_versions(opts \\ []) do
+ worker_name = Keyword.get(opts, :worker_name, Config.default_worker)
+ Server.call(worker_name, {:api_versions})
+ end
- ## Example
-
- ```elixir
- iex> KafkaEx.create_worker(:mt)
- ```
+ @doc """
+ Create topics. Must provide a list of CreateTopicsRequest, each containing
+ all the information needed for the creation of a new topic.
"""
@spec create_topics([CreateTopicsRequest.t], Keyword.t) :: CreateTopicsResponse.t
def create_topics(requests, opts \\ []) do

diff:
@@ -12,6 +12,7 @@ defmodule KafkaEx.Protocol do
@heartbeat_request 12
@leave_group_request 13
@sync_group_request 14
+ @api_versions_request 18
@create_topics_request 19
# DescribeConfigs 32
# AlterConfigs 33 Valid resource types are "Topic" and "Broker".
@@ -62,6 +63,10 @@ defmodule KafkaEx.Protocol do
@sync_group_request
end
+ defp api_key(:api_versions) do
+ @api_versions_request
+ end
+
defp api_key(:create_topics) do
@create_topics_request
end

change_type: ADD

diff:
+
+defmodule KafkaEx.Protocol.ApiVersions do
+ alias KafkaEx.Protocol
+ require Logger
+
+ # the ApiVersions message can also, itself, have different api versions
+ @default_this_api_version 0
+ @default_throttle_time 0
+
+ @moduledoc """
+ Implementation of the Kafka ApiVersions request and response APIs
+
+ See: https://kafka.apache.org/protocol.html#The_Messages_ApiVersions
+
+ Reference implementation in Java:
+ https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java
+ https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java
+
+ """
+
+ defmodule Response do
+ @moduledoc false
+ defstruct error_code: nil, api_versions: nil, throttle_time_ms: nil
+ @type t :: %Response{
+ error_code: atom,
+ api_versions: [ApiVersion],
+ throttle_time_ms: integer,
+ }
+ end
+
+ defmodule ApiVersion do
+ @moduledoc false
+ defstruct api_key: nil, min_version: nil, max_version: nil
+ @type t :: %ApiVersion{
+ api_key: integer,
+ min_version: integer,
+ max_version: integer,
+ }
+ end
+
+ @spec create_request(integer, binary, integer) :: binary
+ def create_request(correlation_id, client_id, this_api_version \\ @default_this_api_version)
+
+ def create_request(correlation_id, client_id, 1), do: create_request(correlation_id, client_id, 0)
+ def create_request(correlation_id, client_id, 2), do: create_request(correlation_id, client_id, 0)
+
+ def create_request(correlation_id, client_id, 0) do
+ Protocol.create_request(:api_versions, correlation_id, client_id)
+ end
+
+ @spec parse_response(binary, integer) :: Response.t
+ def parse_response(binary, this_api_version \\ @default_this_api_version)
+
+ def parse_response(<< _correlation_id :: 32-signed,
+ error_code :: 16-signed,
+ api_versions_count :: 32-signed,
+ rest :: binary
+ >>, this_api_version) do
+ Logger.debug("parse_response for api_versions #{this_api_version}")
+
+ %{ parse_rest_of_response(api_versions_count, rest, this_api_version) | error_code: Protocol.error(error_code) }
+ end
+
+ defp parse_rest_of_response(api_versions_count, data, this_api_version) do
+ {api_versions, remaining_data} = Protocol.Common.read_array(api_versions_count, data, &parse_one_api_version/1)
+
+ case {this_api_version, remaining_data} do
+ {0, ""} -> %Response{api_versions: api_versions, throttle_time_ms: @default_throttle_time}
+ {v, << throttle_time_ms :: 32-signed >>} when v in [1, 2] -> %Response{api_versions: api_versions, throttle_time_ms: throttle_time_ms}
+ end
+ end
+
+ defp parse_one_api_version(<< api_key :: 16-signed, min_version :: 16-signed, max_version :: 16-signed, rest :: binary >>) do
+ {%ApiVersion{ api_key: api_key, min_version: min_version, max_version: max_version }, rest}
+ end
+
+end

diff:
@@ -24,4 +24,15 @@ defmodule KafkaEx.Protocol.Common do
} | parse_topics(topics_size - 1, topics_data, mod)
]
end
+
+ def read_array(0, data_after_array, _read_one) do
+ {[], data_after_array}
+ end
+
+ def read_array(num_items, data, read_one) do
+ {item, rest} = read_one.(data)
+ {items, data_after_array} = read_array(num_items - 1, rest, read_one)
+ {[item | items], data_after_array}
+ end
+
end

diff:
@@ -4,6 +4,8 @@ defmodule KafkaEx.Protocol.CreateTopics do
@moduledoc """
Implementation of the Kafka CreateTopics request and response APIs
+
+ See: https://kafka.apache.org/protocol.html#The_Messages_CreateTopics
"""
# CreateTopics Request (Version: 0) => [create_topic_requests] timeout

diff:
@@ -260,6 +260,10 @@ defmodule KafkaEx.Server do
kafka_server_heartbeat(request, network_timeout, state)
end
+ def handle_call({:api_versions}, _from, state) do
+ kafka_api_versions(state)
+ end
+
def handle_call({:create_topics, requests, network_timeout}, _from, state) do
kafka_create_topics(requests, network_timeout, state)
end

diff:
@@ -4,6 +4,7 @@ defmodule KafkaEx.Server0P10P1 do
"""
use KafkaEx.Server
alias KafkaEx.Protocol.CreateTopics
+ alias KafkaEx.Protocol.ApiVersions
alias KafkaEx.Server0P8P2
alias KafkaEx.Server0P9P0
@@ -94,6 +95,15 @@ defmodule KafkaEx.Server0P10P1 do
{:noreply, update_metadata(state, @metadata_api_version)}
end
+ def kafka_api_versions(state) do
+ response = state.correlation_id
+ |> ApiVersions.create_request(@client_id)
+ |> first_broker_response(state)
+ |> ApiVersions.parse_response
+
+ {:reply, response, %{ state | correlation_id: state.correlation_id + 1 }}
+ end
+
def kafka_create_topics(requests, network_timeout, state) do
create_topics_request = %CreateTopics.Request{
create_topic_requests: requests,

diff:
@@ -63,6 +63,7 @@ defmodule KafkaEx.Server0P8P0 do
def kafka_server_leave_group(_, _, _state), do: raise "Leave Group is not supported in 0.8.0 version of Kafka"
def kafka_server_heartbeat(_, _, _state), do: raise "Heartbeat is not supported in 0.8.0 version of kafka"
def kafka_server_update_consumer_metadata(_state), do: raise "Consumer Group Metadata is not supported in 0.8.0 version of kafka"
+ def kafka_api_versions(_state), do: raise "ApiVersions is not supported in 0.8.0 version of kafka"
def kafka_create_topics(_, _, _state), do: raise "CreateTopic is not supported in 0.8.0 version of kafka"
defp fetch(request, state) do

diff:
@@ -134,11 +134,12 @@ defmodule KafkaEx.Server0P8P2 do
{:noreply, state}
end
- def kafka_server_join_group(_, _, _state), do: raise "Join Group is not supported in 0.8.0 version of kafka"
- def kafka_server_sync_group(_, _, _state), do: raise "Sync Group is not supported in 0.8.0 version of kafka"
- def kafka_server_leave_group(_, _, _state), do: raise "Leave Group is not supported in 0.8.0 version of Kafka"
- def kafka_server_heartbeat(_, _, _state), do: raise "Heartbeat is not supported in 0.8.0 version of kafka"
- def kafka_create_topics(_, _, _state), do: raise "CreateTopic is not supported in 0.8.0 version of kafka"
+ def kafka_server_join_group(_, _, _state), do: raise "Join Group is not supported in 0.8.2 version of kafka"
+ def kafka_server_sync_group(_, _, _state), do: raise "Sync Group is not supported in 0.8.2 version of kafka"
+ def kafka_server_leave_group(_, _, _state), do: raise "Leave Group is not supported in 0.8.2 version of Kafka"
+ def kafka_server_heartbeat(_, _, _state), do: raise "Heartbeat is not supported in 0.8.2 version of kafka"
+ def kafka_api_versions(_state), do: raise "ApiVersions is not supported in 0.8.2 version of kafka"
+ def kafka_create_topics(_, _, _state), do: raise "CreateTopic is not supported in 0.8.2 version of kafka"
defp update_consumer_metadata(state), do: update_consumer_metadata(state, @retry_count, 0)

diff:
@@ -45,6 +45,7 @@ defmodule KafkaEx.Server0P9P0 do
defdelegate kafka_server_offset_commit(offset_commit_request, state), to: Server0P8P2
defdelegate kafka_server_consumer_group_metadata(state), to: Server0P8P2
defdelegate kafka_server_update_consumer_metadata(state), to: Server0P8P2
+ defdelegate kafka_api_versions(state), to: Server0P8P2
defdelegate kafka_create_topics(requests, network_timeout, state),to: Server0P8P2
def kafka_server_init([args]) do

diff:
@@ -4,7 +4,7 @@ defmodule KafkaEx.Server0P10P1.Test do
@moduletag :server_0_p_10_p_1
- @tag :createtopic
+ @tag :create_topic
test "can create a topic" do
name = "create_topic_#{:rand.uniform(2000000)}"
@@ -40,4 +40,42 @@ defmodule KafkaEx.Server0P10P1.Test do
]} = response
{error_code, topic_name}
end
+
+ @tag :api_version
+ test "can retrieve api versions" do
+
+ # api_key, max_version, min_version
+ api_versions_kafka_0_10_1_0 = [
+ [0, 2, 0],
+ [1, 3, 0],
+ [2, 1, 0],
+ [3, 2, 0],
+ [4, 0, 0],
+ [5, 0, 0],
+ [6, 2, 0],
+ [7, 1, 1],
+ [8, 2, 0],
+ [9, 1, 0],
+ [10, 0, 0],
+ [11, 1, 0],
+ [12, 0, 0],
+ [13, 0, 0],
+ [14, 0, 0],
+ [15, 0, 0],
+ [16, 0, 0],
+ [17, 0, 0],
+ [18, 0, 0],
+ [19, 0, 0],
+ [20, 0, 0]
+ ]
+
+ response = KafkaEx.api_versions()
+ %KafkaEx.Protocol.ApiVersions.Response{
+ api_versions: api_versions,
+ error_code: :no_error,
+ throttle_time_ms: _
+ } = response
+
+ assert api_versions_kafka_0_10_1_0 == api_versions |> Enum.map(&([&1.api_key, &1.max_version, &1.min_version]))
+ end
end


message: Remove debug log, fix return type for `@spec api_versions(Keyword.t)`

diff:
@@ -24,6 +24,7 @@ defmodule KafkaEx do
alias KafkaEx.Protocol.SyncGroup.Response, as: SyncGroupResponse
alias KafkaEx.Protocol.CreateTopics.Request, as: CreateTopicsRequest
alias KafkaEx.Protocol.CreateTopics.Response, as: CreateTopicsResponse
+ alias KafkaEx.Protocol.ApiVersions.Response, as: ApiVersionsResponse
alias KafkaEx.Server
alias KafkaEx.Stream
@@ -532,7 +533,7 @@ defmodule KafkaEx do
@doc """
Retrieve supported api versions for each api key.
"""
- @spec api_versions(Keyword.t) :: CreateTopicsResponse.t
+ @spec api_versions(Keyword.t) :: ApiVersionsResponse.t
def api_versions(opts \\ []) do
worker_name = Keyword.get(opts, :worker_name, Config.default_worker)
Server.call(worker_name, {:api_versions})

diff:
@@ -56,8 +56,6 @@ defmodule KafkaEx.Protocol.ApiVersions do
api_versions_count :: 32-signed,
rest :: binary
>>, this_api_version) do
- Logger.debug("parse_response for api_versions #{this_api_version}")
-
%{ parse_rest_of_response(api_versions_count, rest, this_api_version) | error_code: Protocol.error(error_code) }
end


message:
Add callback definitions in `kafka_ex.server`
For `kafka_create_topics` and `kafka_api_versions`.

diff:
@@ -17,6 +17,7 @@ defmodule KafkaEx.Server do
alias KafkaEx.Protocol.Produce
alias KafkaEx.Protocol.Produce.Request, as: ProduceRequest
alias KafkaEx.Protocol.SyncGroup.Request, as: SyncGroupRequest
+ alias KafkaEx.Protocol.CreateTopics.Request, as: CreateTopicsRequest
alias KafkaEx.Socket
defmodule State do
@@ -157,6 +158,10 @@ defmodule KafkaEx.Server do
{:noreply, new_state, timeout | :hibernate} |
{:stop, reason, reply, new_state} |
{:stop, reason, new_state} when reply: term, new_state: term, reason: term
+ @callback kafka_create_topics(CreateTopicsRequest.t, network_timeout :: integer, state :: State.t) ::
+ {:reply, reply, new_state} when reply: term, new_state: term
+ @callback kafka_api_versions(state :: State.t) ::
+ {:reply, reply, new_state} when reply: term, new_state: term
@callback kafka_server_update_metadata(state :: State.t) ::
{:noreply, new_state} |
{:noreply, new_state, timeout | :hibernate} |
@@ -260,14 +265,14 @@ defmodule KafkaEx.Server do
kafka_server_heartbeat(request, network_timeout, state)
end
- def handle_call({:api_versions}, _from, state) do
- kafka_api_versions(state)
- end
-
def handle_call({:create_topics, requests, network_timeout}, _from, state) do
kafka_create_topics(requests, network_timeout, state)
end
+ def handle_call({:api_versions}, _from, state) do
+ kafka_api_versions(state)
+ end
+
def handle_info(:update_metadata, state) do
kafka_server_update_metadata(state)
end


message: Add module to determine API version to use

diff:
+defmodule KafkaEx.ApiVersions do
+
+ def api_versions_map(api_versions) do
+ api_versions
+ |> Enum.map(fn version -> {version.api_key, version} end)
+ |> Map.new
+ end
+
+ def find_api_version(api_versions_map, message_type, {min_implemented_version, max_implemented_version}) do
+ if api_versions_map == [:unsupported] do
+ {:ok, min_implemented_version}
+ else
+ case KafkaEx.Protocol.api_key(message_type) do
+ nil -> :unknown_message_for_client
+ api_key -> case api_versions_map[api_key] do
+ %{min_version: min} when min > max_implemented_version -> :no_version_supported
+ %{max_version: max} when max < min_implemented_version -> :no_version_supported
+ %{max_version: max} -> {:ok, Enum.min([max_implemented_version, max])}
+ _ -> :unknown_message_for_server
+ end
+ end
+ end
+
+ end
+end
\ No newline at end of file

diff:
defmodule KafkaEx.Protocol do
@moduledoc false
- @produce_request 0
- @fetch_request 1
- @offset_request 2
- @metadata_request 3
- @offset_commit_request 8
- @offset_fetch_request 9
- @consumer_metadata_request 10
- @join_group_request 11
- @heartbeat_request 12
- @leave_group_request 13
- @sync_group_request 14
- @api_versions_request 18
- @create_topics_request 19
+ @message_type_to_api_key %{
+ produce: 0,
+ fetch: 1,
+ offset: 2,
+ metadata: 3,
+ offset_commit: 8,
+ offset_fetch: 9,
+ consumer_metadata: 10,
+ join_group: 11,
+ heartbeat: 12,
+ leave_group: 13,
+ sync_group: 14,
+ api_versions: 18,
+ create_topics: 19,
+ }
+
# DescribeConfigs 32
# AlterConfigs 33 Valid resource types are "Topic" and "Broker".
- @api_version 0
-
- defp api_key(:produce) do
- @produce_request
- end
-
- defp api_key(:fetch) do
- @fetch_request
- end
-
- defp api_key(:offset) do
- @offset_request
- end
-
- defp api_key(:metadata) do
- @metadata_request
- end
-
- defp api_key(:offset_commit) do
- @offset_commit_request
- end
-
- defp api_key(:offset_fetch) do
- @offset_fetch_request
- end
-
- defp api_key(:consumer_metadata) do
- @consumer_metadata_request
- end
-
- defp api_key(:join_group) do
- @join_group_request
- end
-
- defp api_key(:heartbeat) do
- @heartbeat_request
- end
-
- defp api_key(:leave_group) do
- @leave_group_request
- end
-
- defp api_key(:sync_group) do
- @sync_group_request
- end
-
- defp api_key(:api_versions) do
- @api_versions_request
- end
-
- defp api_key(:create_topics) do
- @create_topics_request
- end
+ @default_api_version 0
- def create_request(type, correlation_id, client_id) do
- create_request(type, correlation_id, client_id, @api_version)
+ @spec api_key(atom) :: integer | nil
+ def api_key(type) do
+ Map.get(@message_type_to_api_key, type, nil)
end
- def create_request(type, correlation_id, client_id, api_version) do
+ def create_request(type, correlation_id, client_id, api_version \\ @default_api_version) do
<< api_key(type) :: 16, api_version :: 16, correlation_id :: 32,
byte_size(client_id) :: 16, client_id :: binary >>
end

diff:
+defmodule ApiVersionsTest do
+ use ExUnit.Case
+ @api_versions %{
+ 0 => %KafkaEx.Protocol.ApiVersions.ApiVersion {
+ api_key: 0,
+ min_version: 0,
+ max_version: 3,
+ },
+ 1 => %KafkaEx.Protocol.ApiVersions.ApiVersion {
+ api_key: 1,
+ min_version: 7,
+ max_version: 8,
+ },
+ 3 => %KafkaEx.Protocol.ApiVersions.ApiVersion {
+ api_key: 3,
+ min_version: 2,
+ max_version: 4,
+ }
+ }
+
+ test "can correctly determine the adequate api version when api versions is not support" do
+ assert {:ok, 1} == KafkaEx.ApiVersions.find_api_version([:unsupported], :metadata, {1, 3})
+ end
+
+ test "KafkaEx.ApiVersions can correctly determine the adequate api version when a match exists" do
+ assert {:ok, 3} == KafkaEx.ApiVersions.find_api_version(@api_versions, :metadata, {1, 3})
+ assert {:ok, 0} == KafkaEx.ApiVersions.find_api_version(@api_versions, :produce, {0, 0})
+ end
+
+ test "KafkaEx.ApiVersions replies an error when there's no version match" do
+ assert :no_version_supported == KafkaEx.ApiVersions.find_api_version(@api_versions, :fetch, {0, 6})
+ end
+
+ test "KafkaEx.ApiVersions replies an error when the api_key is unknown to the server" do
+ assert :unknown_message_for_server == KafkaEx.ApiVersions.find_api_version(@api_versions, :create_topics, {0, 1})
+ end
+
+ test "KafkaEx.ApiVersions replies an error when the api_key is unknown to the client" do
+ assert :unknown_message_for_client == KafkaEx.ApiVersions.find_api_version(@api_versions, :this_does_not_exist, {0, 1})
+ end
+end
+


message: Use api_versions to control the version of Medatadata being called.

diff:
@@ -357,7 +357,7 @@ The 0.9 client includes functionality that cannot be tested with older
clusters.
```
-mix test --include integration --include consumer_group --include server_0_p_10_p_1 --include server_0_p_9_p_0 --include server_0_p_8_p_0
+./all_tests.sh
```
##### Kafka >= 0.9.0

diff:
+#! /bin/sh
+
+# WARN: when changing something here, there should probably also be a change in scripts/ci_tests.sh
+
+mix test --include integration --include consumer_group --include server_0_p_10_and_later --include server_0_p_9_p_0 --include server_0_p_8_p_0

diff:
@@ -20,6 +20,5 @@ defmodule KafkaEx.ApiVersions do
end
end
end
-
end
end
\ No newline at end of file

diff:
@@ -81,7 +81,7 @@ defmodule KafkaEx.Config do
defp server("0.8.0"), do: KafkaEx.Server0P8P0
defp server("0.8.2"), do: KafkaEx.Server0P8P2
defp server("0.9.0"), do: KafkaEx.Server0P9P0
- defp server(_), do: KafkaEx.Server0P10P1
+ defp server(_), do: KafkaEx.Server0P10AndLater
# ssl_options should be an empty list by default if use_ssl is false

diff:
@@ -2,6 +2,7 @@ defmodule KafkaEx.Protocol.Metadata do
alias KafkaEx.Protocol
import KafkaEx.Protocol.Common
+ @supported_versions_range {0, 1}
@default_api_version 0
@moduledoc """
@@ -97,8 +98,19 @@ defmodule KafkaEx.Protocol.Metadata do
}
end
+ def api_version(api_versions) do
+ case KafkaEx.ApiVersions.find_api_version(api_versions, :metadata, @supported_versions_range) do
+ {:ok, version} -> version
+ _ -> @default_api_version
+ end
+ end
+
def create_request(correlation_id, client_id, topics, api_version \\ @default_api_version)
+ def create_request(correlation_id, client_id, nil, api_version) do
+ create_request(correlation_id, client_id, "", api_version)
+ end
+
def create_request(correlation_id, client_id, "", api_version) do
topic_count = if 0 == api_version, do: 0, else: -1
KafkaEx.Protocol.create_request(:metadata, correlation_id, client_id, api_version) <> << topic_count :: 32-signed >>

diff:
@@ -37,7 +37,8 @@ defmodule KafkaEx.Server do
consumer_group_update_interval: nil,
worker_name: KafkaEx.Server,
ssl_options: [],
- use_ssl: false
+ use_ssl: false,
+ api_versions: []
)
@type t :: %State{
@@ -51,6 +52,7 @@ defmodule KafkaEx.Server do
worker_name: atom,
ssl_options: KafkaEx.ssl_options,
use_ssl: boolean,
+ api_versions: [KafkaEx.Protocol.ApiVersions.ApiVersion],
}
@spec increment_correlation_id(t) :: t
@@ -313,8 +315,8 @@ defmodule KafkaEx.Server do
def kafka_server_produce_send_request(correlation_id, produce_request, produce_request_data, state) do
{broker, state, corr_id} = case MetadataResponse.broker_for_topic(state.metadata, state.brokers, produce_request.topic, produce_request.partition) do
nil ->
- {retrieved_corr_id, _} = retrieve_metadata(state.brokers, state.correlation_id, config_sync_timeout(), produce_request.topic)
- state = %{update_metadata(state) | correlation_id: retrieved_corr_id}
+ {retrieved_corr_id, _} = retrieve_metadata(state.brokers, state.correlation_id, config_sync_timeout(), produce_request.topic, state.api_versions)
+ state = update_metadata(%{state | correlation_id: retrieved_corr_id})
{
MetadataResponse.broker_for_topic(state.metadata, state.brokers, produce_request.topic, produce_request.partition),
state,
@@ -383,7 +385,7 @@ defmodule KafkaEx.Server do
end
def kafka_server_metadata(topic, state) do
- {correlation_id, metadata} = retrieve_metadata(state.brokers, state.correlation_id, config_sync_timeout(), topic)
+ {correlation_id, metadata} = retrieve_metadata(state.brokers, state.correlation_id, config_sync_timeout(), topic, state.api_versions)
updated_state = %{state | metadata: metadata, correlation_id: correlation_id}
{:reply, metadata, updated_state}
end
@@ -392,10 +394,8 @@ defmodule KafkaEx.Server do
{:noreply, update_metadata(state)}
end
- def update_metadata(state), do: update_metadata(state, 0)
-
- def update_metadata(state, api_version) do
- {correlation_id, metadata} = retrieve_metadata(state.brokers, state.correlation_id, config_sync_timeout(), [], api_version)
+ def update_metadata(state) do
+ {correlation_id, metadata} = retrieve_metadata(state.brokers, state.correlation_id, config_sync_timeout(), nil, state.api_versions)
metadata_brokers = metadata.brokers |> Enum.map(&(%{&1 | is_controller: &1.node_id == metadata.controller_id}))
brokers = state.brokers
|> remove_stale_brokers(metadata_brokers)
@@ -404,30 +404,27 @@ defmodule KafkaEx.Server do
end
# credo:disable-for-next-line Credo.Check.Refactor.FunctionArity
- def retrieve_metadata(brokers, correlation_id, sync_timeout, topic \\ [], api_version \\ 0) do
- retrieve_metadata(brokers, correlation_id, sync_timeout, topic, @retry_count, 0, api_version)
+ def retrieve_metadata(brokers, correlation_id, sync_timeout, topic \\ [], server_api_versions \\ [:unsupported]) do
+ retrieve_metadata(brokers, correlation_id, sync_timeout, topic, @retry_count, 0, server_api_versions)
end
- def retrieve_metadata(brokers, correlation_id, sync_timeout, topic, retry, error_code, api_version \\ 0)
-
# credo:disable-for-next-line Credo.Check.Refactor.FunctionArity
- def retrieve_metadata(_, correlation_id, _sync_timeout, topic, 0, error_code, api_version) do
- Logger.log(:error, "Metadata request for topic #{inspect topic} failed with error_code #{inspect error_code}")
- {correlation_id, %Metadata.Response{}}
+ def retrieve_metadata(brokers, correlation_id, sync_timeout, topic, retry, error_code, server_api_versions \\ [:unsupported]) do
+ api_version = Metadata.api_version(server_api_versions)
+ retrieve_metadata_with_version(brokers, correlation_id, sync_timeout, topic, retry, error_code, api_version)
end
# credo:disable-for-next-line Credo.Check.Refactor.FunctionArity
- def retrieve_metadata(brokers, correlation_id, sync_timeout, topic, retry, _error_code, api_version) do
+ def retrieve_metadata_with_version(brokers, correlation_id, sync_timeout, topic, retry, _error_code, api_version) do
metadata_request = Metadata.create_request(correlation_id, @client_id, topic, api_version)
data = first_broker_response(metadata_request, brokers, sync_timeout)
if data do
response = Metadata.parse_response(data, api_version)
-
case Enum.find(response.topic_metadatas, &(&1.error_code == :leader_not_available)) do
nil -> {correlation_id + 1, response}
topic_metadata ->
:timer.sleep(300)
- retrieve_metadata(brokers, correlation_id + 1, sync_timeout, topic, retry - 1, topic_metadata.error_code, api_version)
+ retrieve_metadata_with_version(brokers, correlation_id + 1, sync_timeout, topic, retry - 1, topic_metadata.error_code, api_version)
end
else
message = "Unable to fetch metadata from any brokers. Timeout is #{sync_timeout}."
@@ -437,6 +434,13 @@ defmodule KafkaEx.Server do
end
end
+ # credo:disable-for-next-line Credo.Check.Refactor.FunctionArity
+ def retrieve_metadata_with_version(_, correlation_id, _sync_timeout, topic, 0, error_code, server_api_versions) do
+ Logger.log(:error, "Metadata request for topic #{inspect topic} failed with error_code #{inspect error_code}")
+ {correlation_id, %Metadata.Response{}}
+ end
+
+
defoverridable [
kafka_server_produce: 2, kafka_server_offset: 4,
kafka_server_metadata: 2, kafka_server_update_metadata: 1,
@@ -470,7 +474,8 @@ defmodule KafkaEx.Server do
metadata_update_interval: metadata_update_interval,
ssl_options: ssl_options,
use_ssl: use_ssl,
- worker_name: name
+ worker_name: name,
+ api_versions: [:unsupported]
}
state = update_metadata(state)

change_type: RENAME

diff:
-defmodule KafkaEx.Server0P10P1 do
+defmodule KafkaEx.Server0P10AndLater do
@moduledoc """
Implements kafkaEx.Server behaviors for kafka 0.10.1 API.
"""
@@ -16,7 +16,6 @@ defmodule KafkaEx.Server0P10P1 do
require Logger
- @metadata_api_version 1
@consumer_group_update_interval 30_000
@@ -66,10 +65,26 @@ defmodule KafkaEx.Server0P10P1 do
brokers = Enum.map(uris, fn({host, port}) -> %Broker{host: host, port: port, socket: NetworkClient.create_socket(host, port, ssl_options, use_ssl)} end)
- {correlation_id, metadata} = retrieve_metadata(brokers, 0, config_sync_timeout(), [], @metadata_api_version)
- state = %State{metadata: metadata, brokers: brokers, correlation_id: correlation_id, consumer_group: consumer_group, metadata_update_interval: metadata_update_interval, consumer_group_update_interval: consumer_group_update_interval, worker_name: name, ssl_options: ssl_options, use_ssl: use_ssl}
+ { _, %KafkaEx.Protocol.ApiVersions.Response{ api_versions: api_versions, error_code: :no_error }, state } = kafka_api_versions(%State{brokers: brokers})
+ api_versions = KafkaEx.ApiVersions.api_versions_map(api_versions)
+
+ {correlation_id, metadata} = retrieve_metadata(brokers, state.correlation_id, config_sync_timeout(), [], api_versions)
+
+ state = %State{
+ metadata: metadata,
+ brokers: brokers,
+ correlation_id: correlation_id,
+ consumer_group: consumer_group,
+ metadata_update_interval: metadata_update_interval,
+ consumer_group_update_interval: consumer_group_update_interval,
+ worker_name: name,
+ ssl_options: ssl_options,
+ use_ssl: use_ssl,
+ api_versions: api_versions,
+ }
+
# Get the initial "real" broker list and start a regular refresh cycle.
- state = update_metadata(state, @metadata_api_version)
+ state = update_metadata(state)
{:ok, _} = :timer.send_interval(state.metadata_update_interval, :update_metadata)
state =
@@ -86,13 +101,13 @@ defmodule KafkaEx.Server0P10P1 do
end
def kafka_server_metadata(topic, state) do
- {correlation_id, metadata} = retrieve_metadata(state.brokers, state.correlation_id, config_sync_timeout(), topic, @metadata_api_version)
+ {correlation_id, metadata} = retrieve_metadata(state.brokers, state.correlation_id, config_sync_timeout(), topic, state.api_versions)
updated_state = %{state | metadata: metadata, correlation_id: correlation_id}
{:reply, metadata, updated_state}
end
def kafka_server_update_metadata(state) do
- {:noreply, update_metadata(state, @metadata_api_version)}
+ {:noreply, update_metadata(state)}
end
def kafka_api_versions(state) do

diff:
@@ -53,7 +53,16 @@ defmodule KafkaEx.Server0P8P2 do
brokers = Enum.map(uris, fn({host, port}) -> %Broker{host: host, port: port, socket: NetworkClient.create_socket(host, port)} end)
{correlation_id, metadata} = retrieve_metadata(brokers, 0, config_sync_timeout())
- state = %State{metadata: metadata, brokers: brokers, correlation_id: correlation_id, consumer_group: consumer_group, metadata_update_interval: metadata_update_interval, consumer_group_update_interval: consumer_group_update_interval, worker_name: name}
+ state = %State{
+ metadata: metadata,
+ brokers: brokers,
+ correlation_id: correlation_id,
+ consumer_group: consumer_group,
+ metadata_update_interval: metadata_update_interval,
+ consumer_group_update_interval: consumer_group_update_interval,
+ worker_name: name,
+ api_versions: [:unsupported]
+ }
# Get the initial "real" broker list and start a regular refresh cycle.
state = update_metadata(state)
{:ok, _} = :timer.send_interval(state.metadata_update_interval, :update_metadata)

diff:
@@ -69,7 +69,18 @@ defmodule KafkaEx.Server0P9P0 do
brokers = Enum.map(uris, fn({host, port}) -> %Broker{host: host, port: port, socket: NetworkClient.create_socket(host, port, ssl_options, use_ssl)} end)
{correlation_id, metadata} = retrieve_metadata(brokers, 0, config_sync_timeout())
- state = %State{metadata: metadata, brokers: brokers, correlation_id: correlation_id, consumer_group: consumer_group, metadata_update_interval: metadata_update_interval, consumer_group_update_interval: consumer_group_update_interval, worker_name: name, ssl_options: ssl_options, use_ssl: use_ssl}
+ state = %State{
+ metadata: metadata,
+ brokers: brokers,
+ correlation_id: correlation_id,
+ consumer_group: consumer_group,
+ metadata_update_interval: metadata_update_interval,
+ consumer_group_update_interval: consumer_group_update_interval,
+ worker_name: name,
+ ssl_options: ssl_options,
+ use_ssl: use_ssl,
+ api_versions: [:unsupported]
+ }
# Get the initial "real" broker list and start a regular refresh cycle.
state = update_metadata(state)
{:ok, _} = :timer.send_interval(state.metadata_update_interval, :update_metadata)

diff:
@@ -16,7 +16,7 @@ else
TEST_COMMAND=test
fi
-INCLUDED_TESTS="--include integration --include consumer_group --include server_0_p_10_p_1 --include server_0_p_9_p_0 --include server_0_p_8_p_0"
+INCLUDED_TESTS="--include integration --include consumer_group --include server_0_p_10_and_later --include server_0_p_9_p_0 --include server_0_p_8_p_0"
mix "$TEST_COMMAND" $INCLUDED_TESTS

diff:
+defmodule KafkaEx.Server0P10P1AndLater.Test do
+ use ExUnit.Case
+ import TestHelper
+
+ @moduletag :server_0_p_10_and_later
+
+ @tag :create_topic
+ test "can create a topic" do
+ name = "create_topic_#{:rand.uniform(2000000)}"
+
+ request = %{
+ topic: name,
+ num_partitions: 10,
+ replication_factor: 1,
+ replica_assignment: [],
+ config_entries: [
+ %{config_name: "cleanup.policy", config_value: "compact"},
+ %{config_name: "min.compaction.lag.ms", config_value: "0"}
+ ]}
+
+ resp = KafkaEx.create_topics([request], timeout: 2000)
+ assert {:no_error, name} == parse_create_topic_resp(resp)
+
+ resp = KafkaEx.create_topics([request], timeout: 2000)
+ assert {:topic_already_exists, name} == parse_create_topic_resp(resp)
+
+ wait_for(fn ->
+ topics = KafkaEx.metadata.topic_metadatas |> Enum.map(&(&1.topic))
+ assert Enum.member?(topics, name)
+ end)
+ end
+
+ def parse_create_topic_resp(response) do
+ %KafkaEx.Protocol.CreateTopics.Response{
+ topic_errors: [
+ %KafkaEx.Protocol.CreateTopics.TopicError{
+ error_code: error_code,
+ topic_name: topic_name
+ }
+ ]} = response
+ {error_code, topic_name}
+ end
+end

diff:
defmodule KafkaEx.Server0P10P1.Test do
use ExUnit.Case
- import TestHelper
+ @moduletag :server_0_p_10_and_later
@moduletag :server_0_p_10_p_1
- @tag :create_topic
- test "can create a topic" do
- name = "create_topic_#{:rand.uniform(2000000)}"
-
- request = %{
- topic: name,
- num_partitions: 10,
- replication_factor: 1,
- replica_assignment: [],
- config_entries: [
- %{config_name: "cleanup.policy", config_value: "compact"},
- %{config_name: "min.compaction.lag.ms", config_value: "0"}
- ]}
-
- resp = KafkaEx.create_topics([request], timeout: 2000)
- assert {:no_error, name} == parse_create_topic_resp(resp)
-
- resp = KafkaEx.create_topics([request], timeout: 2000)
- assert {:topic_already_exists, name} == parse_create_topic_resp(resp)
-
- wait_for(fn ->
- topics = KafkaEx.metadata.topic_metadatas |> Enum.map(&(&1.topic))
- assert Enum.member?(topics, name)
- end)
- end
-
- def parse_create_topic_resp(response) do
- %KafkaEx.Protocol.CreateTopics.Response{
- topic_errors: [
- %KafkaEx.Protocol.CreateTopics.TopicError{
- error_code: error_code,
- topic_name: topic_name
- }
- ]} = response
- {error_code, topic_name}
- end
-
+ # specific to this server version because we want to test that the api_versions list is exact
@tag :api_version
test "can retrieve api versions" do


message: Check api_version for create_topics to stay compatible 0.10.0.0

diff:
defmodule KafkaEx.Protocol.CreateTopics do
alias KafkaEx.Protocol
+ @supported_versions_range {0, 0}
+ @default_api_version 0
@moduledoc """
Implementation of the Kafka CreateTopics request and response APIs
@@ -63,8 +65,14 @@ defmodule KafkaEx.Protocol.CreateTopics do
@type t :: %Response{topic_errors: [TopicError]}
end
- @spec create_request(integer, binary, Request.t) :: binary
- def create_request(correlation_id, client_id, create_topics_request) do
+ def api_version(api_versions) do
+ KafkaEx.ApiVersions.find_api_version(api_versions, :create_topics, @supported_versions_range)
+ end
+
+ @spec create_request(integer, binary, Request.t, integer) :: binary
+ def create_request(correlation_id, client_id, create_topics_request, api_version)
+
+ def create_request(correlation_id, client_id, create_topics_request, 0) do
Protocol.create_request(:create_topics, correlation_id, client_id) <>
encode_topic_requests(create_topics_request.create_topic_requests) <>
<< create_topics_request.timeout :: 32-signed >>
@@ -129,8 +137,10 @@ defmodule KafkaEx.Protocol.CreateTopics do
end
end
- @spec parse_response(binary) :: [] | Response.t
- def parse_response(<< _correlation_id :: 32-signed, topic_errors_count :: 32-signed, topic_errors :: binary >>) do
+ @spec parse_response(binary, integer) :: [] | Response.t
+ def parse_response(message, api_version)
+
+ def parse_response(<< _correlation_id :: 32-signed, topic_errors_count :: 32-signed, topic_errors :: binary >>, 0) do
%Response{topic_errors: parse_topic_errors(topic_errors_count, topic_errors)}
end

diff:
@@ -101,6 +101,11 @@ defmodule KafkaEx.Protocol.Metadata do
def api_version(api_versions) do
case KafkaEx.ApiVersions.find_api_version(api_versions, :metadata, @supported_versions_range) do
{:ok, version} -> version
+ # those three should never happen since :metadata is part of the protocol since the beginning.
+ # they are left here as this will server as reference implementation
+ # :unknown_message_for_server ->
+ # :unknown_message_for_client ->
+ # :no_version_supported ->
_ -> @default_api_version
end
end

diff:
@@ -120,12 +120,19 @@ defmodule KafkaEx.Server0P10AndLater do
end
def kafka_create_topics(requests, network_timeout, state) do
+ api_version = case CreateTopics.api_version(state.api_versions) do
+ {:ok, api_version} -> api_version
+ _ -> raise "CreateTopic is not supported in this version of Kafka, or the versions supported by the client do not match the ones supported by the server."
+ end
+
+ IO.puts "API version for create_topics: #{api_version}"
+
create_topics_request = %CreateTopics.Request{
create_topic_requests: requests,
timeout: network_timeout
}
- mainRequest = CreateTopics.create_request(state.correlation_id, @client_id, create_topics_request)
+ mainRequest = CreateTopics.create_request(state.correlation_id, @client_id, create_topics_request, api_version)
broker = state.brokers |> Enum.find(&(&1.is_controller))
@@ -138,7 +145,7 @@ defmodule KafkaEx.Server0P10AndLater do
|> NetworkClient.send_sync_request(mainRequest, config_sync_timeout())
|> case do
{:error, reason} -> {:error, reason}
- response -> CreateTopics.parse_response(response)
+ response -> CreateTopics.parse_response(response, api_version)
end
{response, %{state | correlation_id: state.correlation_id + 1}}
end


message: Make api_versions compatible with Elixir 1.1

diff:
@@ -2,10 +2,12 @@ defmodule KafkaEx.ApiVersions do
def api_versions_map(api_versions) do
api_versions
- |> Enum.map(fn version -> {version.api_key, version} end)
- |> Map.new
+ |> Enum.reduce(%{}, fn version, version_map ->
+ version_map |> Map.put(version.api_key, version)
+ end)
end
+
def find_api_version(api_versions_map, message_type, {min_implemented_version, max_implemented_version}) do
if api_versions_map == [:unsupported] do
{:ok, min_implemented_version}

diff:
@@ -125,8 +125,6 @@ defmodule KafkaEx.Server0P10AndLater do
_ -> raise "CreateTopic is not supported in this version of Kafka, or the versions supported by the client do not match the ones supported by the server."
end
- IO.puts "API version for create_topics: #{api_version}"
-
create_topics_request = %CreateTopics.Request{
create_topic_requests: requests,
timeout: network_timeout


message: Run "mix format" on the project

diff:
@@ -250,10 +250,13 @@ defmodule KafkaEx.GenConsumer do
Any other return value will cause the `start_link/5` to return `{:error,
error}` and the process to exit.
"""
- @callback init(topic :: binary, partition :: non_neg_integer, extra_args :: map()) ::
+ @callback init(
+ topic :: binary,
+ partition :: non_neg_integer,
+ extra_args :: map()
+ ) ::
{:ok, state :: term}
-
@doc """
Invoked for each message set consumed from a Kafka topic partition.
@@ -367,7 +370,11 @@ defmodule KafkaEx.GenConsumer do
{:noreply, consumer_state}
end
- defoverridable init: 2, init: 3, handle_call: 3, handle_cast: 2, handle_info: 2
+ defoverridable init: 2,
+ init: 3,
+ handle_call: 3,
+ handle_cast: 2,
+ handle_info: 2
end
end
@@ -524,7 +531,9 @@ defmodule KafkaEx.GenConsumer do
:extra_consumer_args
)
- {:ok, consumer_state} = consumer_module.init(topic, partition, extra_consumer_args)
+ {:ok, consumer_state} =
+ consumer_module.init(topic, partition, extra_consumer_args)
+
worker_opts = Keyword.take(opts, [:uris])
{:ok, worker_name} =

diff:
@@ -2,9 +2,10 @@ defmodule KafkaEx.Server0P10P1.Test do
use ExUnit.Case
@moduletag :server_0_p_10_and_later
- @moduletag :server_0_p_10_p_1
# specific to this server version because we want to test that the api_versions list is exact
+ @moduletag :server_0_p_10_p_1
+
@tag :api_version
test "can retrieve api versions" do
# api_key, max_version, min_version

diff:
@@ -3,8 +3,7 @@ defmodule KafkaEx.Protocol.JoinGroup.Test do
alias KafkaEx.Protocol.JoinGroup
test "create_request creates a valid join group request" do
- good_request =
- <<
+ good_request = <<
# Preamble
11::16,
0::16,

original_message: Run "mix format" on the project


message: Log useful information when a response fails to parse

diff:
@@ -360,7 +360,7 @@ clusters.
./all_tests.sh
```
-##### Kafka >= 0.9.0
+##### Kafka = 0.9.0
The 0.9 client includes functionality that cannot be tested with older
clusters.

diff:
@@ -839,8 +839,27 @@ defmodule KafkaEx.Server do
config_sync_timeout()
)
|> case do
- {:error, reason} -> {:error, reason}
- response -> module.parse_response(response)
+ {:error, reason} ->
+ {:error, reason}
+
+ response ->
+ try do
+ module.parse_response(response)
+ rescue
+ _ ->
+ Logger.error(
+ "Failed to parse a response from the server: #{
+ inspect(response)
+ }"
+ )
+
+ Kernel.reraise(
+ "Parse error during #{inspect(module)}.parse_response. Couldn't parse: #{
+ inspect(response)
+ }",
+ __STACKTRACE__
+ )
+ end
end
state_out = State.increment_correlation_id(updated_state)

diff:
@@ -6,6 +6,7 @@ ExUnit.configure(
integration: true,
consumer_group: true,
server_0_p_10_p_1: true,
+ server_0_p_10_and_later: true,
server_0_p_9_p_0: true,
server_0_p_8_p_0: true
]


message: Add support for DeleteTopics message (API message 20)

diff:
@@ -24,6 +24,8 @@ defmodule KafkaEx do
alias KafkaEx.Protocol.SyncGroup.Response, as: SyncGroupResponse
alias KafkaEx.Protocol.CreateTopics.Request, as: CreateTopicsRequest
alias KafkaEx.Protocol.CreateTopics.Response, as: CreateTopicsResponse
+ alias KafkaEx.Protocol.DeleteTopics.Request, as: DeleteTopicsRequest
+ alias KafkaEx.Protocol.DeleteTopics.Response, as: DeleteTopicsResponse
alias KafkaEx.Protocol.ApiVersions.Response, as: ApiVersionsResponse
alias KafkaEx.Server
alias KafkaEx.Stream
@@ -618,6 +620,17 @@ defmodule KafkaEx do
Server.call(worker_name, {:create_topics, requests, timeout})
end
+ @doc """
+ Delete topics. Must provide a list of topic names.
+ """
+ @spec delete_topics([DeleteTopicsRequest.t()], Keyword.t()) ::
+ DeleteTopicsResponse.t()
+ def delete_topics(requests, opts \\ []) do
+ worker_name = Keyword.get(opts, :worker_name, Config.default_worker())
+ timeout = Keyword.get(opts, :timeout, 4000)
+ Server.call(worker_name, {:delete_topics, requests, timeout})
+ end
+
# OTP API
def start(_type, _args) do
max_restarts = Application.get_env(:kafka_ex, :max_restarts, 10)

diff:
@@ -14,7 +14,8 @@ defmodule KafkaEx.Protocol do
leave_group: 13,
sync_group: 14,
api_versions: 18,
- create_topics: 19
+ create_topics: 19,
+ delete_topics: 20
}
# DescribeConfigs 32

diff:
@@ -43,4 +43,28 @@ defmodule KafkaEx.Protocol.Common do
{items, data_after_array} = read_array(num_items - 1, rest, read_one)
{[item | items], data_after_array}
end
+
+ @spec encode_nullable_string(String.t()) :: binary
+ def encode_nullable_string(text) do
+ case text do
+ nil -> <<-1::16-signed>>
+ _ -> encode_string(text)
+ end
+ end
+
+ @spec encode_string(String.t()) :: binary
+ def encode_string(text) do
+ <<byte_size(text)::16-signed, text::binary>>
+ end
+
+ def map_encode(elems, function) do
+ if nil == elems or [] == elems do
+ <<0::32-signed>>
+ else
+ <<length(elems)::32-signed>> <>
+ (elems
+ |> Enum.map(function)
+ |> Enum.reduce(&(&1 <> &2)))
+ end
+ end
end

diff:
defmodule KafkaEx.Protocol.CreateTopics do
alias KafkaEx.Protocol
+ import KafkaEx.Protocol.Common
@supported_versions_range {0, 0}
@moduledoc """
@@ -126,30 +127,6 @@ defmodule KafkaEx.Protocol.CreateTopics do
encode_nullable_string(config_entry.config_value)
end
- @spec encode_nullable_string(String.t()) :: binary
- defp encode_nullable_string(text) do
- case text do
- nil -> <<-1::16-signed>>
- _ -> encode_string(text)
- end
- end
-
- @spec encode_string(String.t()) :: binary
- defp encode_string(text) do
- <<byte_size(text)::16-signed, text::binary>>
- end
-
- defp map_encode(elems, function) do
- if nil == elems or [] == elems do
- <<0::32-signed>>
- else
- <<length(elems)::32-signed>> <>
- (elems
- |> Enum.map(function)
- |> Enum.reduce(&(&1 <> &2)))
- end
- end
-
@spec parse_response(binary, integer) :: [] | Response.t()
def parse_response(
<<_correlation_id::32-signed, topic_errors_count::32-signed,

diff:
+defmodule KafkaEx.Protocol.DeleteTopics do
+ alias KafkaEx.Protocol
+ import KafkaEx.Protocol.Common
+ @supported_versions_range {0, 0}
+
+ @moduledoc """
+ Implementation of the Kafka DeleteTopics request and response APIs
+
+ See: https://kafka.apache.org/protocol.html#The_Messages_DeleteTopics
+ """
+
+ # DeleteTopics Request (Version: 0) => [topics] timeout
+ # topics => STRING
+ # timeout => INT32
+
+ # DeleteTopics Response (Version: 0) => [topic_error_codes]
+ # topic_error_codes => topic error_code
+ # topic => STRING
+ # error_code => INT16
+
+ defmodule Request do
+ @moduledoc false
+ defstruct topics: nil, timeout: nil
+ @type t :: %Request{topics: [String.t()], timeout: integer}
+ end
+
+ defmodule TopicError do
+ @moduledoc false
+ defstruct topic_name: nil, error_code: nil
+ @type t :: %TopicError{topic_name: binary, error_code: atom}
+ end
+
+ defmodule Response do
+ @moduledoc false
+ defstruct topic_errors: nil
+ @type t :: %Response{topic_errors: [TopicError]}
+ end
+
+ def api_version(api_versions) do
+ KafkaEx.ApiVersions.find_api_version(
+ api_versions,
+ :delete_topics,
+ @supported_versions_range
+ )
+ end
+
+ @spec create_request(integer, binary, Request.t(), integer) :: binary
+ def create_request(
+ correlation_id,
+ client_id,
+ delete_topics_request,
+ api_version
+ )
+
+ def create_request(correlation_id, client_id, delete_topics_request, 0) do
+ Protocol.create_request(:delete_topics, correlation_id, client_id) <>
+ encode_topics(delete_topics_request.topics) <>
+ <<delete_topics_request.timeout::32-signed>>
+ end
+
+ @spec encode_topics([String.t()]) :: binary
+ defp encode_topics(topics) do
+ topics |> map_encode(&encode_string/1)
+ end
+
+ @spec parse_response(binary, integer) :: [] | Response.t()
+ def parse_response(
+ <<_correlation_id::32-signed, topic_errors_count::32-signed,
+ topic_errors::binary>>,
+ 0
+ ) do
+ %Response{
+ topic_errors: parse_topic_errors(topic_errors_count, topic_errors)
+ }
+ end
+
+ @spec parse_topic_errors(integer, binary) :: [TopicError.t()]
+ defp parse_topic_errors(0, _), do: []
+
+ defp parse_topic_errors(
+ topic_errors_count,
+ <<topic_name_size::16-signed, topic_name::size(topic_name_size)-binary,
+ error_code::16-signed, rest::binary>>
+ ) do
+ [
+ %TopicError{
+ topic_name: topic_name,
+ error_code: Protocol.error(error_code)
+ }
+ | parse_topic_errors(topic_errors_count - 1, rest)
+ ]
+ end
+end

diff:
@@ -18,6 +18,7 @@ defmodule KafkaEx.Server do
alias KafkaEx.Protocol.Produce.Request, as: ProduceRequest
alias KafkaEx.Protocol.SyncGroup.Request, as: SyncGroupRequest
alias KafkaEx.Protocol.CreateTopics.Request, as: CreateTopicsRequest
+ alias KafkaEx.Protocol.DeleteTopics.Request, as: DeleteTopicsRequest
alias KafkaEx.Socket
defmodule State do
@@ -212,6 +213,12 @@ defmodule KafkaEx.Server do
state :: State.t()
) :: {:reply, reply, new_state}
when reply: term, new_state: term
+ @callback kafka_delete_topics(
+ [DeleteTopicsRequest.t()],
+ network_timeout :: integer,
+ state :: State.t()
+ ) :: {:reply, reply, new_state}
+ when reply: term, new_state: term
@callback kafka_api_versions(state :: State.t()) :: {:reply, reply, new_state}
when reply: term, new_state: term
@callback kafka_server_update_metadata(state :: State.t()) ::
@@ -334,6 +341,10 @@ defmodule KafkaEx.Server do
kafka_create_topics(requests, network_timeout, state)
end
+ def handle_call({:delete_topics, topics, network_timeout}, _from, state) do
+ kafka_delete_topics(topics, network_timeout, state)
+ end
+
def handle_call({:api_versions}, _from, state) do
kafka_api_versions(state)
end
@@ -857,7 +868,7 @@ defmodule KafkaEx.Server do
"Parse error during #{inspect(module)}.parse_response. Couldn't parse: #{
inspect(response)
}",
- __STACKTRACE__
+ System.stacktrace()
)
end
end

diff:
@@ -4,6 +4,7 @@ defmodule KafkaEx.Server0P10AndLater do
"""
use KafkaEx.Server
alias KafkaEx.Protocol.CreateTopics
+ alias KafkaEx.Protocol.DeleteTopics
alias KafkaEx.Protocol.ApiVersions
alias KafkaEx.Server0P8P2
alias KafkaEx.Server0P9P0
@@ -179,6 +180,53 @@ defmodule KafkaEx.Server0P10AndLater do
{:reply, response, %{state | correlation_id: state.correlation_id + 1}}
end
+ def kafka_delete_topics(topics, network_timeout, state) do
+ api_version =
+ case DeleteTopics.api_version(state.api_versions) do
+ {:ok, api_version} ->
+ api_version
+
+ _ ->
+ raise "DeleteTopic is not supported in this version of Kafka, or the versions supported by the client do not match the ones supported by the server."
+ end
+
+ main_request =
+ DeleteTopics.create_request(
+ state.correlation_id,
+ @client_id,
+ %DeleteTopics.Request{
+ topics: topics,
+ timeout: network_timeout
+ },
+ api_version
+ )
+
+ broker = state.brokers |> Enum.find(& &1.is_controller)
+
+ {response, state} =
+ case broker do
+ nil ->
+ Logger.log(:error, "Coordinator for topic is not available")
+ {:topic_not_found, state}
+
+ _ ->
+ response =
+ broker
+ |> NetworkClient.send_sync_request(
+ main_request,
+ config_sync_timeout()
+ )
+ |> case do
+ {:error, reason} -> {:error, reason}
+ response -> DeleteTopics.parse_response(response, api_version)
+ end
+
+ {response, %{state | correlation_id: state.correlation_id + 1}}
+ end
+
+ {:reply, response, state}
+ end
+
def kafka_create_topics(requests, network_timeout, state) do
api_version =
case CreateTopics.api_version(state.api_versions) do

diff:
@@ -15,6 +15,7 @@ defmodule KafkaEx.Server0P8P0 do
{:nowarn_function, kafka_server_offset_commit: 2},
{:nowarn_function, kafka_server_offset_fetch: 2},
{:nowarn_function, kafka_create_topics: 3},
+ {:nowarn_function, kafka_delete_topics: 3},
{:nowarn_function, kafka_api_versions: 1}
]
@@ -94,6 +95,9 @@ defmodule KafkaEx.Server0P8P0 do
def kafka_create_topics(_, _, _state),
do: raise("CreateTopic is not supported in 0.8.0 version of kafka")
+ def kafka_delete_topics(_, _, _state),
+ do: raise("DeleteTopic is not supported in 0.8.0 version of kafka")
+
defp fetch(request, state) do
case network_request(request, Fetch, state) do
{{:error, error}, state_out} -> {error, state_out}

diff:
@@ -10,6 +10,7 @@ defmodule KafkaEx.Server0P8P2 do
{:nowarn_function, kafka_server_join_group: 3},
{:nowarn_function, kafka_server_leave_group: 3},
{:nowarn_function, kafka_create_topics: 3},
+ {:nowarn_function, kafka_delete_topics: 3},
{:nowarn_function, kafka_api_versions: 1}
]
@@ -199,6 +200,9 @@ defmodule KafkaEx.Server0P8P2 do
def kafka_create_topics(_, _, _state),
do: raise("CreateTopic is not supported in 0.8.2 version of kafka")
+ def kafka_delete_topics(_, _, _state),
+ do: raise("DeleteTopic is not supported in 0.8.2 version of kafka")
+
defp update_consumer_metadata(state),
do: update_consumer_metadata(state, @retry_count, 0)

diff:
@@ -7,6 +7,7 @@ defmodule KafkaEx.Server0P9P0 do
# these functions aren't implemented for 0.9.0
@dialyzer [
{:nowarn_function, kafka_create_topics: 3},
+ {:nowarn_function, kafka_delete_topics: 3},
{:nowarn_function, kafka_api_versions: 1}
]
@@ -55,6 +56,9 @@ defmodule KafkaEx.Server0P9P0 do
defdelegate kafka_create_topics(requests, network_timeout, state),
to: Server0P8P2
+ defdelegate kafka_delete_topics(requests, network_timeout, state),
+ to: Server0P8P2
+
def kafka_server_init([args]) do
kafka_server_init([args, self()])
end

diff:
@@ -120,3 +120,6 @@ zookeeper.connect=localhost:2181
# Timeout in ms for connecting to zookeeper
zookeeper.connection.timeout.ms=6000
+
+# Allow deletion of topics
+delete.topic.enable=true

diff:
@@ -8,30 +8,46 @@ defmodule KafkaEx.Server0P10P1AndLater.Test do
test "can create a topic" do
name = "create_topic_#{:rand.uniform(2_000_000)}"
- request = %{
- topic: name,
- num_partitions: 10,
- replication_factor: 1,
- replica_assignment: [],
- config_entries: [
+ config = [
%{config_name: "cleanup.policy", config_value: "compact"},
%{config_name: "min.compaction.lag.ms", config_value: "0"}
]
- }
- resp = KafkaEx.create_topics([request], timeout: 2000)
+ resp = create_topic(name, config)
assert {:no_error, name} == parse_create_topic_resp(resp)
- resp = KafkaEx.create_topics([request], timeout: 2000)
+ resp = create_topic(name, config)
assert {:topic_already_exists, name} == parse_create_topic_resp(resp)
wait_for(fn ->
- topics = KafkaEx.metadata().topic_metadatas |> Enum.map(& &1.topic)
- assert Enum.member?(topics, name)
+ Enum.member?(existing_topics(), name)
end)
+
+ assert Enum.member?(existing_topics(), name)
end
- def parse_create_topic_resp(response) do
+ @tag :delete_topic
+ test "can delete a topic" do
+ name = "delete_topic_#{:rand.uniform(2_000_000)}"
+
+ resp = create_topic(name, [])
+ assert {:no_error, name} == parse_create_topic_resp(resp)
+
+ wait_for(fn ->
+ Enum.member?(existing_topics(), name)
+ end)
+
+ resp = KafkaEx.delete_topics([name], timeout: 5_000)
+ assert {:no_error, name} = parse_delete_topic_resp(resp)
+
+ wait_for(fn ->
+ not Enum.member?(existing_topics(), name)
+ end)
+
+ assert not Enum.member?(existing_topics(), name)
+ end
+
+ defp parse_create_topic_resp(response) do
%KafkaEx.Protocol.CreateTopics.Response{
topic_errors: [
%KafkaEx.Protocol.CreateTopics.TopicError{
@@ -43,4 +59,36 @@ defmodule KafkaEx.Server0P10P1AndLater.Test do
{error_code, topic_name}
end
+
+ defp parse_delete_topic_resp(response) do
+ %KafkaEx.Protocol.DeleteTopics.Response{
+ topic_errors: [
+ %KafkaEx.Protocol.DeleteTopics.TopicError{
+ error_code: error_code,
+ topic_name: topic_name
+ }
+ ]
+ } = response
+
+ {error_code, topic_name}
+ end
+
+ defp create_topic(name, config) do
+ KafkaEx.create_topics(
+ [
+ %{
+ topic: name,
+ num_partitions: 10,
+ replication_factor: 1,
+ replica_assignment: [],
+ config_entries: config
+ }
+ ],
+ timeout: 5_000
+ )
+ end
+
+ defp existing_topics() do
+ KafkaEx.metadata().topic_metadatas |> Enum.map(& &1.topic)
+ end
end


message: ConsumerGroup can accept alternative Consumer implementations

old_path: .gitignore

new_path: .gitignore

diff:
@@ -8,3 +8,5 @@ kafka1
kafka2
kafka3
doc
+
+.elixir_ls
\ No newline at end of file

diff:
@@ -137,12 +137,32 @@ defmodule KafkaEx.ConsumerGroup do
"""
@spec start_link(module, binary, [binary], options) :: Supervisor.on_start()
def start_link(consumer_module, group_name, topics, opts \\ []) do
+ start_link(KafkaEx.GenConsumer, consumer_module, group_name, topics, opts)
+ end
+
+ @doc """
+ Starts a consumer group process tree with an alternate consumer implementation.
+
+ By default starts `KafkaEx.GenConsumer` But can be passed any module
+ implementing `GenServer`. The consumer is responsible for pulling messages and
+ managing its commit state. For example `KafkExGenStageConsumer` implements a
+ `GenStage` producer which pulls messages from kafka according to demand.
+ """
+ @spec start_link(module, module, binary, [binary], options) ::
+ Supervisor.on_start()
+ def start_link(
+ gen_consumer_module,
+ consumer_module,
+ group_name,
+ topics,
+ opts
+ ) do
{supervisor_opts, module_opts} =
Keyword.split(opts, [:name, :strategy, :max_restarts, :max_seconds])
Supervisor.start_link(
__MODULE__,
- {consumer_module, group_name, topics, module_opts},
+ {gen_consumer_module, consumer_module, group_name, topics, module_opts},
supervisor_opts
)
end
@@ -284,11 +304,18 @@ defmodule KafkaEx.ConsumerGroup do
# used by ConsumerGroup.Manager to set partition assignments
@doc false
- def start_consumer(pid, consumer_module, group_name, assignments, opts) do
+ def start_consumer(
+ pid,
+ gen_consumer_module,
+ consumer_module,
+ group_name,
+ assignments,
+ opts
+ ) do
child =
supervisor(
KafkaEx.GenConsumer.Supervisor,
- [consumer_module, group_name, assignments, opts],
+ [gen_consumer_module, consumer_module, group_name, assignments, opts],
id: :consumer
)
@@ -311,13 +338,13 @@ defmodule KafkaEx.ConsumerGroup do
end
@doc false
- def init({consumer_module, group_name, topics, opts}) do
+ def init({gen_consumer_module, consumer_module, group_name, topics, opts}) do
opts = Keyword.put(opts, :supervisor_pid, self())
children = [
worker(
KafkaEx.ConsumerGroup.Manager,
- [consumer_module, group_name, topics, opts]
+ [gen_consumer_module, consumer_module, group_name, topics, opts]
)
]

diff:
@@ -23,6 +23,7 @@ defmodule KafkaEx.ConsumerGroup.Manager do
:worker_name,
:heartbeat_interval,
:session_timeout,
+ :gen_consumer_module,
:consumer_module,
:consumer_opts,
:partition_assignment_callback,
@@ -50,22 +51,34 @@ defmodule KafkaEx.ConsumerGroup.Manager do
@doc false
# use `KafkaEx.ConsumerGroup.start_link/4` instead
- @spec start_link(module, binary, [binary], KafkaEx.GenConsumer.options()) ::
+ @spec start_link(
+ module,
+ module,
+ binary,
+ [binary],
+ KafkaEx.GenConsumer.options()
+ ) ::
GenServer.on_start()
- def start_link(consumer_module, group_name, topics, opts \\ []) do
+ def start_link(
+ gen_consumer_module,
+ consumer_module,
+ group_name,
+ topics,
+ opts \\ []
+ ) do
gen_server_opts = Keyword.get(opts, :gen_server_opts, [])
consumer_opts = Keyword.drop(opts, [:gen_server_opts])
GenServer.start_link(
__MODULE__,
- {consumer_module, group_name, topics, consumer_opts},
+ {gen_consumer_module, consumer_module, group_name, topics, consumer_opts},
gen_server_opts
)
end
# GenServer callbacks
- def init({consumer_module, group_name, topics, opts}) do
+ def init({gen_consumer_module, consumer_module, group_name, topics, opts}) do
heartbeat_interval =
Keyword.get(
opts,
@@ -114,6 +127,7 @@ defmodule KafkaEx.ConsumerGroup.Manager do
heartbeat_interval: heartbeat_interval,
session_timeout: session_timeout,
consumer_module: consumer_module,
+ gen_consumer_module: gen_consumer_module,
partition_assignment_callback: partition_assignment_callback,
consumer_opts: consumer_opts,
group_name: group_name,
@@ -380,6 +394,7 @@ defmodule KafkaEx.ConsumerGroup.Manager do
defp start_consumer(
%State{
consumer_module: consumer_module,
+ gen_consumer_module: gen_consumer_module,
consumer_opts: consumer_opts,
group_name: group_name,
supervisor_pid: pid
@@ -389,6 +404,7 @@ defmodule KafkaEx.ConsumerGroup.Manager do
{:ok, consumer_supervisor_pid} =
ConsumerGroup.start_consumer(
pid,
+ gen_consumer_module,
consumer_module,
group_name,
assignments,

diff:
@@ -175,7 +175,7 @@ defmodule KafkaEx.GenConsumer do
@partition 0
setup do
- {:ok, state} = ExampleGenConsumer.init(@topic, @partition)
+ {k:ok, state} = ExampleGenConsumer.init(@topic, @partition)
{:ok, %{state: state}}
end

diff:
@@ -38,11 +38,11 @@ defmodule KafkaEx.GenConsumer.Supervisor do
],
KafkaEx.GenConsumer.options()
) :: Elixir.Supervisor.on_start()
- def start_link(consumer_module, group_name, assignments, opts \\ []) do
+ def start_link(gen_consumer_module, consumer_module, group_name, assignments, opts \\ []) do
start_link_result =
Elixir.Supervisor.start_link(
__MODULE__,
- {consumer_module, group_name, assignments, opts}
+ {gen_consumer_module, consumer_module, group_name, assignments, opts}
)
case start_link_result do
@@ -77,9 +77,9 @@ defmodule KafkaEx.GenConsumer.Supervisor do
|> Enum.any?(&Process.alive?/1)
end
- def init({consumer_module, group_name, _assignments, _opts}) do
+ def init({gen_consumer_module, consumer_module, group_name, _assignments, _opts}) do
children = [
- worker(KafkaEx.GenConsumer, [consumer_module, group_name])
+ worker(gen_consumer_module, [consumer_module, group_name])
]
supervise(children, strategy: :simple_one_for_one)


message: Use global default timeout for delete_topics and create_topics

diff:
@@ -616,7 +616,7 @@ defmodule KafkaEx do
CreateTopicsResponse.t()
def create_topics(requests, opts \\ []) do
worker_name = Keyword.get(opts, :worker_name, Config.default_worker())
- timeout = Keyword.get(opts, :timeout, 4000)
+ timeout = Keyword.get(opts, :timeout)
Server.call(worker_name, {:create_topics, requests, timeout})
end
@@ -627,7 +627,7 @@ defmodule KafkaEx do
DeleteTopicsResponse.t()
def delete_topics(requests, opts \\ []) do
worker_name = Keyword.get(opts, :worker_name, Config.default_worker())
- timeout = Keyword.get(opts, :timeout, 4000)
+ timeout = Keyword.get(opts, :timeout)
Server.call(worker_name, {:delete_topics, requests, timeout})
end

diff:
@@ -196,7 +196,7 @@ defmodule KafkaEx.Server0P10AndLater do
@client_id,
%DeleteTopics.Request{
topics: topics,
- timeout: network_timeout
+ timeout: config_sync_timeout(network_timeout)
},
api_version
)
@@ -239,7 +239,7 @@ defmodule KafkaEx.Server0P10AndLater do
create_topics_request = %CreateTopics.Request{
create_topic_requests: requests,
- timeout: network_timeout
+ timeout: config_sync_timeout(network_timeout)
}
main_request =


message: Correct spec & var names for KafkaEx.delete_topics

diff:
@@ -623,12 +623,12 @@ defmodule KafkaEx do
@doc """
Delete topics. Must provide a list of topic names.
"""
- @spec delete_topics([DeleteTopicsRequest.t()], Keyword.t()) ::
+ @spec delete_topics([String.t()], Keyword.t()) ::
DeleteTopicsResponse.t()
- def delete_topics(requests, opts \\ []) do
+ def delete_topics(topics, opts \\ []) do
worker_name = Keyword.get(opts, :worker_name, Config.default_worker())
timeout = Keyword.get(opts, :timeout)
- Server.call(worker_name, {:delete_topics, requests, timeout})
+ Server.call(worker_name, {:delete_topics, topics, timeout})
end
# OTP API

diff:
@@ -214,7 +214,7 @@ defmodule KafkaEx.Server do
) :: {:reply, reply, new_state}
when reply: term, new_state: term
@callback kafka_delete_topics(
- [DeleteTopicsRequest.t()],
+ [String.t()],
network_timeout :: integer,
state :: State.t()
) :: {:reply, reply, new_state}


message: Correct defdelegate for v0.9.0 unsupported features

diff:
@@ -24,7 +24,6 @@ defmodule KafkaEx do
alias KafkaEx.Protocol.SyncGroup.Response, as: SyncGroupResponse
alias KafkaEx.Protocol.CreateTopics.Request, as: CreateTopicsRequest
alias KafkaEx.Protocol.CreateTopics.Response, as: CreateTopicsResponse
- alias KafkaEx.Protocol.DeleteTopics.Request, as: DeleteTopicsRequest
alias KafkaEx.Protocol.DeleteTopics.Response, as: DeleteTopicsResponse
alias KafkaEx.Protocol.ApiVersions.Response, as: ApiVersionsResponse
alias KafkaEx.Server

diff:
@@ -18,7 +18,6 @@ defmodule KafkaEx.Server do
alias KafkaEx.Protocol.Produce.Request, as: ProduceRequest
alias KafkaEx.Protocol.SyncGroup.Request, as: SyncGroupRequest
alias KafkaEx.Protocol.CreateTopics.Request, as: CreateTopicsRequest
- alias KafkaEx.Protocol.DeleteTopics.Request, as: DeleteTopicsRequest
alias KafkaEx.Socket
defmodule State do

diff:
@@ -51,13 +51,15 @@ defmodule KafkaEx.Server0P9P0 do
defdelegate kafka_server_consumer_group_metadata(state), to: Server0P8P2
defdelegate kafka_server_update_consumer_metadata(state), to: Server0P8P2
- defdelegate kafka_api_versions(state), to: Server0P8P2
- defdelegate kafka_create_topics(requests, network_timeout, state),
- to: Server0P8P2
+ def kafka_api_versions(_state),
+ do: raise("ApiVersions is not supported in 0.9.0 version of kafka")
- defdelegate kafka_delete_topics(requests, network_timeout, state),
- to: Server0P8P2
+ def kafka_create_topics(_, _, _state),
+ do: raise("CreateTopic is not supported in 0.9.0 version of kafka")
+
+ def kafka_delete_topics(_, _, _state),
+ do: raise("DeleteTopic is not supported in 0.9.0 version of kafka")
def kafka_server_init([args]) do
kafka_server_init([args, self()])


message: Wrap gen_consumer_module and consumer_module in a tuple to prevent linter complaints about arity

diff:
@@ -137,7 +137,7 @@ defmodule KafkaEx.ConsumerGroup do
"""
@spec start_link(module, binary, [binary], options) :: Supervisor.on_start()
def start_link(consumer_module, group_name, topics, opts \\ []) do
- start_link(KafkaEx.GenConsumer, consumer_module, group_name, topics, opts)
+ start_link({KafkaEx.GenConsumer, consumer_module}, group_name, topics, opts)
end
@doc """
@@ -148,11 +148,10 @@ defmodule KafkaEx.ConsumerGroup do
managing its commit state. For example `KafkExGenStageConsumer` implements a
`GenStage` producer which pulls messages from kafka according to demand.
"""
- @spec start_link(module, module, binary, [binary], options) ::
+ @spec start_link({module, module}, binary, [binary], options) ::
Supervisor.on_start()
def start_link(
- gen_consumer_module,
- consumer_module,
+ {gen_consumer_module, consumer_module},
group_name,
topics,
opts
@@ -162,7 +161,7 @@ defmodule KafkaEx.ConsumerGroup do
Supervisor.start_link(
__MODULE__,
- {gen_consumer_module, consumer_module, group_name, topics, module_opts},
+ {{gen_consumer_module, consumer_module}, group_name, topics, module_opts},
supervisor_opts
)
end
@@ -306,8 +305,7 @@ defmodule KafkaEx.ConsumerGroup do
@doc false
def start_consumer(
pid,
- gen_consumer_module,
- consumer_module,
+ {gen_consumer_module, consumer_module},
group_name,
assignments,
opts
@@ -315,7 +313,7 @@ defmodule KafkaEx.ConsumerGroup do
child =
supervisor(
KafkaEx.GenConsumer.Supervisor,
- [gen_consumer_module, consumer_module, group_name, assignments, opts],
+ [{gen_consumer_module, consumer_module}, group_name, assignments, opts],
id: :consumer
)
@@ -338,13 +336,13 @@ defmodule KafkaEx.ConsumerGroup do
end
@doc false
- def init({gen_consumer_module, consumer_module, group_name, topics, opts}) do
+ def init({{gen_consumer_module, consumer_module}, group_name, topics, opts}) do
opts = Keyword.put(opts, :supervisor_pid, self())
children = [
worker(
KafkaEx.ConsumerGroup.Manager,
- [gen_consumer_module, consumer_module, group_name, topics, opts]
+ [{gen_consumer_module, consumer_module}, group_name, topics, opts]
)
]

diff:
@@ -52,8 +52,7 @@ defmodule KafkaEx.ConsumerGroup.Manager do
@doc false
# use `KafkaEx.ConsumerGroup.start_link/4` instead
@spec start_link(
- module,
- module,
+ {module, module},
binary,
[binary],
KafkaEx.GenConsumer.options()
@@ -71,14 +70,15 @@ defmodule KafkaEx.ConsumerGroup.Manager do
GenServer.start_link(
__MODULE__,
- {gen_consumer_module, consumer_module, group_name, topics, consumer_opts},
+ {{gen_consumer_module, consumer_module}, group_name, topics,
+ consumer_opts},
gen_server_opts
)
end
# GenServer callbacks
- def init({gen_consumer_module, consumer_module, group_name, topics, opts}) do
+ def init({{gen_consumer_module, consumer_module}, group_name, topics, opts}) do
heartbeat_interval =
Keyword.get(
opts,
@@ -404,8 +404,7 @@ defmodule KafkaEx.ConsumerGroup.Manager do
{:ok, consumer_supervisor_pid} =
ConsumerGroup.start_consumer(
pid,
- gen_consumer_module,
- consumer_module,
+ {gen_consumer_module, consumer_module},
group_name,
assignments,
consumer_opts

diff:
@@ -31,18 +31,22 @@ defmodule KafkaEx.GenConsumer.Supervisor do
returns `{:ok, pid}`, where `pid` is the PID of the supervisor.
"""
@spec start_link(
- callback_module :: module,
- consumer_group_name :: binary,
+ {callback_module :: module, consumer_group_name :: binary},
assigned_partitions :: [
{topic_name :: binary, partition_id :: non_neg_integer}
],
KafkaEx.GenConsumer.options()
) :: Elixir.Supervisor.on_start()
- def start_link(gen_consumer_module, consumer_module, group_name, assignments, opts \\ []) do
+ def start_link(
+ {gen_consumer_module, consumer_module},
+ group_name,
+ assignments,
+ opts \\ []
+ ) do
start_link_result =
Elixir.Supervisor.start_link(
__MODULE__,
- {gen_consumer_module, consumer_module, group_name, assignments, opts}
+ {{gen_consumer_module, consumer_module}, group_name, assignments, opts}
)
case start_link_result do
@@ -77,7 +81,10 @@ defmodule KafkaEx.GenConsumer.Supervisor do
|> Enum.any?(&Process.alive?/1)
end
- def init({gen_consumer_module, consumer_module, group_name, _assignments, _opts}) do
+ def init(
+ {{gen_consumer_module, consumer_module}, group_name, _assignments,
+ _opts}
+ ) do
children = [
worker(gen_consumer_module, [consumer_module, group_name])
]


message: Add guard to prevent tuple passed as consumer_module

diff:
@@ -136,7 +136,7 @@ defmodule KafkaEx.ConsumerGroup do
This function has the same return values as `Supervisor.start_link/3`.
"""
@spec start_link(module, binary, [binary], options) :: Supervisor.on_start()
- def start_link(consumer_module, group_name, topics, opts \\ []) do
+ def start_link(consumer_module, group_name, topics, opts \\ []) when is_atom(consumer_module) do
start_link({KafkaEx.GenConsumer, consumer_module}, group_name, topics, opts)
end


message: improve documentation of tuple GenConsumer arguments

diff:
@@ -120,11 +120,20 @@ defmodule KafkaEx.ConsumerGroup do
This can be used to start a `KafkaEx.ConsumerGroup` as part of a supervision
tree.
- `module` is a module that implements the `KafkaEx.GenConsumer` behaviour.
- `group_name` is the name of the consumer group. `topics` is a list of topics
- that the consumer group should consume from. `opts` can be composed of
- options for the supervisor as well as for the `KafkEx.GenConsumer` processes
- that will be spawned by the supervisor. See `t:option/0` for details.
+ `consumer_module` is
+ - a module that implements the `KafkaEx.GenConsumer`
+ behaviour.
+ - a tuple of `{gen_consumer_module, consumer_module}` can substitute another
+ `GenServer` implementation for `KafkaEx.GenConsumer`. When a single module
+ is passed it is transformed to `{KafkaEx.GenConsumer, consumer_module}`.
+
+ `group_name` is the name of the consumer group.
+
+ `topics` is a list of topics that the consumer group should consume from.
+
+ `opts` can be composed of options for the supervisor as well as for the
+ `KafkEx.GenConsumer` processes that will be spawned by the supervisor. See
+ `t:option/0` for details.
*Note* When starting a consumer group with multiple topics, you should
propagate this configuration change to your consumers. If you add a topic to
@@ -135,21 +144,15 @@ defmodule KafkaEx.ConsumerGroup do
This function has the same return values as `Supervisor.start_link/3`.
"""
- @spec start_link(module | {module, module}, binary, [binary], options) :: Supervisor.on_start()
+ @spec start_link(module | {module, module}, binary, [binary], options) ::
+ Supervisor.on_start()
def start_link(consumer_module, group_name, topics, opts \\ [])
+
def start_link(consumer_module, group_name, topics, opts)
when is_atom(consumer_module) do
start_link({KafkaEx.GenConsumer, consumer_module}, group_name, topics, opts)
end
- @doc """
- Starts a consumer group process tree with an alternate consumer implementation.
-
- By default starts `KafkaEx.GenConsumer` But can be passed any module
- implementing `GenServer`. The consumer is responsible for pulling messages and
- managing its commit state. For example `KafkExGenStageConsumer` implements a
- `GenStage` producer which pulls messages from kafka according to demand.
- """
def start_link(
{gen_consumer_module, consumer_module},
group_name,

diff:
@@ -439,7 +439,7 @@ defmodule KafkaEx.GenConsumer do
This function has the same return values as `GenServer.start_link/3`.
"""
@spec start_link(
- callback_module :: module,
+ consumer_module :: module,
consumer_group_name :: binary,
topic_name :: binary,
partition_id :: non_neg_integer,

diff:
@@ -18,7 +18,9 @@ defmodule KafkaEx.GenConsumer.Supervisor do
@doc """
Starts a `GenConsumer.Supervisor` process linked to the current process.
- `module` is a module that implements the `GenConsumer` behaviour.
+ `gen_consumer_module` is a module that implements the `GenServer` behaviour
+ which consumes events from kafka.
+ `consumer_module` is a module that implements the `GenConsumer` behaviour.
`group_name` is the name of a consumer group, and `assignments` is a list of
partitions for the `GenConsumer`s to consume. `opts` accepts the same
options as `KafkaEx.GenConsumer.start_link/5`.
@@ -31,7 +33,8 @@ defmodule KafkaEx.GenConsumer.Supervisor do
returns `{:ok, pid}`, where `pid` is the PID of the supervisor.
"""
@spec start_link(
- {callback_module :: module, consumer_group_name :: binary},
+ {gen_consumer_module ::module, consumer_module :: module},
+ consumer_group_name :: binary,
assigned_partitions :: [
{topic_name :: binary, partition_id :: non_neg_integer}
],


message: Rename callbacks for consistency (kafka_* => kafka_server_*)

diff:
@@ -206,19 +206,20 @@ defmodule KafkaEx.Server do
| {:stop, reason, reply, new_state}
| {:stop, reason, new_state}
when reply: term, new_state: term, reason: term
- @callback kafka_create_topics(
+ @callback kafka_server_create_topics(
[CreateTopicsRequest.t()],
network_timeout :: integer,
state :: State.t()
) :: {:reply, reply, new_state}
when reply: term, new_state: term
- @callback kafka_delete_topics(
+ @callback kafka_server_delete_topics(
[String.t()],
network_timeout :: integer,
state :: State.t()
) :: {:reply, reply, new_state}
when reply: term, new_state: term
- @callback kafka_api_versions(state :: State.t()) :: {:reply, reply, new_state}
+ @callback kafka_server_api_versions(state :: State.t()) ::
+ {:reply, reply, new_state}
when reply: term, new_state: term
@callback kafka_server_update_metadata(state :: State.t()) ::
{:noreply, new_state}
@@ -337,15 +338,15 @@ defmodule KafkaEx.Server do
end
def handle_call({:create_topics, requests, network_timeout}, _from, state) do
- kafka_create_topics(requests, network_timeout, state)
+ kafka_server_create_topics(requests, network_timeout, state)
end
def handle_call({:delete_topics, topics, network_timeout}, _from, state) do
- kafka_delete_topics(topics, network_timeout, state)
+ kafka_server_delete_topics(topics, network_timeout, state)
end
def handle_call({:api_versions}, _from, state) do
- kafka_api_versions(state)
+ kafka_server_api_versions(state)
end
def handle_info(:update_metadata, state) do

diff:
@@ -95,6 +95,7 @@ defmodule KafkaEx.Server0P10AndLater do
check_brokers_sockets!(brokers)
{_,
+
%KafkaEx.Protocol.ApiVersions.Response{
api_versions: api_versions,
error_code: error_code
@@ -181,7 +182,7 @@ defmodule KafkaEx.Server0P10AndLater do
{:noreply, update_metadata(state)}
end
- def kafka_api_versions(state) do
+ def kafka_server_api_versions(state) do
response =
state.correlation_id
|> ApiVersions.create_request(@client_id)
@@ -191,7 +192,7 @@ defmodule KafkaEx.Server0P10AndLater do
{:reply, response, %{state | correlation_id: state.correlation_id + 1}}
end
- def kafka_delete_topics(topics, network_timeout, state) do
+ def kafka_server_delete_topics(topics, network_timeout, state) do
api_version =
case DeleteTopics.api_version(state.api_versions) do
{:ok, api_version} ->
@@ -238,7 +239,7 @@ defmodule KafkaEx.Server0P10AndLater do
{:reply, response, state}
end
- def kafka_create_topics(requests, network_timeout, state) do
+ def kafka_server_create_topics(requests, network_timeout, state) do
api_version =
case CreateTopics.api_version(state.api_versions) do
{:ok, api_version} ->

diff:
@@ -14,9 +14,9 @@ defmodule KafkaEx.Server0P8P0 do
{:nowarn_function, kafka_server_consumer_group: 1},
{:nowarn_function, kafka_server_offset_commit: 2},
{:nowarn_function, kafka_server_offset_fetch: 2},
- {:nowarn_function, kafka_create_topics: 3},
- {:nowarn_function, kafka_delete_topics: 3},
- {:nowarn_function, kafka_api_versions: 1}
+ {:nowarn_function, kafka_server_create_topics: 3},
+ {:nowarn_function, kafka_server_delete_topics: 3},
+ {:nowarn_function, kafka_server_api_versions: 1}
]
use KafkaEx.Server
@@ -89,13 +89,13 @@ defmodule KafkaEx.Server0P8P0 do
"Consumer Group Metadata is not supported in 0.8.0 version of kafka"
)
- def kafka_api_versions(_state),
+ def kafka_server_api_versions(_state),
do: raise("ApiVersions is not supported in 0.8.0 version of kafka")
- def kafka_create_topics(_, _, _state),
+ def kafka_server_create_topics(_, _, _state),
do: raise("CreateTopic is not supported in 0.8.0 version of kafka")
- def kafka_delete_topics(_, _, _state),
+ def kafka_server_delete_topics(_, _, _state),
do: raise("DeleteTopic is not supported in 0.8.0 version of kafka")
defp fetch(request, state) do

diff:
@@ -9,9 +9,9 @@ defmodule KafkaEx.Server0P8P2 do
{:nowarn_function, kafka_server_sync_group: 3},
{:nowarn_function, kafka_server_join_group: 3},
{:nowarn_function, kafka_server_leave_group: 3},
- {:nowarn_function, kafka_create_topics: 3},
- {:nowarn_function, kafka_delete_topics: 3},
- {:nowarn_function, kafka_api_versions: 1}
+ {:nowarn_function, kafka_server_create_topics: 3},
+ {:nowarn_function, kafka_server_delete_topics: 3},
+ {:nowarn_function, kafka_server_api_versions: 1}
]
use KafkaEx.Server
@@ -200,13 +200,13 @@ defmodule KafkaEx.Server0P8P2 do
def kafka_server_heartbeat(_, _, _state),
do: raise("Heartbeat is not supported in 0.8.2 version of kafka")
- def kafka_api_versions(_state),
+ def kafka_server_api_versions(_state),
do: raise("ApiVersions is not supported in 0.8.2 version of kafka")
- def kafka_create_topics(_, _, _state),
+ def kafka_server_create_topics(_, _, _state),
do: raise("CreateTopic is not supported in 0.8.2 version of kafka")
- def kafka_delete_topics(_, _, _state),
+ def kafka_server_delete_topics(_, _, _state),
do: raise("DeleteTopic is not supported in 0.8.2 version of kafka")
defp update_consumer_metadata(state),

diff:
@@ -6,9 +6,9 @@ defmodule KafkaEx.Server0P9P0 do
# these functions aren't implemented for 0.9.0
@dialyzer [
- {:nowarn_function, kafka_create_topics: 3},
- {:nowarn_function, kafka_delete_topics: 3},
- {:nowarn_function, kafka_api_versions: 1}
+ {:nowarn_function, kafka_server_create_topics: 3},
+ {:nowarn_function, kafka_server_delete_topics: 3},
+ {:nowarn_function, kafka_server_api_versions: 1}
]
use KafkaEx.Server
@@ -52,13 +52,13 @@ defmodule KafkaEx.Server0P9P0 do
defdelegate kafka_server_consumer_group_metadata(state), to: Server0P8P2
defdelegate kafka_server_update_consumer_metadata(state), to: Server0P8P2
- def kafka_api_versions(_state),
+ def kafka_server_api_versions(_state),
do: raise("ApiVersions is not supported in 0.9.0 version of kafka")
- def kafka_create_topics(_, _, _state),
+ def kafka_server_create_topics(_, _, _state),
do: raise("CreateTopic is not supported in 0.9.0 version of kafka")
- def kafka_delete_topics(_, _, _state),
+ def kafka_server_delete_topics(_, _, _state),
do: raise("DeleteTopic is not supported in 0.9.0 version of kafka")
def kafka_server_init([args]) do


message: Update metadata after creating/deleting a topic

diff:
# WARN: when changing something here, there should probably also be a change in scripts/ci_tests.sh
-mix test --include integration --include consumer_group --include server_0_p_10_and_later --include server_0_p_9_p_0 --include server_0_p_8_p_0
+mix test --include integration --include consumer_group --include server_0_p_10_and_later --include server_0_p_10_p_1 --include server_0_p_9_p_0 --include server_0_p_8_p_0

diff:
@@ -236,6 +236,8 @@ defmodule KafkaEx.Server0P10AndLater do
{response, %{state | correlation_id: state.correlation_id + 1}}
end
+ state = update_metadata(state)
+
{:reply, response, state}
end
@@ -285,6 +287,8 @@ defmodule KafkaEx.Server0P10AndLater do
{response, %{state | correlation_id: state.correlation_id + 1}}
end
+ state = update_metadata(state)
+
{:reply, response, state}
end

diff:
@@ -3,6 +3,7 @@ defmodule KafkaEx.Server0P10P1AndLater.Test do
import TestHelper
@moduletag :server_0_p_10_and_later
+ @num_partitions 10
@tag :create_topic
test "can create a topic" do
@@ -19,11 +20,14 @@ defmodule KafkaEx.Server0P10P1AndLater.Test do
resp = create_topic(name, config)
assert {:topic_already_exists, name} == parse_create_topic_resp(resp)
- wait_for(fn ->
Enum.member?(existing_topics(), name)
- end)
- assert Enum.member?(existing_topics(), name)
+ assert @num_partitions ==
+ KafkaEx.Protocol.Metadata.Response.partitions_for_topic(
+ KafkaEx.metadata(),
+ name
+ )
+ |> Enum.count()
end
@tag :delete_topic
@@ -39,11 +43,6 @@ defmodule KafkaEx.Server0P10P1AndLater.Test do
resp = KafkaEx.delete_topics([name], timeout: 5_000)
assert {:no_error, name} = parse_delete_topic_resp(resp)
-
- wait_for(fn ->
- not Enum.member?(existing_topics(), name)
- end)
-
assert not Enum.member?(existing_topics(), name)
end
@@ -78,13 +77,13 @@ defmodule KafkaEx.Server0P10P1AndLater.Test do
[
%{
topic: name,
- num_partitions: 10,
+ num_partitions: @num_partitions,
replication_factor: 1,
replica_assignment: [],
config_entries: config
}
],
- timeout: 5_000
+ timeout: 10_000
)
end


message: Fix spec errors

diff:
@@ -22,7 +22,7 @@ defmodule KafkaEx do
alias KafkaEx.Protocol.Produce.Message
alias KafkaEx.Protocol.SyncGroup.Request, as: SyncGroupRequest
alias KafkaEx.Protocol.SyncGroup.Response, as: SyncGroupResponse
- alias KafkaEx.Protocol.CreateTopics.Request, as: CreateTopicsRequest
+ alias KafkaEx.Protocol.CreateTopics.TopicRequest, as: CreateTopicsRequest
alias KafkaEx.Protocol.CreateTopics.Response, as: CreateTopicsResponse
alias KafkaEx.Protocol.DeleteTopics.Response, as: DeleteTopicsResponse
alias KafkaEx.Protocol.ApiVersions.Response, as: ApiVersionsResponse

diff:
@@ -46,15 +46,15 @@ defmodule KafkaEx.Protocol.CreateTopics do
topic: binary,
num_partitions: integer,
replication_factor: integer,
- replica_assignment: [ReplicaAssignment],
- config_entries: [ConfigEntry]
+ replica_assignment: [ReplicaAssignment.t()],
+ config_entries: [ConfigEntry.t()]
}
end
defmodule Request do
@moduledoc false
defstruct create_topic_requests: nil, timeout: nil
- @type t :: %Request{create_topic_requests: [TopicRequest], timeout: integer}
+ @type t :: %Request{create_topic_requests: [TopicRequest.t()], timeout: integer}
end
defmodule TopicError do
@@ -66,7 +66,7 @@ defmodule KafkaEx.Protocol.CreateTopics do
defmodule Response do
@moduledoc false
defstruct topic_errors: nil
- @type t :: %Response{topic_errors: [TopicError]}
+ @type t :: %Response{topic_errors: [TopicError.t()]}
end
def api_version(api_versions) do

diff:
@@ -33,7 +33,7 @@ defmodule KafkaEx.Protocol.DeleteTopics do
defmodule Response do
@moduledoc false
defstruct topic_errors: nil
- @type t :: %Response{topic_errors: [TopicError]}
+ @type t :: %Response{topic_errors: [TopicError.t()]}
end
def api_version(api_versions) do

diff:
@@ -17,7 +17,7 @@ defmodule KafkaEx.Server do
alias KafkaEx.Protocol.Produce
alias KafkaEx.Protocol.Produce.Request, as: ProduceRequest
alias KafkaEx.Protocol.SyncGroup.Request, as: SyncGroupRequest
- alias KafkaEx.Protocol.CreateTopics.Request, as: CreateTopicsRequest
+ alias KafkaEx.Protocol.CreateTopics.TopicRequest, as: CreateTopicsRequest
alias KafkaEx.Socket
defmodule State do

original_message: Fix spec errors


message: allow partition to be nil in KafkaEx.Protocol.Produce.Request

diff:
@@ -31,7 +31,7 @@ defmodule KafkaEx.Protocol.Produce do
@type t :: %Request{
topic: binary,
- partition: integer,
+ partition: integer | nil,
required_acks: integer,
timeout: integer,
compression: atom,


message: add :uris as a valid option for ConsumerGroup

diff:
@@ -25,6 +25,7 @@ KakfaEx supports the following Kafka features:
* Message Compression with Snappy and gzip
* Offset Management (fetch / commit / autocommit)
* Consumer Groups
+* Topics Management (create / delete)
See [Kafka Protocol Documentation](http://kafka.apache.org/protocol.html) and
[A Guide to the Kafka Protocol](https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol)

diff:
@@ -94,6 +94,7 @@ defmodule KafkaEx.ConsumerGroup do
* `:max_restarts`, `:max_seconds` - Supervisor restart policy parameters
* `:partition_assignment_callback` - See
`t:KafkaEx.ConsumerGroup.PartitionAssignment.callback/0`
+ * `:uris` - See `KafkaEx.create_worker/2`
Note `:session_timeout` is registered with the broker and determines how long
before the broker will de-register a consumer from which it has not heard a
@@ -111,6 +112,7 @@ defmodule KafkaEx.ConsumerGroup do
| {:name, Supervisor.name()}
| {:max_restarts, non_neg_integer}
| {:max_seconds, non_neg_integer}
+ | {:uris, KafkaEx.uri()}
@type options :: [option]

diff:
@@ -99,7 +99,7 @@ defmodule KafkaEx.Server0P10AndLater do
%KafkaEx.Protocol.ApiVersions.Response{
api_versions: api_versions,
error_code: error_code
- }, state} = kafka_api_versions(%State{brokers: brokers})
+ }, state} = kafka_server_api_versions(%State{brokers: brokers})
if error_code == :no_response do
sleep_for_reconnect()
raise "Brokers sockets are closed"

diff:
@@ -20,7 +20,7 @@ defmodule KafkaEx.Server0P10P1AndLater.Test do
resp = create_topic(name, config)
assert {:topic_already_exists, name} == parse_create_topic_resp(resp)
- Enum.member?(existing_topics(), name)
+ assert Enum.member?(existing_topics(), name)
assert @num_partitions ==
KafkaEx.Protocol.Metadata.Response.partitions_for_topic(


message: Use env vars in docker-compose to configure everything

diff:
@@ -14,9 +14,9 @@ config :kafka_ex,
# server.properties file.
# In the case below you would set "advertised.host.name=localhost"
brokers: [
- {"localhost", 9092},
{"localhost", 9093},
- {"localhost", 9094}
+ {"localhost", 9094},
+ {"localhost", 9095}
],
#
# OR:

diff:
+#################################
+# Common Kafka config
+#################################
+
+# Note: any property `something.bla` can be configure by setting `KAFKA_SOMETHING_BLA`.
+
+######## topic creation
+
+KAFKA_CREATE_TOPICS=consumer_group_implementation_test:4:2,test0p8p0:4:2
+
+######## zookeeper
+
+KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS=6000
+
+######## advertised hosts names and protocols
+
+# alternative to KAFKA_ADVERTISED_HOST_NAME is: HOSTNAME_COMMAND: ip addr | grep -o "inet [0-9.]*" | grep -v "127\.0\.0\.1" | grep -o "[0-9.]*"
+KAFKA_ADVERTISED_HOST_NAME=localhost
+KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:SSL,OUTSIDE:SSL
+KAFKA_ADVERTISED_PROTOCOL_NAME=OUTSIDE
+KAFKA_PROTOCOL_NAME=INSIDE
+
+######## SSL
+
+KAFKA_SSL_KEYSTORE_LOCATION=/ssl/server.keystore.jks
+KAFKA_SSL_KEYSTORE_PASSWORD=kafka_ex
+KAFKA_SSL_KEY_PASSWORD=kafka_ex
+KAFKA_SSL_TRUSTSTORE_LOCATION=/ssl/server.truststore.jks
+KAFKA_SSL_TRUSTSTORE_PASSWORD=kafka_ex
+KAFKA_SSL_SECURE_RANDOM_IMPLEMENTATION=SHA1PRNG
+
+######## Config
+
+KAFKA_DELETE_TOPIC_ENABLE=true
+# KAFKA_NUM_NETWORK_THREADS=3
+# KAFKA_NUM_IO_THREADS=8
+# KAFKA_SOCKET_SEND_BUFFER_BYTES=102400
+# KAFKA_SOCKET_RECEIVE_BUFFER_BYTES=102400
+# KAFKA_SOCKET_REQUEST_MAX_BYTES=104857600
+KAFKA_LOG_DIRS=/tmp/kafka_log
+KAFKA_NUM_PARTITIONS=1
+KAFKA_NUM_RECOVERY_THREADS_PER_DATA_DIR=1
+KAFKA_LOG_RETENTION_HOURS=168
+# KAFKA_LOG_SEGMENT_BYTES=1073741824
+# KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS=300000

diff:
version: '3.2'
+
services:
zookeeper:
image: wurstmeister/zookeeper:3.4.6
@@ -7,30 +8,38 @@ services:
kafka1:
image: wurstmeister/kafka:0.11.0.1
ports:
- - "9092:9092"
+ - "9093:9093"
depends_on:
- zookeeper
volumes:
- - ./kafka1/server.properties.in:/opt/kafka/config/server.properties.in
- - ./scripts/docker-start-kafka.sh:/usr/bin/start-kafka.sh
- ./ssl:/ssl
+ env_file: docker-compose-kafka.env
+ environment:
+ KAFKA_BROKER_ID: 1
+ KAFKA_ADVERTISED_PORT: 9093
+
kafka2:
image: wurstmeister/kafka:0.11.0.1
ports:
- - "9093:9093"
+ - "9094:9094"
depends_on:
- zookeeper
volumes:
- - ./kafka2/server.properties.in:/opt/kafka/config/server.properties.in
- - ./scripts/docker-start-kafka.sh:/usr/bin/start-kafka.sh
- ./ssl:/ssl
+ env_file: docker-compose-kafka.env
+ environment:
+ KAFKA_BROKER_ID: 2
+ KAFKA_ADVERTISED_PORT: 9094
+
kafka3:
image: wurstmeister/kafka:0.11.0.1
ports:
- - "9094:9094"
+ - "9095:9095"
depends_on:
- zookeeper
volumes:
- - ./kafka3/server.properties.in:/opt/kafka/config/server.properties.in
- - ./scripts/docker-start-kafka.sh:/usr/bin/start-kafka.sh
- ./ssl:/ssl
+ env_file: docker-compose-kafka.env
+ environment:
+ KAFKA_BROKER_ID: 3
+ KAFKA_ADVERTISED_PORT: 9095

diff:
+# Scripts
+
The scripts in this directory are used for testing KafkaEx and are not part of
the release package. Each script should have a comment near the top explaining
its purpose.

change_type: DELETE

diff:
-#!/bin/bash
-
-# This script is used when the docker kafka container starts
-#
-# This version is specialized from
-# https://github.com/wurstmeister/kafka-docker/blob/master/start-kafka.sh
-# to work with our setup (esp. with ssl)
-#
-# Note that we mainly manage the server.properties file during `docker_up.sh`;
-# the wurstmeister/kafka behavior of translating KAFKA_* env vars to settings
-# is removed from this script.
-
-set -ev
-
-cp ${KAFKA_HOME}/config/server.properties.in ${KAFKA_HOME}/config/server.properties
-
-KAFKA_PID=0
-
-# see https://medium.com/@gchudnov/trapping-signals-in-docker-containers-7a57fdda7d86#.bh35ir4u5
-term_handler() {
- echo 'Stopping Kafka....'
- if [ $KAFKA_PID -ne 0 ]; then
- kill -s TERM "$KAFKA_PID"
- wait "$KAFKA_PID"
- fi
- echo 'Kafka stopped.'
- exit
-}
-
-
-# Capture kill requests to stop properly
-trap "term_handler" SIGHUP SIGINT SIGTERM
-$KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/config/server.properties &
-KAFKA_PID=$!
-
-wait "$KAFKA_PID"

diff:
-#!/bin/bash
-# Finds a private IP for your machine. Tested on MacOS 10.14, Ubuntu 16.04, and Centos 7
-
-# 192.168.x.x, 172.[16-31].x.x, 10.x.x.x
-regex='inet (192\.168\.|172\.1[6789]\.|172\.2[0-9]\.|172\.3[01]\.|10\.)'
-
-if [ "$(uname)" == "Darwin" ]; then
- # MacOS
- ifconfig | grep -E "$regex" | grep broadcast | awk -F ' ' '{print $2}' | head -n 1
-else
- # Linux
- ip addr | grep -E "$regex" | grep brd | awk -F ' ' '{print $2}' | cut -d/ -f1 | head -n 1
-fi
-

diff:
#!/bin/bash
-# Launches a dockerized kafka cluster configured for testing with KafkaEx
-#
-# This script attempts to auto-detect the ip address of an active network
-# interface using `./scripts/docker_ip.sh`.
-#
-# This script should be run from the project root
-
-set -e
-
-# Kafka needs to know our ip address so that it can advertise valid
-# connnection details
-
-export DOCKER_IP=$(./scripts/docker_ip.sh)
-
-# for debugging purposes
-echo Detected active ip address ${DOCKER_IP}
-
-for i in 1 2 3
-do
- port=$(expr 9092 + ${i} - 1)
- mkdir -p kafka${i}
- target=kafka${i}/server.properties.in
- cp ./server.properties ${target}
- # configure broker and port
- sed -i.bak "s/@broker_id@/${i}/g" ${target}
- sed -i.bak "s/@port@/${port}/g" ${target}
- # the @pwd become root directory so we get /ssl
- sed -i.bak "s|@pwd@||g" ${target}
- # point at zookeeper in docker
- sed -i.bak "s/localhost:2181/zookeeper:2181/" ${target}
- # delete the existing listeners line
- sed -i.bak "/^listeners=/d" ${target}
- # add an advertised.listeners and listeners together at the end
- echo "advertised.listeners=SSL://${DOCKER_IP}:${port}" >> ${target}
- echo "listeners=SSL://0.0.0.0:${port}" >> ${target}
-done
+# This script starts zookeeper and the 3 kafka nodes used for testing
docker-compose up -d
-
-# create topics needed for testing
-docker-compose exec kafka3 /bin/bash -c "KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 KAFKA_PORT=9094 KAFKA_CREATE_TOPICS=consumer_group_implementation_test:4:2,test0p8p0:4:2 create-topics.sh"

diff:
-#!/bin/bash
-
-# see docker_up.sh
-# This script should be run from the project root
-
-set -e
-
-IP_IFACE="docker0"
-DOCKER_IP=$(ip addr show dev $IP_IFACE | grep -o "inet[^/]*" | grep -o "[0-9]\{1,3\}[.][0-9.]*")
-
-for i in 1 2 3
-do
- port=$(expr 9092 + ${i} - 1)
- mkdir -p kafka${i}
- target=kafka${i}/server.properties.in
- cp ./server.properties ${target}
- # configure broker and port
- sed -i.bak "s/@broker_id@/${i}/g" ${target}
- sed -i.bak "s/@port@/${port}/g" ${target}
- # the @pwd become root directory so we get /ssl
- sed -i.bak "s|@pwd@||g" ${target}
- # point at zookeeper in docker
- sed -i.bak "s/localhost:2181/zookeeper:2181/" ${target}
- # delete the existing listeners line
- sed -i.bak "/^listeners=/d" ${target}
- # add an advertised.listeners and listeners together at the end
- echo "advertised.listeners=SSL://${DOCKER_IP}:${port}" >> ${target}
- echo "listeners=SSL://0.0.0.0:${port}" >> ${target}
-done
-
-docker-compose up -d
-
-# create topics needed for testing
-docker-compose exec kafka3 /bin/bash -c "KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 KAFKA_PORT=9094 KAFKA_CREATE_TOPICS=consumer_group_implementation_test:4:2,test0p8p0:4:2 create-topics.sh"