message: Added interface iButton

diff:
+/*
+ * Author: Mihai Stefanescu <mihai.stefanescu@rinftech.com>
+ * Copyright (c) 2018 Intel Corporation.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#pragma once
+
+namespace upm
+{
+/**
+ * @brief Interface for Button sensors
+*/
+ class iButton
+ {
+ public:
+ virtual ~iButton() {}
+
+ /**
+ * Gets the current button press state.
+ *
+ * @returns Button state
+ */
+ virtual bool isPressed() = 0;
+ };
+}

diff:
@@ -59,6 +59,11 @@ int Button::value()
return mraa_gpio_read(m_gpio);
}
+bool Button::isPressed()
+{
+ return (bool) Button::value();
+}
+
void Button::installISR(mraa::Edge level, void (*isr)(void *), void *arg)
{
if (m_isrInstalled)

diff:
#include <string>
#include <mraa/gpio.hpp>
+#include <interfaces/iButton.hpp>
namespace upm {
@@ -54,7 +55,7 @@ namespace upm {
* @image html button.jpg
* @snippet button.cxx Interesting
*/
-class Button{
+class Button : virtual public iButton {
public:
/**
* button constructor
@@ -79,6 +80,13 @@ class Button{
*/
int value();
+ /**
+ * Gets the current button press state.
+ *
+ * @returns Button state
+ */
+ virtual bool isPressed();
+
/**
* Installs an interrupt service routine (ISR) to be called when
* the button is activated or deactivated.

diff:
@@ -58,6 +58,11 @@ int GroveButton::value()
return mraa_gpio_read(m_gpio);
}
+bool GroveButton::isPressed()
+{
+ return (bool) GroveButton::value();
+}
+
void GroveButton::installISR(mraa::Edge level, void (*isr)(void *), void *arg)
{
if (m_isrInstalled)

diff:
#include <string>
#include <mraa/gpio.hpp>
#include "grovebase.hpp"
+#include <interfaces/iButton.hpp>
namespace upm {
@@ -51,7 +52,7 @@ namespace upm {
* @image html grovebutton.jpg
* @snippet grove-grovebutton.cxx Interesting
*/
-class GroveButton: public Grove {
+class GroveButton: public Grove, virtual public iButton {
public:
/**
* Grove button constructor
@@ -76,6 +77,13 @@ class GroveButton: public Grove {
*/
int value();
+ /**
+ * Gets the current button press state.
+ *
+ * @returns Button state
+ */
+ virtual bool isPressed();
+
/**
* Installs an interrupt service routine (ISR) to be called when
* the button is activated or deactivated.

diff:
#include <string>
#include <mraa/gpio.hpp>
+#include <interfaces/iButton.hpp>
namespace upm {
/**
@@ -52,7 +53,7 @@ namespace upm {
* @image html ttp223.jpg
* @snippet ttp223.cxx Interesting
*/
-class TTP223 {
+class TTP223 : virtual public iButton {
public:
/**
* TTP223 constructor
@@ -85,7 +86,7 @@ class TTP223 {
*
* @return True if touched, false otherwise
*/
- bool isPressed();
+ virtual bool isPressed();
/**
* Installs an interrupt service routine (ISR) to be called when

original_message:
Added interface iButton

Signed-off-by: Stefan Andritoiu <stefan.andritoiu@gmail.com>
Signed-off-by: Mihai Tudor Panu <mihai.tudor.panu@intel.com>


message: Added iVDiv interface

change_type: ADD

diff:
+/*
+ * Author: Mihai Stefanescu <mihai.stefanescu@rinftech.com>
+ * Copyright (c) 2018 Intel Corporation.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#pragma once
+
+namespace upm
+{
+/**
+ * @brief Interface for Voltage Dividing sensors
+*/
+ class iVDiv
+ {
+ public:
+ virtual ~iVDiv() {}
+
+ /**
+ * Gets the conversion value from the sensor
+ *
+ * @return ADC conversion value
+ */
+ virtual unsigned int getValue() = 0;
+
+ };
+}

change_type: MODIFY

diff:
@@ -60,9 +60,13 @@ unsigned int GroveVDiv::value(unsigned int samples)
return (sum / samples);
}
+unsigned int GroveVDiv::getValue()
+{
+ return GroveVDiv::value(1);
+}
+
float GroveVDiv::computedValue(uint8_t gain, unsigned int val, int vref, int res)
{
return ((float(gain) * float(val) * float(vref) / float(res)) / 1000.0);
}
-

diff:
#include <iostream>
#include <stdint.h>
#include <mraa/aio.h>
+#include <interfaces/iVDiv.hpp>
// reference voltage in millivolts
#define GROVEVDIV_VREF 4980
@@ -59,7 +60,7 @@ namespace upm {
* @image html grovevdiv.jpg
* @snippet grovevdiv.cxx Interesting
*/
- class GroveVDiv {
+ class GroveVDiv : virtual public iVDiv {
public:
/**
* Grove Voltage Divider sensor constructor
@@ -81,6 +82,13 @@ namespace upm {
*/
unsigned int value(unsigned int samples);
+ /**
+ * Gets the conversion value from the sensor
+ *
+ * @return ADC conversion value
+ */
+ virtual unsigned int getValue();
+
/**
* Computes the measured voltage
*

diff:
@@ -60,9 +60,13 @@ unsigned int VDiv::value(unsigned int samples)
return (sum / samples);
}
+unsigned int VDiv::getValue()
+{
+ return VDiv::value(1);
+}
+
float VDiv::computedValue(uint8_t gain, unsigned int val, int vref, int res)
{
return ((float(gain) * float(val) * float(vref) / float(res)) / 1000.0);
}
-

diff:
#include <iostream>
#include <stdint.h>
#include <mraa/aio.h>
+#include <interfaces/iVDiv.hpp>
// reference voltage in millivolts
#define VDIV_VREF 4980
@@ -58,7 +59,7 @@ namespace upm {
* @image html vdiv.jpg
* @snippet vdiv.cxx Interesting
*/
- class VDiv {
+ class VDiv : virtual public iVDiv {
public:
/**
* Voltage Divider sensor constructor
@@ -80,6 +81,13 @@ namespace upm {
*/
unsigned int value(unsigned int samples);
+ /**
+ * Gets the conversion value from the sensor
+ *
+ * @return ADC conversion value
+ */
+ virtual unsigned int getValue();
+
/**
* Computes the measured voltage
*

license: MIT License

repo: eclipse/upm

original_message:
Added iVDiv interface

Signed-off-by: Stefan Andritoiu <stefan.andritoiu@gmail.com>
Signed-off-by: Mihai Tudor Panu <mihai.tudor.panu@intel.com>


message: Added iWater interface

diff:
+/*
+ * Author: Stefan Andritoiu <stefan.andritoiu@gmail.com>
+ * Copyright (c) 2018 Intel Corporation.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#pragma once
+
+namespace upm
+{
+/**
+ * @brief Interface for Water sensors
+*/
+ class iWater
+ {
+ public:
+ virtual ~iWater() {}
+
+ /**
+ * Gets the water (wet/not wet) value from the sensor
+ *
+ * @return True if the sensor is wet, false otherwise
+ */
+ virtual bool isWet() = 0;
+
+ };
+}

diff:
#include <string>
#include "water.h"
+#include <interfaces/iWater.hpp>
namespace upm {
/**
@@ -49,7 +50,7 @@ namespace upm {
* @image html water.jpg
* @snippet water.cxx Interesting
*/
- class Water {
+ class Water : virtual public iWater {
public:
/**
* digital water sensor constructor
@@ -66,7 +67,7 @@ namespace upm {
*
* @return True if the sensor is wet, false otherwise
*/
- bool isWet();
+ virtual bool isWet();
private:
/* Disable implicit copy and assignment operators */

original_message:
Added iWater interface

Signed-off-by: Stefan Andritoiu <stefan.andritoiu@gmail.com>
Signed-off-by: Mihai Tudor Panu <mihai.tudor.panu@intel.com>


message: Added iGps interface

diff:
+/*
+ * Author: Stefan Andritoiu <stefan.andritoiu@gmail.com>
+ * Copyright (c) 2018 Intel Corporation.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#pragma once
+
+namespace upm
+{
+/**
+ * @brief Interface for GPS modules
+*/
+ class iGps
+ {
+ public:
+ virtual ~iGps() {}
+
+ /**
+ * Enable or disable the device.
+ *
+ * @param enable true to enable the device, false otherwise.
+ */
+ virtual void enable(bool enable) = 0;
+
+ /**
+ * Set the baudrate of the device.
+ *
+ * @param baudrate The baud rate to set for the device.
+ */
+ virtual void setBaudrate(unsigned int baudrate) = 0;
+
+ };
+}

diff:
#include <vector>
#include "nmea_gps.h"
+#include <interfaces/iGps.hpp>
namespace upm {
/**
@@ -192,7 +193,7 @@ namespace upm {
std::string __str__();
};
- class NMEAGPS {
+ class NMEAGPS : virtual public iGps {
public:
/**
@@ -254,7 +255,7 @@ namespace upm {
*
* @param enable true to enable the device, false otherwise.
*/
- void enable(bool enable);
+ virtual void enable(bool enable);
/**
* Set the baudrate of the device. By default, the constructor
@@ -263,7 +264,7 @@ namespace upm {
*
* @param baudrate The baud rate to set for the device.
*/
- void setBaudrate(unsigned int baudrate);
+ virtual void setBaudrate(unsigned int baudrate);
/**
* Determine whether there is data available to be read. In the

original_message:
Added iGps interface

Signed-off-by: Stefan Andritoiu <stefan.andritoiu@gmail.com>
Signed-off-by: Mihai Tudor Panu <mihai.tudor.panu@intel.com>


message: Added iElectromagnet interface

diff:
+/*
+ * Author: Mihai Stefanescu <mihai.stefanescu@rinftech.com>
+ * Copyright (c) 2018 Intel Corporation.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#pragma once
+
+namespace upm
+{
+/**
+ * @brief Interface for Electromagnet modules
+*/
+ class iElectromagnet
+ {
+ public:
+ virtual ~iElectromagnet() {}
+
+ /**
+ * Turns the magnet on
+ */
+ virtual void on() = 0;
+ /**
+ * Turns the magnet off
+ */
+ virtual void off() = 0;
+
+ };
+}

diff:
#pragma once
#include <mraa/gpio.h>
+#include <interfaces/iElectromagnet.hpp>
#define HIGH 1
#define LOW 0
@@ -50,7 +51,7 @@ namespace upm {
* @image html electromagnet.jpg
* @snippet electromagnet.cxx Interesting
*/
- class Electromagnet {
+ class Electromagnet : virtual public iElectromagnet {
public:
/**
* Electromagnet constructor
@@ -65,11 +66,11 @@ namespace upm {
/**
* Turns the magnet on
*/
- void on();
+ virtual void on();
/**
* Turns the magnet off
*/
- void off();
+ virtual void off();
private:
mraa_gpio_context m_gpio;

diff:
#pragma once
#include <mraa/gpio.h>
+#include <interfaces/iElectromagnet.hpp>
#define HIGH 1
#define LOW 0
@@ -52,7 +53,7 @@ namespace upm {
* @image html groveelectromagnet.jpg
* @snippet groveelectromagnet.cxx Interesting
*/
- class GroveElectromagnet {
+ class GroveElectromagnet : virtual public iElectromagnet {
public:
/**
* Grove Electromagnet constructor
@@ -67,11 +68,11 @@ namespace upm {
/**
* Turns the magnet on
*/
- void on();
+ virtual void on();
/**
* Turns the magnet off
*/
- void off();
+ virtual void off();
private:
mraa_gpio_context m_gpio;

original_message:
Added iElectromagnet interface

Signed-off-by: Stefan Andritoiu <stefan.andritoiu@gmail.com>
Signed-off-by: Mihai Tudor Panu <mihai.tudor.panu@intel.com>


message: Cleanup compiler warnings around naked function calls.

diff:
@@ -24,7 +24,7 @@ defmodule KafkaEx.Server0P8P0 do
alias KafkaEx.NetworkClient
def kafka_server_init([args]) do
- kafka_server_init([args, self])
+ kafka_server_init([args, self()])
end
def kafka_server_init([args, name]) do
@@ -85,7 +85,7 @@ defmodule KafkaEx.Server0P8P0 do
end
ref = Process.send_after(
- self, {:start_streaming, %{fetch_request | offset: offset}}, 500
+ self(), {:start_streaming, %{fetch_request | offset: offset}}, 500
)
{:noreply, %{state | stream_timer: ref}}

diff:
@@ -34,7 +34,7 @@ defmodule KafkaEx.Server0P8P2 do
end
def kafka_server_init([args]) do
- kafka_server_init([args, self])
+ kafka_server_init([args, self()])
end
def kafka_server_init([args, name]) do
@@ -132,7 +132,7 @@ defmodule KafkaEx.Server0P8P2 do
end
ref = Process.send_after(
- self, {:start_streaming, %{fetch_request | offset: offset}}, 500
+ self(), {:start_streaming, %{fetch_request | offset: offset}}, 500
)
{:noreply, %{state | stream_timer: ref}}

diff:
@@ -30,7 +30,7 @@ defmodule KafkaEx.Server0P9P0 do
end
def kafka_server_init([args]) do
- kafka_server_init([args, self])
+ kafka_server_init([args, self()])
end
def kafka_server_init([args, name]) do
@@ -167,7 +167,7 @@ defmodule KafkaEx.Server0P9P0 do
end
ref = Process.send_after(
- self, {:start_streaming, %{fetch_request | offset: offset}}, 500
+ self(), {:start_streaming, %{fetch_request | offset: offset}}, 500
)
{:noreply, %{state | stream_timer: ref}}

diff:
@@ -12,9 +12,9 @@ defmodule KafkaEx.Mixfile do
"-Wrace_conditions",
]
],
- description: description,
- package: package,
- deps: deps,
+ description: description(),
+ package: package(),
+ deps: deps(),
docs: [main: "KafkaEx"]]
end


message: Test against only latest patch releases on travis.


message: Add Dan Swain to list of maintainers

diff:
@@ -42,7 +42,7 @@ defmodule KafkaEx.Mixfile do
end
defp package do
- [maintainers: ["Abejide Ayodele", "Jack Lund"],
+ [maintainers: ["Abejide Ayodele", "Dan Swain", "Jack Lund"],
files: ["lib", "mix.exs", "README.md"],
licenses: ["MIT"],
links: %{"Github" => "https://github.com/kafkaex/kafka_ex"}]

original_message: Add Dan Swain to list of maintainers


message: Cleanup code duplicated between Server0P9P0 and Server0P8P2.

diff:
@@ -5,18 +5,14 @@ defmodule KafkaEx.Server0P9P0 do
use KafkaEx.Server
alias KafkaEx.Protocol.ConsumerMetadata
alias KafkaEx.Protocol.ConsumerMetadata.Response, as: ConsumerMetadataResponse
- alias KafkaEx.Protocol.Fetch
- alias KafkaEx.Protocol.Fetch.Request, as: FetchRequest
alias KafkaEx.Protocol.Heartbeat
alias KafkaEx.Protocol.JoinGroup
alias KafkaEx.Protocol.JoinGroup.Request, as: JoinGroupRequest
alias KafkaEx.Protocol.Metadata.Broker
- alias KafkaEx.Protocol.Metadata.Response, as: MetadataResponse
- alias KafkaEx.Protocol.OffsetFetch
- alias KafkaEx.Protocol.OffsetCommit
alias KafkaEx.Protocol.SyncGroup
alias KafkaEx.Server.State
alias KafkaEx.NetworkClient
+ alias KafkaEx.Server0P8P2
@consumer_group_update_interval 30_000
@@ -29,6 +25,15 @@ defmodule KafkaEx.Server0P9P0 do
GenServer.start_link(__MODULE__, [args, name], [name: name])
end
+ defdelegate kafka_server_consumer_group(state), to: Server0P8P2
+ defdelegate kafka_server_fetch(fetch_request, state), to: Server0P8P2
+ defdelegate kafka_server_offset_fetch(offset_fetch, state), to: Server0P8P2
+ defdelegate kafka_server_offset_commit(offset_commit_request, state), to: Server0P8P2
+
+ defdelegate kafka_server_consumer_group_metadata(state), to: Server0P8P2
+ defdelegate kafka_server_start_streaming(fetch_request, state), to: Server0P8P2
+ defdelegate kafka_server_update_consumer_metadata(state), to: Server0P8P2
+
def kafka_server_init([args]) do
kafka_server_init([args, self()])
end
@@ -61,54 +66,6 @@ defmodule KafkaEx.Server0P9P0 do
{:ok, state}
end
- def kafka_server_consumer_group(state) do
- {:reply, state.consumer_group, state}
- end
-
- def kafka_server_fetch(fetch_request, state) do
- true = consumer_group_if_auto_commit?(fetch_request.auto_commit, state)
- {response, state} = fetch(fetch_request, state)
-
- {:reply, response, state}
- end
-
- def kafka_server_offset_fetch(offset_fetch, state) do
- true = consumer_group?(state)
- {broker, state} = broker_for_consumer_group_with_update(state)
-
- # if the request is for a specific consumer group, use that
- # otherwise use the worker's consumer group
- consumer_group = offset_fetch.consumer_group || state.consumer_group
- offset_fetch = %{offset_fetch | consumer_group: consumer_group}
-
- offset_fetch_request = OffsetFetch.create_request(state.correlation_id, @client_id, offset_fetch)
-
- {response, state} = case broker do
- nil ->
- Logger.log(:error, "Coordinator for topic #{offset_fetch.topic} is not available")
- {:topic_not_found, state}
- _ ->
- response = broker
- |> NetworkClient.send_sync_request(offset_fetch_request, state.sync_timeout)
- |> OffsetFetch.parse_response
- {response, %{state | correlation_id: state.correlation_id + 1}}
- end
-
- {:reply, response, state}
- end
-
- def kafka_server_offset_commit(offset_commit_request, state) do
- {response, state} = offset_commit(state, offset_commit_request)
-
- {:reply, response, state}
- end
-
- def kafka_server_consumer_group_metadata(state) do
- true = consumer_group?(state)
- {consumer_metadata, state} = update_consumer_metadata(state)
- {:reply, consumer_metadata, state}
- end
-
def kafka_server_join_group(topics, session_timeout, state) do
true = consumer_group?(state)
{broker, state} = broker_for_consumer_group_with_update(state)
@@ -146,39 +103,6 @@ defmodule KafkaEx.Server0P9P0 do
{:reply, response, %{state | correlation_id: state.correlation_id + 1}}
end
- def kafka_server_start_streaming(_, state = %State{event_pid: nil}) do
- # our streaming could have been canceled with a streaming update in-flight
- {:noreply, state}
- end
- def kafka_server_start_streaming(fetch_request, state) do
- true = consumer_group_if_auto_commit?(fetch_request.auto_commit, state)
-
- {response, state} = fetch(fetch_request, state)
- offset = case response do
- :topic_not_found ->
- fetch_request.offset
- _ ->
- message = response |> hd |> Map.get(:partitions) |> hd
- Enum.each(message.message_set, fn(message_set) -> GenEvent.notify(state.event_pid, message_set) end)
- case message.last_offset do
- nil -> fetch_request.offset
- last_offset -> last_offset + 1
- end
- end
-
- ref = Process.send_after(
- self(), {:start_streaming, %{fetch_request | offset: offset}}, 500
- )
-
- {:noreply, %{state | stream_timer: ref}}
- end
-
- def kafka_server_update_consumer_metadata(state) do
- true = consumer_group?(state)
- {_, state} = update_consumer_metadata(state)
- {:noreply, state}
- end
-
defp update_consumer_metadata(state), do: update_consumer_metadata(state, @retry_count, 0)
defp update_consumer_metadata(state = %State{consumer_group: consumer_group}, 0, error_code) do
@@ -199,64 +123,6 @@ defmodule KafkaEx.Server0P9P0 do
end
end
- defp updated_broker_for_topic(state, partition, topic) do
- case MetadataResponse.broker_for_topic(state.metadata, state.brokers, topic, partition) do
- nil ->
- updated_state = update_metadata(state)
- {MetadataResponse.broker_for_topic(updated_state.metadata, updated_state.brokers, topic, partition), updated_state}
- broker -> {broker, state}
- end
- end
-
- defp fetch(fetch_request, state) do
- true = consumer_group_if_auto_commit?(fetch_request.auto_commit, state)
- fetch_data = Fetch.create_request(%FetchRequest{
- fetch_request |
- client_id: @client_id,
- correlation_id: state.correlation_id,
- })
- {broker, state} = updated_broker_for_topic(state, fetch_request.partition, fetch_request.topic)
-
- case broker do
- nil ->
- Logger.log(:error, "Leader for topic #{fetch_request.topic} is not available")
- {:topic_not_found, state}
- _ ->
- response = broker
- |> NetworkClient.send_sync_request(fetch_data, state.sync_timeout)
- |> Fetch.parse_response
- state = %{state | correlation_id: state.correlation_id + 1}
- last_offset = response |> hd |> Map.get(:partitions) |> hd |> Map.get(:last_offset)
- if last_offset != nil && fetch_request.auto_commit do
- offset_commit_request = %OffsetCommit.Request{
- topic: fetch_request.topic,
- offset: last_offset,
- partition: fetch_request.partition,
- consumer_group: state.consumer_group}
- {_, state} = offset_commit(state, offset_commit_request)
- {response, state}
- else
- {response, state}
- end
- end
- end
-
- defp offset_commit(state, offset_commit_request) do
- {broker, state} = broker_for_consumer_group_with_update(state, true)
-
- # if the request has a specific consumer group, use that
- # otherwise use the worker's consumer group
- consumer_group = offset_commit_request.consumer_group || state.consumer_group
- offset_commit_request = %{offset_commit_request | consumer_group: consumer_group}
-
- offset_commit_request_payload = OffsetCommit.create_request(state.correlation_id, @client_id, offset_commit_request)
- response = broker
- |> NetworkClient.send_sync_request(offset_commit_request_payload, state.sync_timeout)
- |> OffsetCommit.parse_response
-
- {response, %{state | correlation_id: state.correlation_id + 1}}
- end
-
defp broker_for_consumer_group(state) do
ConsumerMetadataResponse.broker_for_consumer_group(state.brokers, state.consumer_metadata)
end


message: Add comment depicting where to find the delegated functions.

diff:
@@ -25,11 +25,13 @@ defmodule KafkaEx.Server0P9P0 do
GenServer.start_link(__MODULE__, [args, name], [name: name])
end
+ # The functions below are all defined in KafkaEx.Server0P8P2 and their
+ # implementation is exactly same across both versions of kafka.
+
defdelegate kafka_server_consumer_group(state), to: Server0P8P2
defdelegate kafka_server_fetch(fetch_request, state), to: Server0P8P2
defdelegate kafka_server_offset_fetch(offset_fetch, state), to: Server0P8P2
defdelegate kafka_server_offset_commit(offset_commit_request, state), to: Server0P8P2
-
defdelegate kafka_server_consumer_group_metadata(state), to: Server0P8P2
defdelegate kafka_server_start_streaming(fetch_request, state), to: Server0P8P2
defdelegate kafka_server_update_consumer_metadata(state), to: Server0P8P2


diff:
@@ -262,8 +262,6 @@ Optional arguments(KeywordList)
Optional arguments(KeywordList)
- worker_name: the worker we want to run this metadata request through, when none is provided the default worker `:kafka_ex` is used
- offset: When supplied the fetch would start from this offset, otherwise would start from the last committed offset of the consumer_group the worker belongs to. For Kafka < 0.8.2 you should explicitly specify this.
- - handler: the handler we want to handle the streaming events, when none is provided the default KafkaEx.Handler is used
- - handler_init: initial state for the handler - leave the default value [] when using the default handler
- auto_commit: specifies if the last offset should be commited or not. Default is true. You must set this to false when using Kafka < 0.8.2 or `:no_consumer_group`.
@@ -283,32 +281,21 @@ Optional arguments(KeywordList)
def stream(topic, partition, opts \\ []) do
worker_name = Keyword.get(opts, :worker_name, Config.default_worker)
supplied_offset = Keyword.get(opts, :offset)
- handler = Keyword.get(opts, :handler, KafkaEx.Handler)
- handler_init = Keyword.get(opts, :handler_init, [])
auto_commit = Keyword.get(opts, :auto_commit, true)
wait_time = Keyword.get(opts, :wait_time, @wait_time)
min_bytes = Keyword.get(opts, :min_bytes, @min_bytes)
max_bytes = Keyword.get(opts, :max_bytes, @max_bytes)
-
- event_stream = Server.call(worker_name, {:create_stream, handler, handler_init})
+ consumer_group = Keyword.get(opts, :consumer_group)
retrieved_offset = current_offset(supplied_offset, partition, topic, worker_name)
- send(worker_name, {
- :start_streaming,
- %FetchRequest{
+ fetch_request = %FetchRequest{
auto_commit: auto_commit,
topic: topic, partition: partition,
offset: retrieved_offset, wait_time: wait_time,
min_bytes: min_bytes, max_bytes: max_bytes
}
- })
- event_stream
- end
- @spec stop_streaming(Keyword.t) :: :stop_streaming
- def stop_streaming(opts \\ []) do
- worker_name = Keyword.get(opts, :worker_name, Config.default_worker)
- send(worker_name, :stop_streaming)
+ %KafkaEx.Stream{worker_name: worker_name, fetch_request: fetch_request, consumer_group: consumer_group}
end
defp build_worker_options(worker_init) do

diff:
@@ -9,13 +9,23 @@ defmodule KafkaEx.Stream do
def reduce(data, acc, fun) do
next_fun = fn offset ->
if data.fetch_request.auto_commit do
- GenServer.call(data.worker_name, {:offset_commit, %OffsetCommitRequest{consumer_group: data.consumer_group, topic: data.fetch_request.topic, partition: data.fetch_request.partition, offset: offset, metadata: ""}})
+ GenServer.call(data.worker_name, {
+ :offset_commit,
+ %OffsetCommitRequest{
+ consumer_group: data.consumer_group,
+ topic: data.fetch_request.topic,
+ partition: data.fetch_request.partition,
+ offset: offset, metadata: ""
+ }
+ })
end
- response = GenServer.call(data.worker_name, {:fetch, %{data.fetch_request| offset: offset}})
- last_offset = response |> hd |> Map.get(:partitions) |> hd |> Map.get(:last_offset)
- {response, last_offset}
+ response = data.worker_name
+ |> GenServer.call({:fetch, %{data.fetch_request| offset: offset}})
+ |> hd |> Map.get(:partitions) |> hd
+ last_offset = response |> Map.get(:last_offset)
+ {response.message_set, last_offset}
end
- Stream.resource(fn -> data.fetch_request.offset end, next_fun, fn offset -> offset end).(acc, fun)
+ Stream.resource(&(data.fetch_request.offset), next_fun, &(&1)).(acc, fun)
end
def count(_stream) do
{:error, __MODULE__}

diff:
@@ -211,7 +211,7 @@ defmodule KafkaEx.ConsumerGroup.Test do
})
stream = KafkaEx.stream(random_string, 0, worker_name: :stream_auto_commit, offset: 0)
log = TestHelper.wait_for_any(
- fn() -> GenEvent.call(stream.manager, KafkaEx.Handler, :messages) |> Enum.take(2) end
+ fn() -> stream |> Enum.take(2) end
)
refute Enum.empty?(log)
@@ -240,13 +240,11 @@ defmodule KafkaEx.ConsumerGroup.Test do
stream = KafkaEx.stream(random_string, 0, worker_name: worker_name)
log = TestHelper.wait_for_any(
- fn() -> GenEvent.call(stream.manager, KafkaEx.Handler, :messages) |> Enum.take(2) end
+ fn() -> stream |> Enum.take(2) end
)
refute Enum.empty?(log)
-
first_message = log |> hd
-
assert first_message.offset == 4
end
@@ -258,7 +256,7 @@ defmodule KafkaEx.ConsumerGroup.Test do
# make sure we consume at least one message before we assert that there is no offset committed
_log = TestHelper.wait_for_any(
- fn() -> GenEvent.call(stream.manager, KafkaEx.Handler, :messages) |> Enum.take(2) end
+ fn() -> stream |> Enum.take(2) end
)
offset_fetch_response = KafkaEx.offset_fetch(:stream_no_auto_commit, %Proto.OffsetFetch.Request{topic: random_string, partition: 0}) |> hd

diff:
@@ -339,7 +339,7 @@ defmodule KafkaEx.Integration.Test do
stream = KafkaEx.stream(random_string, 0, worker_name: :stream, offset: 0, auto_commit: false)
log = TestHelper.wait_for_accum(
- fn() -> GenEvent.call(stream.manager, KafkaEx.Handler, :messages) end,
+ fn() -> stream |> Enum.take(2) end,
2
)
@@ -354,85 +354,4 @@ defmodule KafkaEx.Integration.Test do
KafkaEx.stream(random_string, 0, offset: 0)
KafkaEx.stream(random_string, 0, offset: 0)
end
-
- test "stop_streaming stops streaming, and stream starts it up again" do
- random_string = generate_random_string()
- KafkaEx.create_worker(:stream2, uris: uris())
- stream = KafkaEx.stream(random_string, 0, worker_name: :stream2, offset: 0, auto_commit: false)
-
- KafkaEx.create_worker(:producer, uris: uris())
- KafkaEx.produce(%Proto.Produce.Request{topic: random_string, partition: 0, required_acks: 1, messages: [
- %Proto.Produce.Message{value: "one"},
- %Proto.Produce.Message{value: "two"},
- ]
- }, worker_name: :producer)
-
- log = TestHelper.wait_for_accum(
- fn() -> GenEvent.call(stream.manager, KafkaEx.Handler, :messages) end,
- 2
- )
-
- last_offset = hd(Enum.reverse(log)).offset
-
- KafkaEx.stop_streaming(worker_name: :stream2)
- :ok = TestHelper.wait_for(fn() -> !Process.alive?(stream.manager) end)
-
- KafkaEx.produce(%Proto.Produce.Request{topic: random_string, partition: 0, required_acks: 1, messages: [
- %Proto.Produce.Message{value: "three"},
- %Proto.Produce.Message{value: "four"},
- ]
- }, worker_name: :producer)
-
- stream = KafkaEx.stream(random_string, 0, worker_name: :stream2, offset: last_offset+1, auto_commit: false)
-
- :ok = TestHelper.wait_for(fn() -> Process.alive?(stream.manager) end)
-
- log = GenEvent.call(stream.manager, KafkaEx.Handler, :messages)
- assert length(log) == 0
-
- KafkaEx.produce(%Proto.Produce.Request{topic: random_string, partition: 0, required_acks: 1, messages: [
- %Proto.Produce.Message{value: "five"},
- %Proto.Produce.Message{value: "six"},
- ]
- }, worker_name: :producer)
-
- log = TestHelper.wait_for_accum(
- fn() -> GenEvent.call(stream.manager, KafkaEx.Handler, :messages) end,
- 4
- )
-
- assert length(log) == 4
- end
-
- test "streams kafka logs with custom handler and initial state" do
- random_string = generate_random_string()
- KafkaEx.create_worker(:stream3, uris: uris())
- {:ok, offset} = KafkaEx.produce(%Proto.Produce.Request{topic: random_string, partition: 0, required_acks: 1, messages: [
- %Proto.Produce.Message{value: "hey"},
- %Proto.Produce.Message{value: "hi"},
- ]
- }, worker_name: :stream3)
-
- defmodule CustomHandlerSendingMessage do
- use GenEvent
-
- def init(pid) do
- {:ok, pid}
- end
-
- def handle_event(message, pid) do
- send(pid, message)
- {:ok, pid}
- end
- end
- KafkaEx.stream(random_string, 0, worker_name: :stream3, offset: offset, auto_commit: false, handler: CustomHandlerSendingMessage, handler_init: self())
-
- assert_receive %KafkaEx.Protocol.Fetch.Message{key: nil, value: "hey", offset: ^offset, attributes: 0, crc: 4264455069}
- offset = offset + 1
- assert_receive %KafkaEx.Protocol.Fetch.Message{key: nil, value: "hi", offset: ^offset, attributes: 0, crc: 4251893211}
- receive do
- _ -> assert(false, "Should not have received a third message")
- after 100 -> :ok
- end
- end
end

original_message:
Replace the current GenEvent based stream implementation with a really
lazy stream implementation, this closes #145.


message: Replace short form of function call with the explicit form.

diff:
@@ -25,7 +25,7 @@ defmodule KafkaEx.Stream do
last_offset = response |> Map.get(:last_offset)
{response.message_set, last_offset}
end
- Stream.resource(&(data.fetch_request.offset), next_fun, &(&1)).(acc, fun)
+ Stream.resource(fn -> data.fetch_request.offset end, next_fun, &(&1)).(acc, fun)
end
def count(_stream) do
{:error, __MODULE__}


message:
Update the documentation on the stream function to reflect it's halting
behavior and also update the implementation to halt if there are no more
messages to stream.

diff:
@@ -15,6 +15,7 @@ defmodule KafkaEx do
alias KafkaEx.Protocol.Produce.Request, as: ProduceRequest
alias KafkaEx.Protocol.Produce.Message
alias KafkaEx.Server
+ alias KafkaEx.Stream
@type uri() :: [{binary|char_list, number}]
@type worker_init :: [worker_setting]
@@ -253,7 +254,7 @@ Optional arguments(KeywordList)
end
@doc """
- Returns a stream that consumes fetched messages.
+ Returns a stream that consumes fetched messages, the stream will halt once the max_bytes number of messages is reached, if you want to halt the stream early supply a small max_bytes, for the inverse supply a large max_bytes and if you don't want the stream to halt, you should recursively call the stream function.
Optional arguments(KeywordList)
- worker_name: the worker we want to run this metadata request through, when none is provided the default worker `:kafka_ex` is used
- offset: When supplied the fetch would start from this offset, otherwise would start from the last committed offset of the consumer_group the worker belongs to. For Kafka < 0.8.2 you should explicitly specify this.
@@ -291,7 +292,7 @@ Optional arguments(KeywordList)
min_bytes: min_bytes, max_bytes: max_bytes
}
- %KafkaEx.Stream{worker_name: worker_name, fetch_request: fetch_request, consumer_group: consumer_group}
+ %Stream{worker_name: worker_name, fetch_request: fetch_request, consumer_group: consumer_group}
end
defp build_worker_options(worker_init) do

diff:
@@ -22,11 +22,16 @@ defmodule KafkaEx.Stream do
response = data.worker_name
|> GenServer.call({:fetch, %{data.fetch_request| offset: offset}})
|> hd |> Map.get(:partitions) |> hd
- last_offset = response |> Map.get(:last_offset)
- {response.message_set, last_offset}
+ if response.error_code == :no_error &&
+ response.last_offset != nil && response.last_offset != offset do
+ {response.message_set, response.last_offset}
+ else
+ {:halt, offset}
+ end
end
Stream.resource(fn -> data.fetch_request.offset end, next_fun, &(&1)).(acc, fun)
end
+
def count(_stream) do
{:error, __MODULE__}
end


message: Delete the dead stream code.

change_type: DELETE

diff:
-defmodule KafkaEx.Handler do
- @moduledoc """
- Default GenEvent handler for KafkaEx.stream
-
- Received message sets are accumulated in the GenEvent state and can
- be retrieved and flushed via the `:messages` call.
- """
-
- use GenEvent
-
- def handle_event(message_set, message_sets) do
- {:ok, [message_set|message_sets]}
- end
-
- def handle_call(:messages, message_sets) do
- {:ok, Enum.reverse(message_sets), []}
- end
-end

diff:
@@ -21,7 +21,6 @@ defmodule KafkaEx.Server do
metadata: %Metadata.Response{},
brokers: [],
event_pid: nil,
- stream_timer: nil,
consumer_metadata: %ConsumerMetadata.Response{},
correlation_id: 0,
consumer_group: nil,
@@ -36,7 +35,6 @@ defmodule KafkaEx.Server do
metadata: Metadata.Response.t,
brokers: [Broker.t],
event_pid: nil | pid,
- stream_timer: reference,
consumer_metadata: ConsumerMetadata.Response.t,
correlation_id: integer,
metadata_update_interval: nil | integer,
@@ -136,21 +134,6 @@ defmodule KafkaEx.Server do
{:noreply, new_state, timeout | :hibernate} |
{:stop, reason, reply, new_state} |
{:stop, reason, new_state} when reply: term, new_state: term, reason: term
- @callback kafka_server_create_stream(handler :: term, handler_init :: term, state :: State.t) ::
- {:reply, reply, new_state} |
- {:reply, reply, new_state, timeout | :hibernate} |
- {:noreply, new_state} |
- {:noreply, new_state, timeout | :hibernate} |
- {:stop, reason, reply, new_state} |
- {:stop, reason, new_state} when reply: term, new_state: term, reason: term
- @callback kafka_server_start_streaming(fetch_request :: FetchRequest.t, state :: State.t) ::
- {:noreply, new_state} |
- {:noreply, new_state, timeout | :hibernate} |
- {:stop, reason :: term, new_state} when new_state: term
- @callback kafka_server_stop_streaming(state :: State.t) ::
- {:noreply, new_state} |
- {:noreply, new_state, timeout | :hibernate} |
- {:stop, reason :: term, new_state} when new_state: term
@callback kafka_server_update_metadata(state :: State.t) ::
{:noreply, new_state} |
{:noreply, new_state, timeout | :hibernate} |
@@ -253,24 +236,6 @@ defmodule KafkaEx.Server do
kafka_server_heartbeat(group_name, generation_id, member_id, state)
end
- def handle_call({:create_stream, handler, handler_init}, _from, state) do
- kafka_server_create_stream(handler, handler_init, state)
- end
-
- def handle_info({:start_streaming, fetch_request}, state) do
- kafka_server_start_streaming(fetch_request, state)
- end
-
- def handle_info(:stop_streaming, state) do
- {:noreply, state} = kafka_server_stop_streaming(state)
- state = case state.stream_timer do
- nil -> state
- ref -> Process.cancel_timer(ref)
- %{state | stream_timer: nil}
- end
- {:noreply, state}
- end
-
def handle_info(:update_metadata, state) do
kafka_server_update_metadata(state)
end
@@ -360,25 +325,6 @@ defmodule KafkaEx.Server do
{:reply, metadata, updated_state}
end
- def kafka_server_create_stream(handler, handler_init, state) do
- new_state = if state.event_pid && Process.alive?(state.event_pid) do
- Logger.log(:warn, "'#{state.worker_name}' already streaming handler '#{handler}'")
- state
- else
- {:ok, event_pid} = GenEvent.start_link
- updated_state = %{state | event_pid: event_pid}
- :ok = GenEvent.add_handler(updated_state.event_pid, handler, handler_init)
- updated_state
- end
- {:reply, GenEvent.stream(new_state.event_pid), new_state}
- end
-
- def kafka_server_stop_streaming(state) do
- Logger.log(:debug, "Stopped worker #{inspect state.worker_name} from streaming")
- GenEvent.stop(state.event_pid)
- {:noreply, %{state | event_pid: nil}}
- end
-
def kafka_server_update_metadata(state) do
{:noreply, update_metadata(state)}
end
@@ -419,8 +365,7 @@ defmodule KafkaEx.Server do
defoverridable [
kafka_server_produce: 2, kafka_server_offset: 4,
- kafka_server_metadata: 2, kafka_server_create_stream: 3,
- kafka_server_stop_streaming: 1, kafka_server_update_metadata: 1,
+ kafka_server_metadata: 2, kafka_server_update_metadata: 1,
]
defp remove_stale_brokers(brokers, metadata_brokers) do

diff:
@@ -67,31 +67,6 @@ defmodule KafkaEx.Server0P8P0 do
def kafka_server_heartbeat(_, _, _, _state), do: raise "Heartbeat is not supported in 0.8.0 version of kafka"
def kafka_server_update_consumer_metadata(_state), do: raise "Consumer Group Metadata is not supported in 0.8.0 version of kafka"
- def kafka_server_start_streaming(_, state = %State{event_pid: nil}) do
- # our streaming could have been canceled with a streaming update in-flight
- {:noreply, state}
- end
- def kafka_server_start_streaming(fetch_request, state) do
- {response, state} = fetch(fetch_request, state)
- offset = case response do
- :topic_not_found ->
- fetch_request.offset
- _ ->
- message = response |> hd |> Map.get(:partitions) |> hd
- Enum.each(message.message_set, fn(message_set) -> GenEvent.notify(state.event_pid, message_set) end)
- case message.last_offset do
- nil -> fetch_request.offset
- last_offset -> last_offset + 1
- end
- end
-
- ref = Process.send_after(
- self(), {:start_streaming, %{fetch_request | offset: offset}}, 500
- )
-
- {:noreply, %{state | stream_timer: ref}}
- end
-
defp fetch(fetch_request, state) do
fetch_data = Fetch.create_request(%FetchRequest{
fetch_request |

diff:
@@ -111,33 +111,6 @@ defmodule KafkaEx.Server0P8P2 do
{:reply, consumer_metadata, state}
end
- def kafka_server_start_streaming(_, state = %State{event_pid: nil}) do
- # our streaming could have been canceled with a streaming update in-flight
- {:noreply, state}
- end
- def kafka_server_start_streaming(fetch_request, state) do
- true = consumer_group_if_auto_commit?(fetch_request.auto_commit, state)
-
- {response, state} = fetch(fetch_request, state)
- offset = case response do
- :topic_not_found ->
- fetch_request.offset
- _ ->
- message = response |> hd |> Map.get(:partitions) |> hd
- Enum.each(message.message_set, fn(message_set) -> GenEvent.notify(state.event_pid, message_set) end)
- case message.last_offset do
- nil -> fetch_request.offset
- last_offset -> last_offset + 1
- end
- end
-
- ref = Process.send_after(
- self(), {:start_streaming, %{fetch_request | offset: offset}}, 500
- )
-
- {:noreply, %{state | stream_timer: ref}}
- end
-
def kafka_server_update_consumer_metadata(state) do
true = consumer_group?(state)
{_, state} = update_consumer_metadata(state)

diff:
@@ -34,7 +34,6 @@ defmodule KafkaEx.Server0P9P0 do
defdelegate kafka_server_offset_fetch(offset_fetch, state), to: Server0P8P2
defdelegate kafka_server_offset_commit(offset_commit_request, state), to: Server0P8P2
defdelegate kafka_server_consumer_group_metadata(state), to: Server0P8P2
- defdelegate kafka_server_start_streaming(fetch_request, state), to: Server0P8P2
defdelegate kafka_server_update_consumer_metadata(state), to: Server0P8P2
def kafka_server_init([args]) do

original_message: Delete the dead stream code.


message:
Refactor Stream.reduce and create an helper
Fetch.Response.partition_messages function.

diff:
@@ -25,6 +25,12 @@ defmodule KafkaEx.Protocol.Fetch do
@moduledoc false
defstruct topic: nil, partitions: []
@type t :: %Response{topic: binary, partitions: list}
+
+ @spec partition_messages(list, binary, integer) :: list
+ def partition_messages(responses, topic, partition) do
+ response = Enum.find(responses, &(&1.topic == topic)) || %Response{}
+ Enum.find(response.partitions, &(&1.partition == partition))
+ end
end
defmodule Message do

diff:
@@ -2,8 +2,11 @@ defmodule KafkaEx.Stream do
@moduledoc false
alias KafkaEx.Protocol.OffsetCommit.Request, as: OffsetCommitRequest
alias KafkaEx.Protocol.Fetch.Request, as: FetchRequest
+ alias KafkaEx.Protocol.Fetch.Response, as: FetchResponse
- defstruct worker_name: nil, fetch_request: %FetchRequest{}, consumer_group: nil
+ defstruct worker_name: nil,
+ fetch_request: %FetchRequest{},
+ consumer_group: nil
defimpl Enumerable do
def reduce(data, acc, fun) do
@@ -19,9 +22,7 @@ defmodule KafkaEx.Stream do
}
})
end
- response = data.worker_name
- |> GenServer.call({:fetch, %{data.fetch_request| offset: offset}})
- |> hd |> Map.get(:partitions) |> hd
+ response = fetch_response(data, offset)
if response.error_code == :no_error &&
response.last_offset != nil && response.last_offset != offset do
{response.message_set, response.last_offset}
@@ -29,7 +30,16 @@ defmodule KafkaEx.Stream do
{:halt, offset}
end
end
- Stream.resource(fn -> data.fetch_request.offset end, next_fun, &(&1)).(acc, fun)
+ Stream.resource(
+ fn -> data.fetch_request.offset end, next_fun, &(&1)
+ ).(acc, fun)
+ end
+
+ defp fetch_response(data, offset) do
+ req = data.fetch_request
+ data.worker_name
+ |> GenServer.call({:fetch, %{req| offset: offset}})
+ |> FetchResponse.partition_messages(req.topic, req.partition)
end
def count(_stream) do


message:
Introduce stream_mode support, exposing control of finite and infinite
stream to the user.

diff:
@@ -254,12 +254,13 @@ Optional arguments(KeywordList)
end
@doc """
- Returns a stream that consumes fetched messages, the stream will halt once the max_bytes number of messages is reached, if you want to halt the stream early supply a small max_bytes, for the inverse supply a large max_bytes and if you don't want the stream to halt, you should recursively call the stream function.
+ Returns a stream that consumes fetched messages, the stream will halt once the max_bytes number of messages is reached, if you want to halt the stream early supply a small max_bytes, for the inverse supply a large max_bytes.
Optional arguments(KeywordList)
+ - stream_mode: the mode the stream will be in, `:infinite` for an infinite stream and `:finite` for a finite stream, a finite stream will return once there's no data left to consume from the partition while an infinite stream will block till there's data to consume from the stream.
- worker_name: the worker we want to run this metadata request through, when none is provided the default worker `:kafka_ex` is used
- offset: When supplied the fetch would start from this offset, otherwise would start from the last committed offset of the consumer_group the worker belongs to. For Kafka < 0.8.2 you should explicitly specify this.
- auto_commit: specifies if the last offset should be commited or not. Default is true. You must set this to false when using Kafka < 0.8.2 or `:no_consumer_group`.
- - consumer_group: Name of the group of consumers, `:no_consumer_group` should be passed for Kafka < 0.8.2, defaults to `Application.get_env(:kafka_ex, :consumer_group)`
+ - consumer_group: Name of the group of consumers, `:no_consumer_group` should be passed for Kafka < 0.8.2, defaults to `Application.get_env(:kafka_ex, :consumer_group)`.
## Example
@@ -276,14 +277,15 @@ Optional arguments(KeywordList)
"""
@spec stream(binary, integer, Keyword.t) :: GenEvent.Stream.t
def stream(topic, partition, opts \\ []) do
- worker_name = Keyword.get(opts, :worker_name, Config.default_worker)
- supplied_offset = Keyword.get(opts, :offset)
auto_commit = Keyword.get(opts, :auto_commit, true)
- wait_time = Keyword.get(opts, :wait_time, @wait_time)
- min_bytes = Keyword.get(opts, :min_bytes, @min_bytes)
- max_bytes = Keyword.get(opts, :max_bytes, @max_bytes)
consumer_group = Keyword.get(opts, :consumer_group)
+ max_bytes = Keyword.get(opts, :max_bytes, @max_bytes)
+ min_bytes = Keyword.get(opts, :min_bytes, @min_bytes)
+ supplied_offset = Keyword.get(opts, :offset)
+ worker_name = Keyword.get(opts, :worker_name, Config.default_worker)
retrieved_offset = current_offset(supplied_offset, partition, topic, worker_name)
+ stream_mode = Keyword.get(opts, :stream_mode, :infinite)
+ wait_time = Keyword.get(opts, :wait_time, @wait_time)
fetch_request = %FetchRequest{
auto_commit: auto_commit,
@@ -292,7 +294,10 @@ Optional arguments(KeywordList)
min_bytes: min_bytes, max_bytes: max_bytes
}
- %Stream{worker_name: worker_name, fetch_request: fetch_request, consumer_group: consumer_group}
+ %Stream{
+ worker_name: worker_name, fetch_request: fetch_request,
+ consumer_group: consumer_group, stream_mode: stream_mode
+ }
end
defp build_worker_options(worker_init) do

diff:
@@ -6,7 +6,8 @@ defmodule KafkaEx.Stream do
defstruct worker_name: nil,
fetch_request: %FetchRequest{},
- consumer_group: nil
+ consumer_group: nil,
+ stream_mode: :infinite
defimpl Enumerable do
def reduce(data, acc, fun) do
@@ -27,7 +28,7 @@ defmodule KafkaEx.Stream do
response.last_offset != nil && response.last_offset != offset do
{response.message_set, response.last_offset}
else
- {:halt, offset}
+ {mode(data.stream_mode), offset}
end
end
Stream.resource(
@@ -35,6 +36,9 @@ defmodule KafkaEx.Stream do
).(acc, fun)
end
+ defp mode(:finite), do: :halt
+ defp mode(:infinite), do: []
+
defp fetch_response(data, offset) do
req = data.fetch_request
data.worker_name


message: Use a function call when a pipeline is only one function long

diff:
@@ -211,7 +211,7 @@ defmodule KafkaEx.ConsumerGroup.Test do
})
stream = KafkaEx.stream(random_string, 0, worker_name: :stream_auto_commit, offset: 0)
log = TestHelper.wait_for_any(
- fn() -> stream |> Enum.take(2) end
+ fn() -> Enum.take(stream, 2) end
)
refute Enum.empty?(log)
@@ -240,7 +240,7 @@ defmodule KafkaEx.ConsumerGroup.Test do
stream = KafkaEx.stream(random_string, 0, worker_name: worker_name)
log = TestHelper.wait_for_any(
- fn() -> stream |> Enum.take(2) end
+ fn() -> Enum.take(stream, 2) end
)
refute Enum.empty?(log)
@@ -256,7 +256,7 @@ defmodule KafkaEx.ConsumerGroup.Test do
# make sure we consume at least one message before we assert that there is no offset committed
_log = TestHelper.wait_for_any(
- fn() -> stream |> Enum.take(2) end
+ fn() -> Enum.take(stream, 2) end
)
offset_fetch_response = KafkaEx.offset_fetch(:stream_no_auto_commit, %Proto.OffsetFetch.Request{topic: random_string, partition: 0}) |> hd


message: Add preliminary support for CreateTopics API

diff:
@@ -63,7 +63,7 @@ config :kafka_ex,
],
# set this to the version of the kafka broker that you are using
# include only major.minor.patch versions. must be at least 0.8.0
- kafka_version: "0.9.0"
+ kafka_version: "0.10.1"
env_config = Path.expand("#{Mix.env}.exs", __DIR__)
if File.exists?(env_config) do

diff:
@@ -5,32 +5,32 @@ services:
ports:
- "2181:2181"
kafka1:
- image: wurstmeister/kafka:0.9.0.1
+ image: wurstmeister/kafka:0.10.1.0
ports:
- "9092:9092"
depends_on:
- zookeeper
volumes:
- - ./kafka1/server.properties.in:/opt/kafka_2.11-0.9.0.1/config/server.properties.in
+ - ./kafka1/server.properties.in:/opt/kafka_2.11-0.10.1.0/config/server.properties.in
- ./scripts/docker-start-kafka.sh:/usr/bin/start-kafka.sh
- ./ssl:/ssl
kafka2:
- image: wurstmeister/kafka:0.9.0.1
+ image: wurstmeister/kafka:0.10.1.0
ports:
- "9093:9093"
depends_on:
- zookeeper
volumes:
- - ./kafka2/server.properties.in:/opt/kafka_2.11-0.9.0.1/config/server.properties.in
+ - ./kafka2/server.properties.in:/opt/kafka_2.11-0.10.1.0/config/server.properties.in
- ./scripts/docker-start-kafka.sh:/usr/bin/start-kafka.sh
- ./ssl:/ssl
kafka3:
- image: wurstmeister/kafka:0.9.0.1
+ image: wurstmeister/kafka:0.10.1.0
ports:
- "9094:9094"
depends_on:
- zookeeper
volumes:
- - ./kafka3/server.properties.in:/opt/kafka_2.11-0.9.0.1/config/server.properties.in
+ - ./kafka3/server.properties.in:/opt/kafka_2.11-0.10.1.0/config/server.properties.in
- ./scripts/docker-start-kafka.sh:/usr/bin/start-kafka.sh
- ./ssl:/ssl

diff:
@@ -527,6 +527,23 @@ defmodule KafkaEx do
def valid_consumer_group?(b) when is_binary(b), do: byte_size(b) > 0
def valid_consumer_group?(_), do: false
+
+ @doc """
+
+ ## Example
+
+ ```elixir
+ iex> KafkaEx.create_worker(:mt)
+
+ ```
+ """
+ @spec create_topics(String.t) :: CreateTopics.Response.t
+ def create_topics(topic_name, opts \\ []) do
+ worker_name = Keyword.get(opts, :worker_name, Config.default_worker)
+ # topic = Keyword.get(opts, :topic, "")
+ Server.call(worker_name, {:create_topics, topic_name})
+ end
+
#OTP API
def start(_type, _args) do
max_restarts = Application.get_env(:kafka_ex, :max_restarts, 10)

diff:
@@ -80,6 +80,7 @@ defmodule KafkaEx.Config do
defp server("0.8.0"), do: KafkaEx.Server0P8P0
defp server("0.8.2"), do: KafkaEx.Server0P8P2
+ defp server("0.10.1"), do: KafkaEx.Server0P10P1
defp server(_), do: KafkaEx.Server0P9P0
# ssl_options should be an empty list by default if use_ssl is false

diff:
@@ -12,6 +12,9 @@ defmodule KafkaEx.Protocol do
@heartbeat_request 12
@leave_group_request 13
@sync_group_request 14
+ @create_topics_request 19
+ # DescribeConfigs 32
+ # AlterConfigs 33 Valid resource types are "Topic" and "Broker".
@api_version 0
@@ -59,8 +62,16 @@ defmodule KafkaEx.Protocol do
@sync_group_request
end
+ defp api_key(:create_topics) do
+ @create_topics_request
+ end
+
def create_request(type, correlation_id, client_id) do
- << api_key(type) :: 16, @api_version :: 16, correlation_id :: 32,
+ create_request(type, correlation_id, client_id, @api_version)
+ end
+
+ def create_request(type, correlation_id, client_id, api_version) do
+ << api_key(type) :: 16, api_version :: 16, correlation_id :: 32,
byte_size(client_id) :: 16, client_id :: binary >>
end

diff:
+
+defmodule KafkaEx.Protocol.CreateTopics do
+ alias KafkaEx.Protocol
+
+ @moduledoc """
+ Implementation of the Kafka CreateTopics request and response APIs
+ """
+
+ # CreateTopics Request (Version: 0) => [create_topic_requests] timeout
+ # create_topic_requests => topic num_partitions replication_factor [replica_assignment] [config_entries]
+ # topic => STRING
+ # num_partitions => INT32
+ # replication_factor => INT16
+ # replica_assignment => partition [replicas]
+ # partition => INT32
+ # replicas => INT32
+ # config_entries => config_name config_value
+ # config_name => STRING
+ # config_value => NULLABLE_STRING
+ # timeout => INT32
+
+ defmodule ReplicaAssignment do
+ defstruct partition: nil, replicas: nil
+ @type t :: %ReplicaAssignment{ partition: integer, replicas: [integer] }
+ end
+
+ defmodule ConfigEntry do
+ defstruct config_name: nil, config_value: nil
+ @type t :: %ConfigEntry{ config_name: binary, config_value: binary }
+ end
+
+ defmodule TopicRequest do
+ defstruct topic: nil,
+ num_partitions: nil,
+ replication_factor: nil,
+ replica_assignment: nil,
+ config_entries: nil
+ @type t :: %TopicRequest{
+ topic: binary,
+ num_partitions: integer,
+ replication_factor: integer,
+ replica_assignment: [ReplicaAssignment],
+ config_entries: [ConfigEntry],
+ }
+ end
+
+ defmodule Request do
+ @moduledoc false
+ defstruct create_topic_requests: nil, timeout: nil
+ @type t :: %Request{create_topic_requests: [TopicRequest], timeout: integer}
+ end
+
+ defmodule TopicError do
+ defstruct topic_name: nil, error_code: nil
+ @type t :: %TopicError{ topic_name: binary, error_code: integer }
+ end
+
+ defmodule Response do
+ @moduledoc false
+ defstruct topic_errors: nil
+ @type t :: %Response{topic_errors: [TopicError]}
+ end
+
+ @spec create_request(integer, binary, Request.t) :: binary
+ def create_request(correlation_id, client_id, create_topics_request) do
+ Protocol.create_request(:create_topics, correlation_id, client_id) <>
+ encode_topic_requests(create_topics_request.create_topic_requests) <>
+ << create_topics_request.timeout :: 32-signed >>
+ end
+
+ @spec encode_topic_requests([TopicRequest.t]) :: binary
+ defp encode_topic_requests(requests) do
+ requests
+ |> map_encode(&encode_topic_request/1)
+ end
+
+ @spec encode_topic_request(TopicRequest.t) :: binary
+ defp encode_topic_request(request) do
+ encode_string(request.topic) <>
+ << request.num_partitions :: 32-signed, request.replication_factor :: 16-signed >> <>
+ encode_replica_assignments(request.replica_assignment) <>
+ encode_config_entries(request.config_entries)
+ end
+
+
+ @spec encode_replica_assignments([ReplicaAssignment.t]) :: binary
+ defp encode_replica_assignments(replica_assignments) do
+ replica_assignments |> map_encode(&encode_replica_assignment/1)
+ end
+
+ @spec encode_replica_assignment(ReplicaAssignment.t) :: binary
+ defp encode_replica_assignment(replica_assignment) do
+ << replica_assignment.partition :: 32-signed >> <>
+ replica_assignment.replicas |> map_encode(&(<< &1 :: 32-signed >>))
+ end
+
+ @spec encode_config_entries([ConfigEntry.t]) :: binary
+ defp encode_config_entries(config_entries) do
+ config_entries |> map_encode(&encode_config_entry/1)
+ end
+
+ @spec encode_config_entry(ConfigEntry.t) :: binary
+ defp encode_config_entry(config_entry) do
+ encode_string(config_entry.config_name) <> encode_nullable_string(config_entry.config_value)
+ end
+
+ @spec encode_nullable_string(String.t) :: binary
+ defp encode_nullable_string(text) do
+ if text == nil do
+ << -1 :: 16-signed >>
+ else
+ encode_string(text)
+ end
+ end
+
+ @spec encode_string(String.t) :: binary
+ defp encode_string(text) do
+ << byte_size(text) :: 16-signed, text :: binary, >>
+ end
+
+ defp map_encode(elems, function) do
+ if nil == elems or 0 == length(elems) do
+ << 0 :: 32-signed >>
+ else
+ << length(elems) :: 32-signed >> <>
+ (elems
+ |> Enum.map(function)
+ |> Enum.reduce(&(&1 <> &2)))
+ end
+
+ end
+
+ @spec parse_response(binary) :: [] | Response.t
+ def parse_response(<< _correlation_id :: 32-signed, topic_errors_count :: 32-signed, topic_errors :: binary >>) do
+ %Response{topic_errors: parse_topic_errors(topic_errors_count, topic_errors)}
+ end
+
+ defp parse_topic_errors(0, _), do: []
+
+ @spec parse_topic_errors(integer, binary) :: [TopicError.t]
+ defp parse_topic_errors(topic_errors_count,
+ << topic_name_size :: 16-signed, topic_name :: size(topic_name_size)-binary, error_code :: 16-signed, rest :: binary >>) do
+ [%TopicError{topic_name: topic_name, error_code: error_code} | parse_topic_errors(topic_errors_count - 1, rest)]
+ end
+
+end

diff:
@@ -2,6 +2,8 @@ defmodule KafkaEx.Protocol.Metadata do
alias KafkaEx.Protocol
import KafkaEx.Protocol.Common
+ @default_api_version 0
+
@moduledoc """
Implementation of the Kafka Hearbeat request and response APIs
"""
@@ -16,8 +18,8 @@ defmodule KafkaEx.Protocol.Metadata do
alias KafkaEx.Socket
- defstruct node_id: -1, host: "", port: 0, socket: nil
- @type t :: %__MODULE__{}
+ defstruct node_id: -1, host: "", port: 0, socket: nil, is_controller: nil
+ @type t :: %__MODULE__{node_id: integer, host: binary, port: integer, socket: KafkaEx.Socket.t, is_controller: boolean }
def connected?(%Broker{} = broker) do
broker.socket != nil && Socket.open?(broker.socket)
@@ -28,10 +30,11 @@ defmodule KafkaEx.Protocol.Metadata do
@moduledoc false
alias KafkaEx.Protocol.Metadata.Broker
alias KafkaEx.Protocol.Metadata.TopicMetadata
- defstruct brokers: [], topic_metadatas: []
+ defstruct brokers: [], topic_metadatas: [], controller_id: nil
@type t :: %Response{
brokers: [Broker.t],
- topic_metadatas: [TopicMetadata.t]
+ topic_metadatas: [TopicMetadata.t],
+ controller_id: integer
}
def broker_for_topic(metadata, brokers, topic, partition) do
@@ -73,10 +76,11 @@ defmodule KafkaEx.Protocol.Metadata do
defmodule TopicMetadata do
@moduledoc false
alias KafkaEx.Protocol.Metadata.PartitionMetadata
- defstruct error_code: 0, topic: nil, partition_metadatas: []
+ defstruct error_code: 0, topic: nil, is_internal: nil, partition_metadatas: []
@type t :: %TopicMetadata{
error_code: integer | :no_error,
topic: nil | binary,
+ is_internal: nil | boolean,
partition_metadatas: [PartitionMetadata.t]
}
end
@@ -93,19 +97,45 @@ defmodule KafkaEx.Protocol.Metadata do
}
end
- def create_request(correlation_id, client_id, ""), do: KafkaEx.Protocol.create_request(:metadata, correlation_id, client_id) <> << 0 :: 32-signed >>
+ def valid_api_version(v) do
+ case v do
+ nil -> @default_api_version
+ v -> v
+ end
+ end
- def create_request(correlation_id, client_id, topic) when is_binary(topic), do: create_request(correlation_id, client_id, [topic])
+ def create_request(correlation_id, client_id, ""), do: create_request(correlation_id, client_id, "", nil)
+ def create_request(correlation_id, client_id, topics) when is_list(topics), do: create_request(correlation_id, client_id, topics, nil)
+
+ def create_request(correlation_id, client_id, "", api_version) do
+ version = valid_api_version(api_version)
+ topic_count = if 0 == version, do: 0, else: -1
+ KafkaEx.Protocol.create_request(:metadata, correlation_id, client_id, version) <> << topic_count :: 32-signed >>
+ end
+ def create_request(correlation_id, client_id, topic, api_version) when is_binary(topic), do: create_request(correlation_id, client_id, [topic], valid_api_version(api_version))
- def create_request(correlation_id, client_id, topics) when is_list(topics) do
- KafkaEx.Protocol.create_request(:metadata, correlation_id, client_id) <> << length(topics) :: 32-signed, topic_data(topics) :: binary >>
+ def create_request(correlation_id, client_id, topics, api_version) when is_list(topics) do
+ KafkaEx.Protocol.create_request(:metadata, correlation_id, client_id, valid_api_version(api_version)) <> << length(topics) :: 32-signed, topic_data(topics) :: binary >>
end
- def parse_response(<< _correlation_id :: 32-signed, brokers_size :: 32-signed, rest :: binary >>) do
+ def parse_response(data) do
+ parse_response(data, nil)
+ end
+
+ def parse_response(<< _correlation_id :: 32-signed, brokers_size :: 32-signed, rest :: binary >>, api_version) do
+ version = valid_api_version(api_version)
+ case version do
+ 1 ->
+ {brokers, rest} = parse_brokers_v1(brokers_size, rest, [])
+ << controller_id :: 32-signed, rest :: binary >> = rest
+ << topic_metadatas_size :: 32-signed, rest :: binary >> = rest
+ %Response{brokers: brokers, controller_id: controller_id, topic_metadatas: parse_topic_metadatas_v1(topic_metadatas_size, rest)}
+ 0 ->
{brokers, rest} = parse_brokers(brokers_size, rest, [])
<< topic_metadatas_size :: 32-signed, rest :: binary >> = rest
%Response{brokers: brokers, topic_metadatas: parse_topic_metadatas(topic_metadatas_size, rest)}
end
+ end
defp parse_brokers(0, rest, brokers), do: {brokers, rest}
@@ -113,6 +143,32 @@ defmodule KafkaEx.Protocol.Metadata do
parse_brokers(brokers_size - 1, rest, [%Broker{node_id: node_id, host: host, port: port} | brokers])
end
+ defp parse_brokers_v1(0, rest, brokers), do: {brokers, rest}
+
+ defp parse_brokers_v1(brokers_size, <<
+ node_id :: 32-signed,
+ host_len :: 16-signed,
+ host :: size(host_len)-binary,
+ port :: 32-signed,
+ # rack is nullable
+ -1 :: 16-signed,
+ rest :: binary
+ >>, brokers) do
+ parse_brokers_v1(brokers_size - 1, rest, [%Broker{node_id: node_id, host: host, port: port} | brokers])
+ end
+
+ defp parse_brokers_v1(brokers_size, <<
+ node_id :: 32-signed,
+ host_len :: 16-signed,
+ host :: size(host_len)-binary,
+ port :: 32-signed,
+ rack_len :: 16-signed,
+ _rack :: size(rack_len)-binary,
+ rest :: binary
+ >>, brokers) do
+ parse_brokers_v1(brokers_size - 1, rest, [%Broker{node_id: node_id, host: host, port: port} | brokers])
+ end
+
defp parse_topic_metadatas(0, _), do: []
defp parse_topic_metadatas(topic_metadatas_size, << error_code :: 16-signed, topic_len :: 16-signed, topic :: size(topic_len)-binary, partition_metadatas_size :: 32-signed, rest :: binary >>) do
@@ -120,6 +176,22 @@ defmodule KafkaEx.Protocol.Metadata do
[%TopicMetadata{error_code: Protocol.error(error_code), topic: topic, partition_metadatas: partition_metadatas} | parse_topic_metadatas(topic_metadatas_size - 1, rest)]
end
+ defp parse_topic_metadatas_v1(0, _), do: []
+
+ defp parse_topic_metadatas_v1(
+ topic_metadatas_size,
+ << error_code :: 16-signed,
+ topic_len :: 16-signed,
+ topic :: size(topic_len)-binary,
+ # booleans are actually 8-signed
+ is_internal :: 8-signed,
+ partition_metadatas_size :: 32-signed,
+ rest :: binary >>) do
+ {partition_metadatas, rest} = parse_partition_metadatas(partition_metadatas_size, [], rest)
+ [%TopicMetadata{error_code: Protocol.error(error_code), topic: topic, partition_metadatas: partition_metadatas, is_internal: is_internal == 1 } |
+ parse_topic_metadatas_v1(topic_metadatas_size - 1, rest)]
+ end
+
defp parse_partition_metadatas(0, partition_metadatas, rest), do: {partition_metadatas, rest}
defp parse_partition_metadatas(partition_metadatas_size, partition_metadatas, << error_code :: 16-signed, partition_id :: 32-signed, leader :: 32-signed, rest :: binary >>) do

diff:
@@ -260,6 +260,10 @@ defmodule KafkaEx.Server do
kafka_server_heartbeat(request, network_timeout, state)
end
+ def handle_call({:create_topics, topic_name}, _from, state) do
+ kafka_create_topics(topic_name, state)
+ end
+
def handle_info(:update_metadata, state) do
kafka_server_update_metadata(state)
end
@@ -367,9 +371,11 @@ defmodule KafkaEx.Server do
{:noreply, update_metadata(state)}
end
- def update_metadata(state) do
- {correlation_id, metadata} = retrieve_metadata(state.brokers, state.correlation_id, config_sync_timeout())
- metadata_brokers = metadata.brokers
+ def update_metadata(state), do: update_metadata(state, nil)
+
+ def update_metadata(state, api_version) do
+ {correlation_id, metadata} = retrieve_metadata_with_version(state.brokers, state.correlation_id, config_sync_timeout(), api_version)
+ metadata_brokers = metadata.brokers |> Enum.map(&(%{&1 | is_controller: &1.node_id == metadata.controller_id}))
brokers = state.brokers
|> remove_stale_brokers(metadata_brokers)
|> add_new_brokers(metadata_brokers, state.ssl_options, state.use_ssl)
@@ -378,17 +384,27 @@ defmodule KafkaEx.Server do
# credo:disable-for-next-line Credo.Check.Refactor.FunctionArity
def retrieve_metadata(brokers, correlation_id, sync_timeout, topic \\ []), do: retrieve_metadata(brokers, correlation_id, sync_timeout, topic, @retry_count, 0)
+
+ # credo:disable-for-next-line Credo.Check.Refactor.FunctionArity
+ def retrieve_metadata_with_version(brokers, correlation_id, sync_timeout, api_version, topic \\ []), do: retrieve_metadata(brokers, correlation_id, sync_timeout, topic, @retry_count, api_version, 0)
+
# credo:disable-for-next-line Credo.Check.Refactor.FunctionArity
def retrieve_metadata(_, correlation_id, _sync_timeout, topic, 0, error_code) do
Logger.log(:error, "Metadata request for topic #{inspect topic} failed with error_code #{inspect error_code}")
{correlation_id, %Metadata.Response{}}
end
+
+ # credo:disable-for-next-line Credo.Check.Refactor.FunctionArity
+ def retrieve_metadata(brokers, correlation_id, sync_timeout, topic, retry, error_code) do
+ retrieve_metadata(brokers, correlation_id, sync_timeout, topic, retry, nil, error_code)
+ end
+
# credo:disable-for-next-line Credo.Check.Refactor.FunctionArity
- def retrieve_metadata(brokers, correlation_id, sync_timeout, topic, retry, _error_code) do
- metadata_request = Metadata.create_request(correlation_id, @client_id, topic)
+ def retrieve_metadata(brokers, correlation_id, sync_timeout, topic, retry, api_version, _error_code) do
+ metadata_request = Metadata.create_request(correlation_id, @client_id, topic, api_version)
data = first_broker_response(metadata_request, brokers, sync_timeout)
if data do
- response = Metadata.parse_response(data)
+ response = Metadata.parse_response(data, api_version)
case Enum.find(response.topic_metadatas, &(&1.error_code == :leader_not_available)) do
nil -> {correlation_id + 1, response}

diff:
+defmodule KafkaEx.Server0P10P1 do
+ @moduledoc """
+ Implements kafkaEx.Server behaviors for kafka 0.10.1 API.
+ """
+ use KafkaEx.Server
+ alias KafkaEx.Protocol.CreateTopics
+ alias KafkaEx.Server0P8P2
+ alias KafkaEx.Server0P9P0
+
+ # alias KafkaEx.ConsumerGroupRequiredError
+ alias KafkaEx.InvalidConsumerGroupError
+ alias KafkaEx.Protocol.ConsumerMetadata
+ alias KafkaEx.Protocol.ConsumerMetadata.Response, as: ConsumerMetadataResponse
+ # alias KafkaEx.Protocol.Heartbeat
+ # alias KafkaEx.Protocol.JoinGroup
+ # alias KafkaEx.Protocol.LeaveGroup
+ alias KafkaEx.Protocol.Metadata.Broker
+ # alias KafkaEx.Protocol.SyncGroup
+ alias KafkaEx.Server.State
+ # alias KafkaEx.NetworkClient
+
+ require Logger
+
+ @metadata_api_version 1
+ @consumer_group_update_interval 30_000
+
+
+ def start_link(args, name \\ __MODULE__)
+
+ def start_link(args, :no_name) do
+ GenServer.start_link(__MODULE__, [args])
+ end
+
+ def start_link(args, name) do
+ GenServer.start_link(__MODULE__, [args, name], [name: name])
+ end
+
+ # The functions below are all defined in KafkaEx.Server0P8P2
+ defdelegate kafka_server_consumer_group(state), to: Server0P8P2
+ defdelegate kafka_server_fetch(fetch_request, state), to: Server0P8P2
+ defdelegate kafka_server_offset_fetch(offset_fetch, state), to: Server0P8P2
+ defdelegate kafka_server_offset_commit(offset_commit_request, state), to: Server0P8P2
+ defdelegate kafka_server_consumer_group_metadata(state), to: Server0P8P2
+ defdelegate kafka_server_update_consumer_metadata(state), to: Server0P8P2
+
+ # The functions below are all defined in KafkaEx.Server0P9P0
+ defdelegate kafka_server_join_group(request, network_timeout, state_in), to: Server0P9P0
+ defdelegate kafka_server_sync_group(request, network_timeout, state_in), to: Server0P9P0
+ defdelegate kafka_server_leave_group(request, network_timeout, state_in), to: Server0P9P0
+ defdelegate kafka_server_heartbeat(request, network_timeout, state_in), to: Server0P9P0
+ defdelegate consumer_group?(state), to: Server0P9P0
+
+ def kafka_server_init([args]) do
+ kafka_server_init([args, self()])
+ end
+
+ def kafka_server_init([args, name]) do
+ uris = Keyword.get(args, :uris, [])
+ metadata_update_interval = Keyword.get(args, :metadata_update_interval, @metadata_update_interval)
+ consumer_group_update_interval = Keyword.get(args, :consumer_group_update_interval, @consumer_group_update_interval)
+
+ # this should have already been validated, but it's possible someone could
+ # try to short-circuit the start call
+ consumer_group = Keyword.get(args, :consumer_group)
+ unless KafkaEx.valid_consumer_group?(consumer_group) do
+ raise InvalidConsumerGroupError, consumer_group
+ end
+
+ use_ssl = Keyword.get(args, :use_ssl, false)
+ ssl_options = Keyword.get(args, :ssl_options, [])
+
+ brokers = Enum.map(uris, fn({host, port}) -> %Broker{host: host, port: port, socket: NetworkClient.create_socket(host, port, ssl_options, use_ssl)} end)
+
+ {correlation_id, metadata} = retrieve_metadata_with_version(brokers, 0, config_sync_timeout(), @metadata_api_version)
+ state = %State{metadata: metadata, brokers: brokers, correlation_id: correlation_id, consumer_group: consumer_group, metadata_update_interval: metadata_update_interval, consumer_group_update_interval: consumer_group_update_interval, worker_name: name, ssl_options: ssl_options, use_ssl: use_ssl}
+ # Get the initial "real" broker list and start a regular refresh cycle.
+ state = update_metadata(state, @metadata_api_version)
+ {:ok, _} = :timer.send_interval(state.metadata_update_interval, :update_metadata)
+
+ state =
+ if consumer_group?(state) do
+ # If we are using consumer groups then initialize the state and start the update cycle
+ {_, updated_state} = update_consumer_metadata(state)
+ {:ok, _} = :timer.send_interval(state.consumer_group_update_interval, :update_consumer_metadata)
+ updated_state
+ else
+ state
+ end
+
+ {:ok, state}
+ end
+
+ def kafka_server_metadata(topic, state) do
+ {correlation_id, metadata} = retrieve_metadata_with_version(state.brokers, state.correlation_id, config_sync_timeout(), @metadata_api_version, topic)
+ updated_state = %{state | metadata: metadata, correlation_id: correlation_id}
+ {:reply, metadata, updated_state}
+ end
+
+ def kafka_server_update_metadata(state) do
+ {:noreply, update_metadata(state, @metadata_api_version)}
+ end
+
+ def kafka_create_topics(requests, state) do
+ create_topics_request = %CreateTopics.Request{
+ create_topic_requests: requests,
+ timeout: 2000
+ }
+
+ mainRequest = CreateTopics.create_request(state.correlation_id, @client_id, create_topics_request)
+
+ broker = state.brokers |> Enum.find(&(&1.is_controller))
+
+ {response, state} = case broker do
+ nil ->
+ Logger.log(:error, "Coordinator for topic is not available")
+ {:topic_not_found, state}
+ _ ->
+ response = broker
+ |> NetworkClient.send_sync_request(mainRequest, config_sync_timeout())
+ |> case do
+ {:error, reason} -> {:error, reason}
+ response -> CreateTopics.parse_response(response)
+ end
+ {response, %{state | correlation_id: state.correlation_id + 1}}
+ end
+
+ {:reply, response, state}
+ end
+
+ defp update_consumer_metadata(state), do: update_consumer_metadata(state, @retry_count, 0)
+
+ defp update_consumer_metadata(%State{consumer_group: consumer_group} = state, 0, error_code) do
+ Logger.log(:error, "Fetching consumer_group #{consumer_group} metadata failed with error_code #{inspect error_code}")
+ {%ConsumerMetadataResponse{error_code: error_code}, state}
+ end
+
+ defp update_consumer_metadata(%State{consumer_group: consumer_group, correlation_id: correlation_id} = state, retry, _error_code) do
+ response = correlation_id
+ |> ConsumerMetadata.create_request(@client_id, consumer_group)
+ |> first_broker_response(state)
+ |> ConsumerMetadata.parse_response
+
+ case response.error_code do
+ :no_error ->
+ {
+ response,
+ %{
+ state |
+ consumer_metadata: response,
+ correlation_id: state.correlation_id + 1
+ }
+ }
+ _ -> :timer.sleep(400)
+ update_consumer_metadata(
+ %{state | correlation_id: state.correlation_id + 1},
+ retry - 1,
+ response.error_code
+ )
+ end
+ end
+
+ defp broker_for_consumer_group(state) do
+ ConsumerMetadataResponse.broker_for_consumer_group(state.brokers, state.consumer_metadata)
+ end
+
+ # refactored from two versions, one that used the first broker as valid answer, hence
+ # the optional extra flag to do that. Wraps broker_for_consumer_group with an update
+ # call if no broker was found.
+ def broker_for_consumer_group_with_update(state, use_first_as_default \\ false) do
+ case broker_for_consumer_group(state) do
+ nil ->
+ {_, updated_state} = update_consumer_metadata(state)
+ default_broker = if use_first_as_default, do: hd(state.brokers), else: nil
+ {broker_for_consumer_group(updated_state) || default_broker, updated_state}
+ broker ->
+ {broker, state}
+ end
+ end
+
+ defp first_broker_response(request, state) do
+ first_broker_response(request, state.brokers, config_sync_timeout())
+ end
+
+end

diff:
@@ -62,6 +62,7 @@ defmodule KafkaEx.Server0P8P0 do
def kafka_server_leave_group(_, _, _state), do: raise "Leave Group is not supported in 0.8.0 version of Kafka"
def kafka_server_heartbeat(_, _, _state), do: raise "Heartbeat is not supported in 0.8.0 version of kafka"
def kafka_server_update_consumer_metadata(_state), do: raise "Consumer Group Metadata is not supported in 0.8.0 version of kafka"
+ def kafka_create_topics(_, _state), do: raise "CreateTopic is not supported in 0.9.0 version of kafka"
defp fetch(request, state) do
case network_request(request, Fetch, state) do

diff:
@@ -138,6 +138,7 @@ defmodule KafkaEx.Server0P8P2 do
def kafka_server_leave_group(_, _, _state), do: raise "Leave Group is not supported in 0.8.0 version of Kafka"
def kafka_server_heartbeat(_, _, _state), do: raise "Heartbeat is not supported in 0.8.0 version of kafka"
defp update_consumer_metadata(state), do: update_consumer_metadata(state, @retry_count, 0)
+ def kafka_create_topics(_, _state), do: raise "CreateTopic is not supported in 0.9.0 version of kafka"
defp update_consumer_metadata(%State{consumer_group: consumer_group} = state, 0, error_code) do
Logger.log(:error, "Fetching consumer_group #{consumer_group} metadata failed with error_code #{inspect error_code}")

diff:
defmodule KafkaEx.Server0P9P0 do
@moduledoc """
- Implements kafkaEx.Server behaviors for kafka 0.9.0 API.
+ Implements kafkaEx.Server behaviors for kafka >= 0.9.0 < 0.10.1 API.
+
"""
+
+ # these functions aren't implemented for 0.9.0
+ @dialyzer [
+ {:nowarn_function, kafka_create_topics: 2},
+ ]
+
use KafkaEx.Server
alias KafkaEx.ConsumerGroupRequiredError
alias KafkaEx.InvalidConsumerGroupError
@@ -38,6 +45,7 @@ defmodule KafkaEx.Server0P9P0 do
defdelegate kafka_server_offset_commit(offset_commit_request, state), to: Server0P8P2
defdelegate kafka_server_consumer_group_metadata(state), to: Server0P8P2
defdelegate kafka_server_update_consumer_metadata(state), to: Server0P8P2
+ def kafka_create_topics(_, _state), do: raise "CreateTopic is not supported in 0.9.0 version of kafka"
def kafka_server_init([args]) do
kafka_server_init([args, self()])
@@ -207,7 +215,7 @@ defmodule KafkaEx.Server0P9P0 do
# refactored from two versions, one that used the first broker as valid answer, hence
# the optional extra flag to do that. Wraps broker_for_consumer_group with an update
# call if no broker was found.
- defp broker_for_consumer_group_with_update(state, use_first_as_default \\ false) do
+ def broker_for_consumer_group_with_update(state, use_first_as_default \\ false) do
case broker_for_consumer_group(state) do
nil ->
{_, updated_state} = update_consumer_metadata(state)
@@ -224,6 +232,7 @@ defmodule KafkaEx.Server0P9P0 do
def consumer_group?(%State{consumer_group: :no_consumer_group}), do: false
def consumer_group?(_), do: true
+
defp first_broker_response(request, state) do
first_broker_response(request, state.brokers, config_sync_timeout())
end

diff:
@@ -14,6 +14,7 @@ set -e
# Kafka needs to know our ip address so that it can advertise valid
# connnection details
+
if [ -z ${IP_IFACE} ]
then
echo Detecting active network interface

diff:
+defmodule KafkaEx.Server0P10P1.Test do
+ use ExUnit.Case
+
+ @moduletag :server_0_p_10_p_1
+
+ @tag :createtopic
+ test "can create a topic" do
+ name = "topic3_#{:rand.uniform(2000000)}"
+
+ request = %{
+ topic: name,
+ num_partitions: 10,
+ replication_factor: 1,
+ replica_assignment: [],
+ config_entries: [
+ %{config_name: "cleanup.policy", config_value: "compact"},
+ %{config_name: "min.compaction.lag.ms", config_value: "0"}
+ ]}
+
+ resp = KafkaEx.create_topics([request])
+ # error = NONE
+ assert {0, name} == parse_create_topic_resp(resp)
+
+ resp = KafkaEx.create_topics([request])
+ # error = TOPIC_ALREADY_EXISTS
+ assert {36, name} == parse_create_topic_resp(resp)
+
+ topics = KafkaEx.metadata.topic_metadatas |> Enum.map(&(&1.topic))
+ assert Enum.member?(topics, name)
+ end
+
+ def parse_create_topic_resp(response) do
+ %KafkaEx.Protocol.CreateTopics.Response{
+ topic_errors: [
+ %KafkaEx.Protocol.CreateTopics.TopicError{
+ error_code: error_code,
+ topic_name: topic_name
+ }
+ ]} = response
+ {error_code, topic_name}
+ end
+end


message: Prevent memory leak in message producer when request is invalid

diff:
@@ -82,12 +82,16 @@ defmodule KafkaEx.Protocol.Produce do
def parse_response(unknown), do: unknown
defp create_message_set([], _compression_type), do: {"", 0}
- defp create_message_set([%Message{key: key, value: value}|messages], :none) do
+ defp create_message_set_uncompressed([%Message{key: key, value: value}|messages]) do
{message, msize} = create_message(value, key)
message_set = [<< 0 :: 64-signed >>, << msize :: 32-signed >>, message]
{message_set2, ms2size} = create_message_set(messages, :none)
{[message_set, message_set2], 8 + 4 + msize + ms2size}
end
+
+ defp create_message_set(messages, :none) do
+ create_message_set_uncompressed(messages)
+ end
defp create_message_set(messages, compression_type) do
{message_set, _} = create_message_set(messages, :none)
{compressed_message_set, attribute} =

diff:
@@ -289,7 +289,19 @@ defmodule KafkaEx.Server do
# credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity
def kafka_server_produce(produce_request, state) do
correlation_id = state.correlation_id + 1
- produce_request_data = Produce.create_request(correlation_id, @client_id, produce_request)
+ produce_request_data = try do
+ Produce.create_request(correlation_id, @client_id, produce_request)
+ rescue
+ e in FunctionClauseError -> nil
+ end
+
+ case produce_request_data do
+ nil -> {:reply, {:error, "Invalid produce request"}, state}
+ _ -> kafka_server_produce_send_request(correlation_id, produce_request, produce_request_data, state)
+ end
+ end
+
+ def kafka_server_produce_send_request(correlation_id, produce_request, produce_request_data, state) do
{broker, state, corr_id} = case MetadataResponse.broker_for_topic(state.metadata, state.brokers, produce_request.topic, produce_request.partition) do
nil ->
{retrieved_corr_id, _} = retrieve_metadata(state.brokers, state.correlation_id, config_sync_timeout(), produce_request.topic)

diff:
@@ -43,6 +43,19 @@ defmodule KafkaEx.Server0P8P0.Test do
end)
end
+ @tag :producer_mem_leak
+ test "does not mem leak when message is not a Produce.Message" do
+ name = "create_topic_#{:rand.uniform(2000000)}"
+
+ assert {:error, "Invalid produce request"} == KafkaEx.produce(%KafkaEx.Protocol.Produce.Request{
+ messages: [%{key: "key1", value: "value1"}],
+ partition: 0,
+ required_acks: 0,
+ timeout: 500,
+ topic: name
+ })
+ end
+
test "when the partition is not found", %{worker: worker} do
partition = 42
assert :topic_not_found == KafkaEx.fetch(


message: Add network timeout artgument to create_topics

diff:
@@ -538,10 +538,10 @@ defmodule KafkaEx do
```
"""
@spec create_topics(String.t) :: CreateTopics.Response.t
- def create_topics(topic_name, opts \\ []) do
+ def create_topics(requests, opts \\ []) do
worker_name = Keyword.get(opts, :worker_name, Config.default_worker)
- # topic = Keyword.get(opts, :topic, "")
- Server.call(worker_name, {:create_topics, topic_name})
+ timeout = Keyword.get(opts, :timeout, 4000)
+ Server.call(worker_name, {:create_topics, requests, timeout})
end
#OTP API

diff:
@@ -31,10 +31,10 @@ defmodule KafkaEx.Protocol.CreateTopics do
defmodule TopicRequest do
defstruct topic: nil,
- num_partitions: nil,
- replication_factor: nil,
- replica_assignment: nil,
- config_entries: nil
+ num_partitions: -1,
+ replication_factor: -1,
+ replica_assignment: [],
+ config_entries: []
@type t :: %TopicRequest{
topic: binary,
num_partitions: integer,

diff:
@@ -260,8 +260,8 @@ defmodule KafkaEx.Server do
kafka_server_heartbeat(request, network_timeout, state)
end
- def handle_call({:create_topics, topic_name}, _from, state) do
- kafka_create_topics(topic_name, state)
+ def handle_call({:create_topics, requests, network_timeout}, _from, state) do
+ kafka_create_topics(requests, network_timeout, state)
end
def handle_info(:update_metadata, state) do

diff:
@@ -7,17 +7,11 @@ defmodule KafkaEx.Server0P10P1 do
alias KafkaEx.Server0P8P2
alias KafkaEx.Server0P9P0
- # alias KafkaEx.ConsumerGroupRequiredError
alias KafkaEx.InvalidConsumerGroupError
alias KafkaEx.Protocol.ConsumerMetadata
alias KafkaEx.Protocol.ConsumerMetadata.Response, as: ConsumerMetadataResponse
- # alias KafkaEx.Protocol.Heartbeat
- # alias KafkaEx.Protocol.JoinGroup
- # alias KafkaEx.Protocol.LeaveGroup
alias KafkaEx.Protocol.Metadata.Broker
- # alias KafkaEx.Protocol.SyncGroup
alias KafkaEx.Server.State
- # alias KafkaEx.NetworkClient
require Logger
@@ -100,10 +94,10 @@ defmodule KafkaEx.Server0P10P1 do
{:noreply, update_metadata(state, @metadata_api_version)}
end
- def kafka_create_topics(requests, state) do
+ def kafka_create_topics(requests, network_timeout, state) do
create_topics_request = %CreateTopics.Request{
create_topic_requests: requests,
- timeout: 2000
+ timeout: network_timeout
}
mainRequest = CreateTopics.create_request(state.correlation_id, @client_id, create_topics_request)

diff:
@@ -62,7 +62,7 @@ defmodule KafkaEx.Server0P8P0 do
def kafka_server_leave_group(_, _, _state), do: raise "Leave Group is not supported in 0.8.0 version of Kafka"
def kafka_server_heartbeat(_, _, _state), do: raise "Heartbeat is not supported in 0.8.0 version of kafka"
def kafka_server_update_consumer_metadata(_state), do: raise "Consumer Group Metadata is not supported in 0.8.0 version of kafka"
- def kafka_create_topics(_, _state), do: raise "CreateTopic is not supported in 0.9.0 version of kafka"
+ def kafka_create_topics(_, _, _state), do: raise "CreateTopic is not supported in 0.8.0 version of kafka"
defp fetch(request, state) do
case network_request(request, Fetch, state) do

diff:
@@ -138,7 +138,7 @@ defmodule KafkaEx.Server0P8P2 do
def kafka_server_leave_group(_, _, _state), do: raise "Leave Group is not supported in 0.8.0 version of Kafka"
def kafka_server_heartbeat(_, _, _state), do: raise "Heartbeat is not supported in 0.8.0 version of kafka"
defp update_consumer_metadata(state), do: update_consumer_metadata(state, @retry_count, 0)
- def kafka_create_topics(_, _state), do: raise "CreateTopic is not supported in 0.9.0 version of kafka"
+ def kafka_create_topics(_, _, _state), do: raise "CreateTopic is not supported in 0.8.0 version of kafka"
defp update_consumer_metadata(%State{consumer_group: consumer_group} = state, 0, error_code) do
Logger.log(:error, "Fetching consumer_group #{consumer_group} metadata failed with error_code #{inspect error_code}")

diff:
@@ -45,7 +45,7 @@ defmodule KafkaEx.Server0P9P0 do
defdelegate kafka_server_offset_commit(offset_commit_request, state), to: Server0P8P2
defdelegate kafka_server_consumer_group_metadata(state), to: Server0P8P2
defdelegate kafka_server_update_consumer_metadata(state), to: Server0P8P2
- def kafka_create_topics(_, _state), do: raise "CreateTopic is not supported in 0.9.0 version of kafka"
+ def kafka_create_topics(_, _, _state), do: raise "CreateTopic is not supported in 0.9.0 version of kafka"
def kafka_server_init([args]) do
kafka_server_init([args, self()])

diff:
@@ -5,7 +5,7 @@ defmodule KafkaEx.Server0P10P1.Test do
@tag :createtopic
test "can create a topic" do
- name = "topic3_#{:rand.uniform(2000000)}"
+ name = "create_topic_#{:rand.uniform(2000000)}"
request = %{
topic: name,
@@ -17,11 +17,11 @@ defmodule KafkaEx.Server0P10P1.Test do
%{config_name: "min.compaction.lag.ms", config_value: "0"}
]}
- resp = KafkaEx.create_topics([request])
+ resp = KafkaEx.create_topics([request], :timeout 2000)
# error = NONE
assert {0, name} == parse_create_topic_resp(resp)
- resp = KafkaEx.create_topics([request])
+ resp = KafkaEx.create_topics([request], :timeout 2000)
# error = TOPIC_ALREADY_EXISTS
assert {36, name} == parse_create_topic_resp(resp)


message: Set 0.10.1 as the new default version

diff:
@@ -80,8 +80,9 @@ defmodule KafkaEx.Config do
defp server("0.8.0"), do: KafkaEx.Server0P8P0
defp server("0.8.2"), do: KafkaEx.Server0P8P2
- defp server("0.10.1"), do: KafkaEx.Server0P10P1
- defp server(_), do: KafkaEx.Server0P9P0
+ defp server("0.9.0"), do: KafkaEx.Server0P9P0
+ defp server(_), do: KafkaEx.Server0P10P1
+
# ssl_options should be an empty list by default if use_ssl is false
defp ssl_options(false, []), do: []

original_message: Set 0.10.1 as the new default version


message: Update dialyxir to get a much nicer UX; fix warnings

diff:
@@ -22,6 +22,8 @@ defmodule KafkaEx do
alias KafkaEx.Protocol.Produce.Message
alias KafkaEx.Protocol.SyncGroup.Request, as: SyncGroupRequest
alias KafkaEx.Protocol.SyncGroup.Response, as: SyncGroupResponse
+ alias KafkaEx.Protocol.CreateTopics.Request, as: CreateTopicsRequest
+ alias KafkaEx.Protocol.CreateTopics.Response, as: CreateTopicsResponse
alias KafkaEx.Server
alias KafkaEx.Stream
@@ -537,7 +539,7 @@ defmodule KafkaEx do
```
"""
- @spec create_topics(CreateTopics.Requests.t, Keyword.t) :: CreateTopics.Response.t
+ @spec create_topics([CreateTopicsRequest.t], Keyword.t) :: CreateTopicsResponse.t
def create_topics(requests, opts \\ []) do
worker_name = Keyword.get(opts, :worker_name, Config.default_worker)
timeout = Keyword.get(opts, :timeout, 4000)

diff:
@@ -88,7 +88,6 @@ defmodule KafkaEx.Protocol.CreateTopics do
replica_assignments |> map_encode(&encode_replica_assignment/1)
end
- @spec encode_replica_assignment(ReplicaAssignment.t) :: binary
defp encode_replica_assignment(replica_assignment) do
<< replica_assignment.partition :: 32-signed >> <>
replica_assignment.replicas |> map_encode(&(<< &1 :: 32-signed >>))

diff:
@@ -36,7 +36,7 @@ defmodule KafkaEx.Mixfile do
defp deps do
[
{:credo, "~> 0.8.10", only: :dev},
- {:dialyxir, "~> 0.5.1", only: :dev},
+ {:dialyxir, "~> 1.0.0-rc.3", only: :dev},
{:excoveralls, "~> 0.7", only: :test},
{:ex_doc, "~> 0.18", only: :dev},
{:snappy,


message: create_topics: replace `if` with `case` and fix errenous spec

diff:
@@ -26,7 +26,7 @@ defmodule KafkaEx.Protocol.CreateTopics do
defmodule ConfigEntry do
defstruct config_name: nil, config_value: nil
- @type t :: %ConfigEntry{ config_name: binary, config_value: binary }
+ @type t :: %ConfigEntry{ config_name: binary, config_value: binary | nil }
end
defmodule TopicRequest do
@@ -105,10 +105,9 @@ defmodule KafkaEx.Protocol.CreateTopics do
@spec encode_nullable_string(String.t) :: binary
defp encode_nullable_string(text) do
- if text == nil do
- << -1 :: 16-signed >>
- else
- encode_string(text)
+ case text do
+ nil -> << -1 :: 16-signed >>
+ _ -> encode_string(text)
end
end


message: Reorder functions in produce.ex

diff:
@@ -82,16 +82,11 @@ defmodule KafkaEx.Protocol.Produce do
def parse_response(unknown), do: unknown
defp create_message_set([], _compression_type), do: {"", 0}
- defp create_message_set_uncompressed([%Message{key: key, value: value}|messages]) do
- {message, msize} = create_message(value, key)
- message_set = [<< 0 :: 64-signed >>, << msize :: 32-signed >>, message]
- {message_set2, ms2size} = create_message_set(messages, :none)
- {[message_set, message_set2], 8 + 4 + msize + ms2size}
- end
defp create_message_set(messages, :none) do
create_message_set_uncompressed(messages)
end
+
defp create_message_set(messages, compression_type) do
{message_set, _} = create_message_set(messages, :none)
{compressed_message_set, attribute} =
@@ -101,6 +96,13 @@ defmodule KafkaEx.Protocol.Produce do
{[<< 0 :: 64-signed >>, << msize :: 32-signed >>, message], 8 + 4 + msize}
end
+ defp create_message_set_uncompressed([%Message{key: key, value: value}|messages]) do
+ {message, msize} = create_message(value, key)
+ message_set = [<< 0 :: 64-signed >>, << msize :: 32-signed >>, message]
+ {message_set2, ms2size} = create_message_set(messages, :none)
+ {[message_set, message_set2], 8 + 4 + msize + ms2size}
+ end
+
defp create_message(value, key, attributes \\ 0) do
{bkey, skey} = bytes(key)
{bvalue, svalue} = bytes(value)

original_message: Reorder functions in produce.ex


message: Remove function valid_api_version in protocol/metadata.ex

diff:
@@ -97,34 +97,30 @@ defmodule KafkaEx.Protocol.Metadata do
}
end
- def valid_api_version(v) do
- case v do
- nil -> @default_api_version
- v -> v
- end
- end
+ def create_request(correlation_id, client_id, ""), do: create_request(correlation_id, client_id, "", @default_api_version)
+ def create_request(correlation_id, client_id, topics) when is_list(topics), do: create_request(correlation_id, client_id, topics, @default_api_version)
- def create_request(correlation_id, client_id, ""), do: create_request(correlation_id, client_id, "", nil)
- def create_request(correlation_id, client_id, topics) when is_list(topics), do: create_request(correlation_id, client_id, topics, nil)
+ def create_request(correlation_id, client_id, "", nil), do: create_request(correlation_id, client_id, "", @default_api_version)
+ def create_request(correlation_id, client_id, topics, nil) when is_list(topics), do: create_request(correlation_id, client_id, topics, @default_api_version)
def create_request(correlation_id, client_id, "", api_version) do
- version = valid_api_version(api_version)
- topic_count = if 0 == version, do: 0, else: -1
- KafkaEx.Protocol.create_request(:metadata, correlation_id, client_id, version) <> << topic_count :: 32-signed >>
+ topic_count = if 0 == api_version, do: 0, else: -1
+ KafkaEx.Protocol.create_request(:metadata, correlation_id, client_id, api_version) <> << topic_count :: 32-signed >>
end
- def create_request(correlation_id, client_id, topic, api_version) when is_binary(topic), do: create_request(correlation_id, client_id, [topic], valid_api_version(api_version))
- def create_request(correlation_id, client_id, topics, api_version) when is_list(topics) do
- KafkaEx.Protocol.create_request(:metadata, correlation_id, client_id, valid_api_version(api_version)) <> << length(topics) :: 32-signed, topic_data(topics) :: binary >>
+ def create_request(correlation_id, client_id, topic, api_version) when is_binary(topic) do
+ create_request(correlation_id, client_id, [topic], api_version)
end
- def parse_response(data) do
- parse_response(data, nil)
+ def create_request(correlation_id, client_id, topics, api_version) when is_list(topics) do
+ KafkaEx.Protocol.create_request(:metadata, correlation_id, client_id, api_version) <> << length(topics) :: 32-signed, topic_data(topics) :: binary >>
end
+ def parse_response(data), do: parse_response(data, @default_api_version)
+ def parse_response(data, nil), do: parse_response(data, @default_api_version)
+
def parse_response(<< _correlation_id :: 32-signed, brokers_size :: 32-signed, rest :: binary >>, api_version) do
- version = valid_api_version(api_version)
- case version do
+ case api_version do
1 ->
{brokers, rest} = parse_brokers_v1(brokers_size, rest, [])
<< controller_id :: 32-signed, rest :: binary >> = rest


message: Use a function parameter for api_version instead of the function name in parse_brokers

diff:
@@ -122,26 +122,26 @@ defmodule KafkaEx.Protocol.Metadata do
def parse_response(<< _correlation_id :: 32-signed, brokers_size :: 32-signed, rest :: binary >>, api_version) do
case api_version do
1 ->
- {brokers, rest} = parse_brokers_v1(brokers_size, rest, [])
+ {brokers, rest} = parse_brokers(1, brokers_size, rest, [])
<< controller_id :: 32-signed, rest :: binary >> = rest
<< topic_metadatas_size :: 32-signed, rest :: binary >> = rest
%Response{brokers: brokers, controller_id: controller_id, topic_metadatas: parse_topic_metadatas_v1(topic_metadatas_size, rest)}
0 ->
- {brokers, rest} = parse_brokers(brokers_size, rest, [])
+ {brokers, rest} = parse_brokers(0, brokers_size, rest, [])
<< topic_metadatas_size :: 32-signed, rest :: binary >> = rest
%Response{brokers: brokers, topic_metadatas: parse_topic_metadatas(topic_metadatas_size, rest)}
end
end
- defp parse_brokers(0, rest, brokers), do: {brokers, rest}
+ defp parse_brokers(_api_version, 0, rest, brokers), do: {brokers, rest}
- defp parse_brokers(brokers_size, << node_id :: 32-signed, host_len :: 16-signed, host :: size(host_len)-binary, port :: 32-signed, rest :: binary >>, brokers) do
- parse_brokers(brokers_size - 1, rest, [%Broker{node_id: node_id, host: host, port: port} | brokers])
+ defp parse_brokers(0, brokers_size, << node_id :: 32-signed, host_len :: 16-signed, host :: size(host_len)-binary, port :: 32-signed, rest :: binary >>, brokers) do
+ parse_brokers(0, brokers_size - 1, rest, [%Broker{node_id: node_id, host: host, port: port} | brokers])
end
- defp parse_brokers_v1(0, rest, brokers), do: {brokers, rest}
+ # defp parse_brokers_v1(0, rest, brokers), do: {brokers, rest}
- defp parse_brokers_v1(brokers_size, <<
+ defp parse_brokers(1, brokers_size, <<
node_id :: 32-signed,
host_len :: 16-signed,
host :: size(host_len)-binary,
@@ -150,10 +150,10 @@ defmodule KafkaEx.Protocol.Metadata do
-1 :: 16-signed,
rest :: binary
>>, brokers) do
- parse_brokers_v1(brokers_size - 1, rest, [%Broker{node_id: node_id, host: host, port: port} | brokers])
+ parse_brokers(1, brokers_size - 1, rest, [%Broker{node_id: node_id, host: host, port: port} | brokers])
end
- defp parse_brokers_v1(brokers_size, <<
+ defp parse_brokers(1, brokers_size, <<
node_id :: 32-signed,
host_len :: 16-signed,
host :: size(host_len)-binary,
@@ -162,7 +162,7 @@ defmodule KafkaEx.Protocol.Metadata do
_rack :: size(rack_len)-binary,
rest :: binary
>>, brokers) do
- parse_brokers_v1(brokers_size - 1, rest, [%Broker{node_id: node_id, host: host, port: port} | brokers])
+ parse_brokers(1, brokers_size - 1, rest, [%Broker{node_id: node_id, host: host, port: port} | brokers])
end
defp parse_topic_metadatas(0, _), do: []