GitHub-Mirror/Tapeti
1
0
Fork 0
mirror of synced 2024-11-22 01:13:49 +00:00
Code Issues Projects Releases Wiki Activity

Fixed #34 Reconnect not working when restarting RabbitMQ

Browse Source 
- Fixed deadlock issue when connection is lost
- Fixed Ack and Cancel being attempted on wrong connection causing channel disconnects
This commit is contained in:
Mark van Renswoude 2021-09-21 16:17:09 +02:00
parent ad7314c42f
commit b22c5200f4
4 changed files with 213 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
Tapeti/Connection/ITapetiClient.cs
View File

@ -78,13 +78,13 @@ namespace Tapeti.Connection
/// <param name="queueName"></param> /// <param name="queueName"></param>
/// <param name="consumer">The consumer implementation which will receive the messages from the queue</param> /// <param name="consumer">The consumer implementation which will receive the messages from the queue</param>
/// <returns>The consumer tag as returned by BasicConsume.</returns> /// <returns>The consumer tag as returned by BasicConsume.</returns>
Task<string> Consume(CancellationToken cancellationToken, string queueName, IConsumer consumer); Task<TapetiConsumerTag> Consume(CancellationToken cancellationToken, string queueName, IConsumer consumer);
/// <summary> /// <summary>
/// Stops the consumer with the specified tag. /// Stops the consumer with the specified tag.
/// </summary> /// </summary>
/// <param name="consumerTag">The consumer tag as returned by Consume.</param> /// <param name="consumerTag">The consumer tag as returned by Consume.</param>
Task Cancel(string consumerTag); Task Cancel(TapetiConsumerTag consumerTag);
/// <summary> /// <summary>
/// Creates a durable queue if it does not already exist, and updates the bindings. /// Creates a durable queue if it does not already exist, and updates the bindings.
@ -129,4 +129,31 @@ namespace Tapeti.Connection
/// </summary> /// </summary>
Task Close(); Task Close();
} }
/// <summary>
/// Represents a consumer for a specific connection.
/// </summary>
public class TapetiConsumerTag
{
/// <summary>
/// The consumer tag as determined by the AMQP protocol.
/// </summary>
public string ConsumerTag { get; }
/// <summary>
/// An internal reference to the connection on which the consume was started.
/// </summary>
public long ConnectionReference { get;}
/// <summary>
/// Creates a new instance of the TapetiConsumerTag class.
/// </summary>
public TapetiConsumerTag(long connectionReference, string consumerTag)
{
ConnectionReference = connectionReference;
ConsumerTag = consumerTag;
}
}
} }

13
Tapeti/Connection/TapetiBasicConsumer.cs
View File

@ -5,6 +5,9 @@ using Tapeti.Default;
namespace Tapeti.Connection namespace Tapeti.Connection
{ {
public delegate Task ResponseFunc(long expectedConnectionReference, ulong deliveryTag, ConsumeResult result);
/// <inheritdoc /> /// <inheritdoc />
/// <summary> /// <summary>
/// Implements the bridge between the RabbitMQ Client consumer and a Tapeti Consumer /// Implements the bridge between the RabbitMQ Client consumer and a Tapeti Consumer
@ -12,13 +15,15 @@ namespace Tapeti.Connection
internal class TapetiBasicConsumer : DefaultBasicConsumer internal class TapetiBasicConsumer : DefaultBasicConsumer
{ {
private readonly IConsumer consumer; private readonly IConsumer consumer;
private readonly Func<ulong, ConsumeResult, Task> onRespond; private readonly long connectionReference;
private readonly ResponseFunc onRespond;
/// <inheritdoc /> /// <inheritdoc />
public TapetiBasicConsumer(IConsumer consumer, Func<ulong, ConsumeResult, Task> onRespond) public TapetiBasicConsumer(IConsumer consumer, long connectionReference, ResponseFunc onRespond)
{ {
this.consumer = consumer; this.consumer = consumer;
this.connectionReference = connectionReference;
this.onRespond = onRespond; this.onRespond = onRespond;
} }
@ -45,11 +50,11 @@ namespace Tapeti.Connection
try try
{ {
var response = await consumer.Consume(exchange, routingKey, new RabbitMQMessageProperties(properties), bodyArray); var response = await consumer.Consume(exchange, routingKey, new RabbitMQMessageProperties(properties), bodyArray);
await onRespond(deliveryTag, response); await onRespond(connectionReference, deliveryTag, response);
} }
catch catch
{ {
await onRespond(deliveryTag, ConsumeResult.Error); await onRespond(connectionReference, deliveryTag, ConsumeResult.Error);
} }
}); });
} }

87
Tapeti/Connection/TapetiClient.cs
View File

@ -51,6 +51,7 @@ namespace Tapeti.Connection
// These fields must be locked using connectionLock // These fields must be locked using connectionLock
private readonly object connectionLock = new(); private readonly object connectionLock = new();
private long connectionReference;
private RabbitMQ.Client.IConnection connection; private RabbitMQ.Client.IConnection connection;
private IModel consumeChannelModel; private IModel consumeChannelModel;
private IModel publishChannelModel; private IModel publishChannelModel;
@ -200,7 +201,7 @@ namespace Tapeti.Connection
/// <inheritdoc /> /// <inheritdoc />
public async Task<string> Consume(CancellationToken cancellationToken, string queueName, IConsumer consumer) public async Task<TapetiConsumerTag> Consume(CancellationToken cancellationToken, string queueName, IConsumer consumer)
{ {
if (deletedQueues.Contains(queueName)) if (deletedQueues.Contains(queueName))
return null; return null;
@ -209,6 +210,7 @@ namespace Tapeti.Connection
throw new ArgumentNullException(nameof(queueName)); throw new ArgumentNullException(nameof(queueName));
long capturedConnectionReference = -1;
string consumerTag = null; string consumerTag = null;
await GetTapetiChannel(TapetiChannelType.Consume).QueueRetryable(channel => await GetTapetiChannel(TapetiChannelType.Consume).QueueRetryable(channel =>
@ -216,33 +218,52 @@ namespace Tapeti.Connection
if (cancellationToken.IsCancellationRequested) if (cancellationToken.IsCancellationRequested)
return; return;
var basicConsumer = new TapetiBasicConsumer(consumer, Respond); capturedConnectionReference = Interlocked.Read(ref connectionReference);
var basicConsumer = new TapetiBasicConsumer(consumer, capturedConnectionReference, Respond);
consumerTag = channel.BasicConsume(queueName, false, basicConsumer); consumerTag = channel.BasicConsume(queueName, false, basicConsumer);
}); });
return consumerTag; return new TapetiConsumerTag(capturedConnectionReference, consumerTag);
} }
/// <inheritdoc /> /// <inheritdoc />
public async Task Cancel(string consumerTag) public async Task Cancel(TapetiConsumerTag consumerTag)
{ {
if (isClosing || string.IsNullOrEmpty(consumerTag)) if (isClosing || string.IsNullOrEmpty(consumerTag.ConsumerTag))
return;
var capturedConnectionReference = Interlocked.Read(ref connectionReference);
// If the connection was re-established in the meantime, don't respond with an
// invalid deliveryTag. The message will be requeued.
if (capturedConnectionReference != consumerTag.ConnectionReference)
return; return;
// No need for a retryable channel here, if the connection is lost // No need for a retryable channel here, if the connection is lost
// so is the consumer. // so is the consumer.
await GetTapetiChannel(TapetiChannelType.Consume).Queue(channel => await GetTapetiChannel(TapetiChannelType.Consume).Queue(channel =>
{ {
channel.BasicCancel(consumerTag); // Check again as a reconnect may have occured in the meantime
var currentConnectionReference = Interlocked.Read(ref connectionReference);
if (currentConnectionReference != consumerTag.ConnectionReference)
return;
channel.BasicCancel(consumerTag.ConsumerTag);
}); });
} }
private async Task Respond(ulong deliveryTag, ConsumeResult result) private async Task Respond(long expectedConnectionReference, ulong deliveryTag, ConsumeResult result)
{ {
await GetTapetiChannel(TapetiChannelType.Consume).Queue(channel => await GetTapetiChannel(TapetiChannelType.Consume).Queue(channel =>
{ {
// If the connection was re-established in the meantime, don't respond with an
// invalid deliveryTag. The message will be requeued.
var currentConnectionReference = Interlocked.Read(ref connectionReference);
if (currentConnectionReference != connectionReference)
return;
// No need for a retryable channel here, if the connection is lost we can't // No need for a retryable channel here, if the connection is lost we can't
// use the deliveryTag anymore. // use the deliveryTag anymore.
switch (result) switch (result)
@ -487,8 +508,8 @@ namespace Tapeti.Connection
await publishChannel.Reset(); await publishChannel.Reset();
// No need to close the channels as the connection will be closed // No need to close the channels as the connection will be closed
capturedConsumeModel.Dispose(); capturedConsumeModel?.Dispose();
capturedPublishModel.Dispose(); capturedPublishModel?.Dispose();
// ReSharper disable once InvertIf // ReSharper disable once InvertIf
if (capturedConnection != null) if (capturedConnection != null)
@ -695,6 +716,7 @@ namespace Tapeti.Connection
if (channel != null && channel.IsOpen) if (channel != null && channel.IsOpen)
return channel; return channel;
}
// If the Disconnect quickly follows the Connect (when an error occurs that is reported back by RabbitMQ // If the Disconnect quickly follows the Connect (when an error occurs that is reported back by RabbitMQ
// not related to the connection), wait for a bit to avoid spamming the connection // not related to the connection), wait for a bit to avoid spamming the connection
@ -728,12 +750,26 @@ namespace Tapeti.Connection
{ {
try try
{ {
if (connection != null) RabbitMQ.Client.IConnection capturedConnection;
IModel capturedConsumeChannelModel;
IModel capturedPublishChannelModel;
lock (connectionLock)
{
capturedConnection = connection;
}
if (capturedConnection != null)
{ {
try try
{ {
if (connection.IsOpen)
connection.Close(); connection.Close();
} }
catch (AlreadyClosedException)
{
}
finally finally
{ {
connection.Dispose(); connection.Dispose();
@ -743,8 +779,13 @@ namespace Tapeti.Connection
} }
logger.Connect(new ConnectContext(connectionParams, isReconnect)); logger.Connect(new ConnectContext(connectionParams, isReconnect));
Interlocked.Increment(ref connectionReference);
lock (connectionLock)
{
connection = connectionFactory.CreateConnection(); connection = connectionFactory.CreateConnection();
capturedConnection = connection;
consumeChannelModel = connection.CreateModel(); consumeChannelModel = connection.CreateModel();
if (consumeChannel == null) if (consumeChannel == null)
throw new BrokerUnreachableException(null); throw new BrokerUnreachableException(null);
@ -753,6 +794,10 @@ namespace Tapeti.Connection
if (publishChannel == null) if (publishChannel == null)
throw new BrokerUnreachableException(null); throw new BrokerUnreachableException(null);
capturedConsumeChannelModel = consumeChannelModel;
capturedPublishChannelModel = publishChannelModel;
}
if (config.Features.PublisherConfirms) if (config.Features.PublisherConfirms)
{ {
@ -771,14 +816,13 @@ namespace Tapeti.Connection
Monitor.Exit(confirmLock); Monitor.Exit(confirmLock);
} }
publishChannelModel.ConfirmSelect(); capturedPublishChannelModel.ConfirmSelect();
} }
if (connectionParams.PrefetchCount > 0) if (connectionParams.PrefetchCount > 0)
consumeChannelModel.BasicQos(0, connectionParams.PrefetchCount, false); capturedPublishChannelModel.BasicQos(0, connectionParams.PrefetchCount, false);
var capturedConsumeChannelModel = consumeChannelModel; capturedPublishChannelModel.ModelShutdown += (_, e) =>
consumeChannelModel.ModelShutdown += (_, e) =>
{ {
lock (connectionLock) lock (connectionLock)
{ {
@ -801,8 +845,7 @@ namespace Tapeti.Connection
GetTapetiChannel(TapetiChannelType.Consume).QueueRetryable(_ => { }); GetTapetiChannel(TapetiChannelType.Consume).QueueRetryable(_ => { });
}; };
var capturedPublishChannelModel = publishChannelModel; capturedPublishChannelModel.ModelShutdown += (_, _) =>
publishChannelModel.ModelShutdown += (_, _) =>
{ {
lock (connectionLock) lock (connectionLock)
{ {
@ -816,16 +859,16 @@ namespace Tapeti.Connection
}; };
publishChannelModel.BasicReturn += HandleBasicReturn; capturedPublishChannelModel.BasicReturn += HandleBasicReturn;
publishChannelModel.BasicAcks += HandleBasicAck; capturedPublishChannelModel.BasicAcks += HandleBasicAck;
publishChannelModel.BasicNacks += HandleBasicNack; capturedPublishChannelModel.BasicNacks += HandleBasicNack;
connectedDateTime = DateTime.UtcNow; connectedDateTime = DateTime.UtcNow;
var connectedEventArgs = new ConnectedEventArgs var connectedEventArgs = new ConnectedEventArgs
{ {
ConnectionParams = connectionParams, ConnectionParams = connectionParams,
LocalPort = connection.LocalPort LocalPort = capturedConnection.LocalPort
}; };
if (isReconnect) if (isReconnect)
@ -833,7 +876,7 @@ namespace Tapeti.Connection
else else
ConnectionEventListener?.Connected(connectedEventArgs); ConnectionEventListener?.Connected(connectedEventArgs);
logger.ConnectSuccess(new ConnectContext(connectionParams, isReconnect, connection.LocalPort)); logger.ConnectSuccess(new ConnectContext(connectionParams, isReconnect, capturedConnection.LocalPort));
isReconnect = true; isReconnect = true;
break; break;
@ -845,6 +888,8 @@ namespace Tapeti.Connection
} }
} }
lock (connectionLock)
{
return channelType == TapetiChannelType.Publish return channelType == TapetiChannelType.Publish
? publishChannelModel ? publishChannelModel
: consumeChannelModel; : consumeChannelModel;

4
Tapeti/Connection/TapetiSubscriber.cs
View File

@ -1,4 +1,4 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Threading; using System.Threading;
@ -13,7 +13,7 @@ namespace Tapeti.Connection
private readonly Func<ITapetiClient> clientFactory; private readonly Func<ITapetiClient> clientFactory;
private readonly ITapetiConfig config; private readonly ITapetiConfig config;
private bool consuming; private bool consuming;
private readonly List<string> consumerTags = new(); private readonly List<TapetiConsumerTag> consumerTags = new();
private CancellationTokenSource initializeCancellationTokenSource; private CancellationTokenSource initializeCancellationTokenSource;