GitHub-Mirror/Tapeti
1
0
Fork 0
mirror of synced 2024-11-22 01:13:49 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'release/2.7'

Browse Source
This commit is contained in:
Mark van Renswoude 2021-09-05 09:30:16 +02:00
commit 574a1fdfc2
47 changed files with 1970 additions and 926 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
Tapeti.Cmd/Commands/ExportCommand.cs
View File

@ -1,46 +0,0 @@
using RabbitMQ.Client;
using Tapeti.Cmd.Serialization;
namespace Tapeti.Cmd.Commands
{
public class ExportCommand
{
public IMessageSerializer MessageSerializer { get; set; }
public string QueueName { get; set; }
public bool RemoveMessages { get; set; }
public int? MaxCount { get; set; }
public int Execute(IModel channel)
{
var messageCount = 0;
while (!MaxCount.HasValue || messageCount < MaxCount.Value)
{
var result = channel.BasicGet(QueueName, false);
if (result == null)
// No more messages on the queue
break;
messageCount++;
MessageSerializer.Serialize(new Message
{
DeliveryTag = result.DeliveryTag,
Redelivered = result.Redelivered,
Exchange = result.Exchange,
RoutingKey = result.RoutingKey,
Queue = QueueName,
Properties = result.BasicProperties,
Body = result.Body.ToArray()
});
if (RemoveMessages)
channel.BasicAck(result.DeliveryTag, false);
}
return messageCount;
}
}
}

33
Tapeti.Cmd/Commands/ImportCommand.cs
View File

@ -1,33 +0,0 @@
using RabbitMQ.Client;
using Tapeti.Cmd.RateLimiter;
using Tapeti.Cmd.Serialization;
namespace Tapeti.Cmd.Commands
{
public class ImportCommand
{
public IMessageSerializer MessageSerializer { get; set; }
public bool DirectToQueue { get; set; }
public int Execute(IModel channel, IRateLimiter rateLimiter)
{
var messageCount = 0;
foreach (var message in MessageSerializer.Deserialize(channel))
{
rateLimiter.Execute(() =>
{
var exchange = DirectToQueue ? "" : message.Exchange;
var routingKey = DirectToQueue ? message.Queue : message.RoutingKey;
channel.BasicPublish(exchange, routingKey, message.Properties, message.Body);
messageCount++;
});
}
return messageCount;
}
}
}

43
Tapeti.Cmd/Commands/ShovelCommand.cs
View File

@ -1,43 +0,0 @@
using RabbitMQ.Client;
using Tapeti.Cmd.RateLimiter;
namespace Tapeti.Cmd.Commands
{
public class ShovelCommand
{
public string QueueName { get; set; }
public string TargetQueueName { get; set; }
public bool RemoveMessages { get; set; }
public int? MaxCount { get; set; }
public int Execute(IModel sourceChannel, IModel targetChannel, IRateLimiter rateLimiter)
{
var messageCount = 0;
while (!MaxCount.HasValue || messageCount < MaxCount.Value)
{
var result = sourceChannel.BasicGet(QueueName, false);
if (result == null)
// No more messages on the queue
break;
// Since RabbitMQ client 6 we need to copy the body before calling another channel method
// like BasicPublish, or the published body will be corrupted
var bodyCopy = result.Body.ToArray();
rateLimiter.Execute(() =>
{
targetChannel.BasicPublish("", TargetQueueName, result.BasicProperties, bodyCopy);
messageCount++;
if (RemoveMessages)
sourceChannel.BasicAck(result.DeliveryTag, false);
});
}
return messageCount;
}
}
}

259
Tapeti.Cmd/ConsoleHelper/ConsoleWrapper.cs Normal file
View File

@ -0,0 +1,259 @@
using System;
using System.Collections.Generic;
using System.Threading;
namespace Tapeti.Cmd.ConsoleHelper
{
public class ConsoleWrapper : IConsole
{
private readonly List<TemporaryWriter> temporaryWriters = new();
private bool temporaryActive;
private int temporaryCursorTop;
public ConsoleWrapper()
{
temporaryCursorTop = Console.CursorTop;
Console.CancelKeyPress += (_, args) =>
{
if (Cancelled)
return;
using var consoleWriter = GetPermanentWriter();
consoleWriter.WriteLine("Cancelling...");
args.Cancel = true;
Cancelled = true;
};
}
public void Dispose()
{
foreach (var writer in temporaryWriters)
writer.Dispose();
Console.CursorVisible = true;
GC.SuppressFinalize(this);
}
public bool Cancelled { get; private set; }
public IConsoleWriter GetPermanentWriter()
{
return new PermanentWriter(this);
}
public IConsoleWriter GetTemporaryWriter()
{
var writer = new TemporaryWriter(this, temporaryWriters.Count);
temporaryWriters.Add(writer);
return writer;
}
private void AcquirePermanent()
{
if (!temporaryActive)
return;
foreach (var writer in temporaryWriters)
{
Console.SetCursorPosition(0, temporaryCursorTop + writer.RelativePosition);
writer.Clear();
}
Console.SetCursorPosition(0, temporaryCursorTop);
Console.CursorVisible = true;
temporaryActive = false;
}
private void ReleasePermanent()
{
if (temporaryWriters.Count == 0)
{
temporaryCursorTop = Console.CursorTop;
return;
}
foreach (var writer in temporaryWriters)
{
writer.Restore();
Console.WriteLine();
}
// Store the cursor position afterwards to account for buffer scrolling
temporaryCursorTop = Console.CursorTop - temporaryWriters.Count;
Console.CursorVisible = false;
temporaryActive = true;
}
private void AcquireTemporary(TemporaryWriter writer)
{
Console.SetCursorPosition(0, temporaryCursorTop + writer.RelativePosition);
if (temporaryActive)
return;
Console.CursorVisible = false;
temporaryActive = true;
}
private void DisposeWriter(BaseWriter writer)
{
if (writer is not TemporaryWriter temporaryWriter)
return;
Console.SetCursorPosition(0, temporaryCursorTop + temporaryWriter.RelativePosition);
temporaryWriter.Clear();
temporaryWriters.Remove(temporaryWriter);
}
private abstract class BaseWriter : IConsoleWriter
{
protected readonly ConsoleWrapper Owner;
protected BaseWriter(ConsoleWrapper owner)
{
Owner = owner;
}
public virtual void Dispose()
{
Owner.DisposeWriter(this);
GC.SuppressFinalize(this);
}
public abstract bool Enabled { get; }
public abstract void WriteLine(string value);
public void Confirm(string message)
{
WriteLine(message);
TryReadKey(false, out var _);
}
public bool ConfirmYesNo(string message)
{
WriteLine($"{message} (Y/N) ");
if (!TryReadKey(true, out var key))
return false;
return key.KeyChar == 'y' || key.KeyChar == 'Y';
}
private bool TryReadKey(bool showKeyOutput, out ConsoleKeyInfo keyInfo)
{
while (!Owner.Cancelled && !Console.KeyAvailable)
Thread.Sleep(50);
if (Owner.Cancelled)
{
keyInfo = default;
return false;
}
keyInfo = Console.ReadKey(!showKeyOutput);
return true;
}
}
private class PermanentWriter : BaseWriter
{
public PermanentWriter(ConsoleWrapper owner) : base(owner)
{
}
public override bool Enabled => true;
public override void WriteLine(string value)
{
Owner.AcquirePermanent();
try
{
Console.WriteLine(value);
}
finally
{
Owner.ReleasePermanent();
}
}
}
private class TemporaryWriter : BaseWriter
{
public int RelativePosition { get; }
private bool isActive;
private string storedValue;
public TemporaryWriter(ConsoleWrapper owner, int relativePosition) : base(owner)
{
RelativePosition = relativePosition;
}
public override bool Enabled => !Console.IsOutputRedirected;
public override void WriteLine(string value)
{
if (!Enabled)
return;
Owner.AcquireTemporary(this);
Console.Write(value);
if (!string.IsNullOrEmpty(storedValue) && storedValue.Length > value.Length)
// Clear characters remaining from the previous value
Console.Write(new string(' ', storedValue.Length - value.Length));
storedValue = value;
isActive = true;
}
public void Clear()
{
if (!isActive)
return;
if (!string.IsNullOrEmpty(storedValue))
Console.Write(new string(' ', storedValue.Length));
isActive = false;
}
public void Restore()
{
if (string.IsNullOrEmpty(storedValue))
return;
Console.Write(storedValue);
isActive = true;
}
}
}
}

44
Tapeti.Cmd/ConsoleHelper/IConsole.cs Normal file
View File

@ -0,0 +1,44 @@
using System;
namespace Tapeti.Cmd.ConsoleHelper
{
/// <summary>
/// Wraps access to the console to provide cooperation between temporary outputs like the
/// progress bar and batch confirmations. Temporary outputs hide the cursor and will be
/// automatically be erased and restored when a permanent writer is called.
/// </summary>
/// <remarks>
/// Temporary outputs are automatically supressed when the console output is redirected.
/// The Enabled property will reflect this.
/// </remarks>
public interface IConsole : IDisposable
{
bool Cancelled { get; }
IConsoleWriter GetPermanentWriter();
IConsoleWriter GetTemporaryWriter();
}
/// <summary>
/// For simplicity outputs only support one line of text.
/// For temporary writers, each call to WriteLine will overwrite the previous and clear any
/// extra characters if the previous value was longer.
/// </summary>
public interface IConsoleWriter : IDisposable
{
bool Enabled { get; }
void WriteLine(string value);
/// <summary>
/// Waits for any user input.
/// </summary>
void Confirm(string message);
/// <summary>
/// Waits for user confirmation (Y/N).
/// </summary>
bool ConfirmYesNo(string message);
}
}

90
Tapeti.Cmd/ConsoleHelper/ProgressBar.cs Normal file
View File

@ -0,0 +1,90 @@
using System;
using System.Text;
namespace Tapeti.Cmd.ConsoleHelper
{
public class ProgressBar : IDisposable, IProgress<int>
{
private static readonly TimeSpan UpdateInterval = TimeSpan.FromMilliseconds(20);
private readonly IConsoleWriter consoleWriter;
private readonly int max;
private readonly int width;
private readonly bool showPosition;
private int position;
private readonly bool enabled;
private DateTime lastUpdate = DateTime.MinValue;
public ProgressBar(IConsole console, int max, int width = 10, bool showPosition = true)
{
if (width <= 0)
throw new ArgumentOutOfRangeException(nameof(width), "Width must be greater than zero");
if (max <= 0)
throw new ArgumentOutOfRangeException(nameof(max), "Max must be greater than zero");
consoleWriter = console.GetTemporaryWriter();
this.max = max;
this.width = width;
this.showPosition = showPosition;
enabled = consoleWriter.Enabled;
if (!enabled)
return;
Redraw();
}
public void Dispose()
{
consoleWriter.Dispose();
}
public void Report(int value)
{
if (!enabled)
return;
value = Math.Max(0, Math.Min(max, value));
position = value;
var now = DateTime.Now;
if (now - lastUpdate < UpdateInterval)
return;
lastUpdate = now;
Redraw();
}
private void Redraw()
{
var output = new StringBuilder("[");
var blockCount = (int)Math.Truncate((decimal)position / max * width);
if (blockCount > 0)
output.Append(new string('#', blockCount));
if (blockCount < width)
output.Append(new string('.', width - blockCount));
output.Append("] ");
if (showPosition)
{
output
.Append(position.ToString("N0")).Append(" / ").Append(max.ToString("N0"))
.Append(" (").Append((int) Math.Truncate((decimal) position / max * 100)).Append("%)");
}
else
output.Append(" ").Append((int)Math.Truncate((decimal)position / max * 100)).Append("%");
consoleWriter.WriteLine(output.ToString());
}
}
}

23
Tapeti.Cmd/Parser/BindingParser.cs Normal file
View File

@ -0,0 +1,23 @@
using System;
using System.Collections.Generic;
using System.Linq;
namespace Tapeti.Cmd.Parser
{
public static class BindingParser
{
public static Tuple<string, string>[] Parse(IEnumerable<string> bindings)
{
return bindings
.Select(b =>
{
var parts = b.Split(':');
if (parts.Length != 2)
throw new InvalidOperationException($"Invalid binding format: {b}");
return new Tuple<string, string>(parts[0], parts[1]);
})
.ToArray();
}
}
}

632
Tapeti.Cmd/Program.cs
View File

@ -1,602 +1,60 @@
using System; using System;
using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.IO;
using System.Linq; using System.Linq;
using System.Text; using System.Reflection;
using CommandLine; using CommandLine;
using RabbitMQ.Client; using Tapeti.Cmd.ConsoleHelper;
using RabbitMQ.Client.Exceptions; using Tapeti.Cmd.Verbs;
using Tapeti.Cmd.Commands;
using Tapeti.Cmd.Mock;
using Tapeti.Cmd.RateLimiter;
using Tapeti.Cmd.Serialization;
namespace Tapeti.Cmd namespace Tapeti.Cmd
{ {
public class Program public class Program
{ {
public class CommonOptions
{
[Option('h', "host", HelpText = "Hostname of the RabbitMQ server.", Default = "localhost")]
public string Host { get; set; }
[Option("port", HelpText = "AMQP port of the RabbitMQ server.", Default = 5672)]
public int Port { get; set; }
[Option('v', "virtualhost", HelpText = "Virtual host used for the RabbitMQ connection.", Default = "/")]
public string VirtualHost { get; set; }
[Option('u', "username", HelpText = "Username used to connect to the RabbitMQ server.", Default = "guest")]
public string Username { get; set; }
[Option('p', "password", HelpText = "Password used to connect to the RabbitMQ server.", Default = "guest")]
public string Password { get; set; }
}
public enum SerializationMethod
{
SingleFileJSON,
EasyNetQHosepipe
}
public class MessageSerializerOptions : CommonOptions
{
[Option('s', "serialization", HelpText = "The method used to serialize the message for import or export. Valid options: SingleFileJSON, EasyNetQHosepipe.", Default = SerializationMethod.SingleFileJSON)]
public SerializationMethod SerializationMethod { get; set; }
}
[Verb("export", HelpText = "Fetch messages from a queue and write it to disk.")]
public class ExportOptions : MessageSerializerOptions
{
[Option('q', "queue", Required = true, HelpText = "The queue to read the messages from.")]
public string QueueName { get; set; }
[Option('o', "output", Required = true, HelpText = "Path or filename (depending on the chosen serialization method) where the messages will be output to.")]
public string OutputPath { get; set; }
[Option('r', "remove", HelpText = "If specified messages are acknowledged and removed from the queue. If not messages are kept.")]
public bool RemoveMessages { get; set; }
[Option('n', "maxcount", HelpText = "(Default: all) Maximum number of messages to retrieve from the queue.")]
public int? MaxCount { get; set; }
}
[Verb("import", HelpText = "Read messages from disk as previously exported and publish them to a queue.")]
public class ImportOptions : MessageSerializerOptions
{
[Option('i', "input", Group = "Input", HelpText = "Path or filename (depending on the chosen serialization method) where the messages will be read from.")]
public string InputFile { get; set; }
[Option('m', "message", Group = "Input", HelpText = "Single message to be sent, in the same format as used for SingleFileJSON. Serialization argument has no effect when using this input.")]
public string InputMessage { get; set; }
[Option('c', "pipe", Group = "Input", HelpText = "Messages are read from the standard input pipe, in the same format as used for SingleFileJSON. Serialization argument has no effect when using this input.")]
public bool InputPipe { get; set; }
[Option('e', "exchange", HelpText = "If specified publishes to the originating exchange using the original routing key. By default these are ignored and the message is published directly to the originating queue.")]
public bool PublishToExchange { get; set; }
[Option("maxrate", HelpText = "The maximum amount of messages per second to import.")]
public int? MaxRate { get; set; }
}
[Verb("example", HelpText = "Output an example SingleFileJSON formatted message.")]
public class ExampleOptions
{
}
[Verb("shovel", HelpText = "Reads messages from a queue and publishes them to another queue, optionally to another RabbitMQ server.")]
public class ShovelOptions : CommonOptions
{
[Option('q', "queue", Required = true, HelpText = "The queue to read the messages from.")]
public string QueueName { get; set; }
[Option('t', "targetqueue", HelpText = "The target queue to publish the messages to. Defaults to the source queue if a different target host, port or virtualhost is specified. Otherwise it must be different from the source queue.")]
public string TargetQueueName { get; set; }
[Option('r', "remove", HelpText = "If specified messages are acknowledged and removed from the source queue. If not messages are kept.")]
public bool RemoveMessages { get; set; }
[Option('n', "maxcount", HelpText = "(Default: all) Maximum number of messages to retrieve from the queue.")]
public int? MaxCount { get; set; }
[Option("targethost", HelpText = "Hostname of the target RabbitMQ server. Defaults to the source host. Note that you may still specify a different targetusername for example.")]
public string TargetHost { get; set; }
[Option("targetport", HelpText = "AMQP port of the target RabbitMQ server. Defaults to the source port.")]
public int? TargetPort { get; set; }
[Option("targetvirtualhost", HelpText = "Virtual host used for the target RabbitMQ connection. Defaults to the source virtualhost.")]
public string TargetVirtualHost { get; set; }
[Option("targetusername", HelpText = "Username used to connect to the target RabbitMQ server. Defaults to the source username.")]
public string TargetUsername { get; set; }
[Option("targetpassword", HelpText = "Password used to connect to the target RabbitMQ server. Defaults to the source password.")]
public string TargetPassword { get; set; }
[Option("maxrate", HelpText = "The maximum amount of messages per second to shovel.")]
public int? MaxRate { get; set; }
}
[Verb("purge", HelpText = "Removes all messages from a queue destructively.")]
public class PurgeOptions : CommonOptions
{
[Option('q', "queue", Required = true, HelpText = "The queue to purge.")]
public string QueueName { get; set; }
[Option("confirm", HelpText = "Confirms the purging of the specified queue. If not provided, an interactive prompt will ask for confirmation.", Default = false)]
public bool Confirm { get; set; }
}
[Verb("declarequeue", HelpText = "Declares a durable queue without arguments, compatible with Tapeti.")]
public class DeclareQueueOptions : CommonOptions
{
[Option('q', "queue", Required = true, HelpText = "The name of the queue to declare.")]
public string QueueName { get; set; }
[Option('b', "bindings", Required = false, HelpText = "One or more bindings to add to the queue. Format: <exchange>:<routingKey>")]
public IEnumerable<string> Bindings { get; set; }
}
[Verb("removequeue", HelpText = "Removes a durable queue.")]
public class RemoveQueueOptions : CommonOptions
{
[Option('q', "queue", Required = true, HelpText = "The name of the queue to remove.")]
public string QueueName { get; set; }
[Option("confirm", HelpText = "Confirms the removal of the specified queue. If not provided, an interactive prompt will ask for confirmation.", Default = false)]
public bool Confirm { get; set; }
[Option("confirmpurge", HelpText = "Confirms the removal of the specified queue even if there still are messages in the queue. If not provided, an interactive prompt will ask for confirmation.", Default = false)]
public bool ConfirmPurge { get; set; }
}
[Verb("bindqueue", HelpText = "Add a binding to a queue.")]
public class BindQueueOptions : CommonOptions
{
[Option('q', "queue", Required = true, HelpText = "The name of the queue to add the binding(s) to.")]
public string QueueName { get; set; }
[Option('b', "bindings", Required = false, HelpText = "One or more bindings to add to the queue. Format: <exchange>:<routingKey>")]
public IEnumerable<string> Bindings { get; set; }
}
[Verb("unbindqueue", HelpText = "Remove a binding from a queue.")]
public class UnbindQueueOptions : CommonOptions
{
[Option('q', "queue", Required = true, HelpText = "The name of the queue to remove the binding(s) from.")]
public string QueueName { get; set; }
[Option('b', "bindings", Required = false, HelpText = "One or more bindings to remove from the queue. Format: <exchange>:<routingKey>")]
public IEnumerable<string> Bindings { get; set; }
}
public static int Main(string[] args) public static int Main(string[] args)
{ {
return Parser.Default.ParseArguments<ExportOptions, ImportOptions, ShovelOptions, PurgeOptions, ExampleOptions, var exitCode = 1;
DeclareQueueOptions, RemoveQueueOptions, BindQueueOptions, UnbindQueueOptions>(args) var verbTypes = Assembly.GetExecutingAssembly().GetTypes()
.MapResult( .Where(t => t.GetCustomAttribute<ExecutableVerbAttribute>() != null)
(ExportOptions o) => ExecuteVerb(o, RunExport),
(ImportOptions o) => ExecuteVerb(o, RunImport),
(ExampleOptions o) => ExecuteVerb(o, RunExample),
(ShovelOptions o) => ExecuteVerb(o, RunShovel),
(PurgeOptions o) => ExecuteVerb(o, RunPurge),
(DeclareQueueOptions o) => ExecuteVerb(o, RunDeclareQueue),
(RemoveQueueOptions o) => ExecuteVerb(o, RunRemoveQueue),
(BindQueueOptions o) => ExecuteVerb(o, RunBindQueue),
(UnbindQueueOptions o) => ExecuteVerb(o, RunUnbindQueue),
errs =>
{
if (!Debugger.IsAttached)
return 1;
Console.WriteLine("Press any Enter key to continue...");
Console.ReadLine();
return 1;
}
);
}
private static int ExecuteVerb<T>(T options, Action<T> execute) where T : class
{
try
{
execute(options);
return 0;
}
catch (Exception e)
{
Console.WriteLine(e.Message);
return 1;
}
}
private static IConnection GetConnection(CommonOptions options)
{
var factory = new ConnectionFactory
{
HostName = options.Host,
Port = options.Port,
VirtualHost = options.VirtualHost,
UserName = options.Username,
Password = options.Password
};
return factory.CreateConnection();
}
private static IMessageSerializer GetMessageSerializer(ImportOptions options)
{
switch (options.SerializationMethod)
{
case SerializationMethod.SingleFileJSON:
return new SingleFileJSONMessageSerializer(GetInputStream(options, out var disposeStream), disposeStream, Encoding.UTF8);
case SerializationMethod.EasyNetQHosepipe:
if (string.IsNullOrEmpty(options.InputFile))
throw new ArgumentException("An input path must be provided when using EasyNetQHosepipe serialization");
return new EasyNetQMessageSerializer(options.InputFile);
default:
throw new ArgumentOutOfRangeException(nameof(options.SerializationMethod), options.SerializationMethod, "Invalid SerializationMethod");
}
}
private static Stream GetInputStream(ImportOptions options, out bool disposeStream)
{
if (options.InputPipe)
{
disposeStream = false;
return Console.OpenStandardInput();
}
if (!string.IsNullOrEmpty(options.InputMessage))
{
disposeStream = true;
return new MemoryStream(Encoding.UTF8.GetBytes(options.InputMessage));
}
disposeStream = true;
return new FileStream(options.InputFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
}
private static IMessageSerializer GetMessageSerializer(ExportOptions options)
{
switch (options.SerializationMethod)
{
case SerializationMethod.SingleFileJSON:
return new SingleFileJSONMessageSerializer(GetOutputStream(options, out var disposeStream), disposeStream, Encoding.UTF8);
case SerializationMethod.EasyNetQHosepipe:
if (string.IsNullOrEmpty(options.OutputPath))
throw new ArgumentException("An output path must be provided when using EasyNetQHosepipe serialization");
return new EasyNetQMessageSerializer(options.OutputPath);
default:
throw new ArgumentOutOfRangeException(nameof(options.SerializationMethod), options.SerializationMethod, "Invalid SerializationMethod");
}
}
private static Stream GetOutputStream(ExportOptions options, out bool disposeStream)
{
disposeStream = true;
return new FileStream(options.OutputPath, FileMode.Create, FileAccess.Write, FileShare.Read);
}
private static IRateLimiter GetRateLimiter(int? maxRate)
{
if (!maxRate.HasValue || maxRate.Value <= 0)
return new NoRateLimiter();
return new SpreadRateLimiter(maxRate.Value, TimeSpan.FromSeconds(1));
}
private static void RunExport(ExportOptions options)
{
int messageCount;
using (var messageSerializer = GetMessageSerializer(options))
using (var connection = GetConnection(options))
using (var channel = connection.CreateModel())
{
messageCount = new ExportCommand
{
MessageSerializer = messageSerializer,
QueueName = options.QueueName,
RemoveMessages = options.RemoveMessages,
MaxCount = options.MaxCount
}.Execute(channel);
}
Console.WriteLine($"{messageCount} message{(messageCount != 1 ? "s" : "")} exported.");
}
private static void RunImport(ImportOptions options)
{
int messageCount;
using (var messageSerializer = GetMessageSerializer(options))
using (var connection = GetConnection(options))
using (var channel = connection.CreateModel())
{
messageCount = new ImportCommand
{
MessageSerializer = messageSerializer,
DirectToQueue = !options.PublishToExchange
}.Execute(channel, GetRateLimiter(options.MaxRate));
}
Console.WriteLine($"{messageCount} message{(messageCount != 1 ? "s" : "")} published.");
}
private static void RunExample(ExampleOptions options)
{
using (var messageSerializer = new SingleFileJSONMessageSerializer(Console.OpenStandardOutput(), false, new UTF8Encoding(false)))
{
messageSerializer.Serialize(new Message
{
Exchange = "example",
Queue = "example.queue",
RoutingKey = "example.routing.key",
DeliveryTag = 42,
Properties = new MockBasicProperties
{
ContentType = "application/json",
DeliveryMode = 2,
Headers = new Dictionary<string, object>
{
{ "classType", Encoding.UTF8.GetBytes("Tapeti.Cmd.Example:Tapeti.Cmd") }
},
ReplyTo = "reply.queue",
Timestamp = new AmqpTimestamp(new DateTimeOffset(DateTime.UtcNow).ToUnixTimeSeconds())
},
Body = Encoding.UTF8.GetBytes("{ \"Hello\": \"world!\" }")
});
}
}
private static void RunShovel(ShovelOptions options)
{
int messageCount;
using (var sourceConnection = GetConnection(options))
using (var sourceChannel = sourceConnection.CreateModel())
{
var shovelCommand = new ShovelCommand
{
QueueName = options.QueueName,
TargetQueueName = !string.IsNullOrEmpty(options.TargetQueueName) ? options.TargetQueueName : options.QueueName,
RemoveMessages = options.RemoveMessages,
MaxCount = options.MaxCount
};
if (RequiresSecondConnection(options))
{
using (var targetConnection = GetTargetConnection(options))
using (var targetChannel = targetConnection.CreateModel())
{
messageCount = shovelCommand.Execute(sourceChannel, targetChannel, GetRateLimiter(options.MaxRate));
}
}
else
messageCount = shovelCommand.Execute(sourceChannel, sourceChannel, GetRateLimiter(options.MaxRate));
}
Console.WriteLine($"{messageCount} message{(messageCount != 1 ? "s" : "")} shoveled.");
}
private static bool RequiresSecondConnection(ShovelOptions options)
{
if (!string.IsNullOrEmpty(options.TargetHost) && options.TargetHost != options.Host)
return true;
if (options.TargetPort.HasValue && options.TargetPort.Value != options.Port)
return true;
if (!string.IsNullOrEmpty(options.TargetVirtualHost) && options.TargetVirtualHost != options.VirtualHost)
return true;
// All relevant target host parameters are either omitted or the same. This means the queue must be different
// to prevent an infinite loop.
if (string.IsNullOrEmpty(options.TargetQueueName) || options.TargetQueueName == options.QueueName)
throw new ArgumentException("Target queue must be different from the source queue when shoveling within the same (virtual) host");
if (!string.IsNullOrEmpty(options.TargetUsername) && options.TargetUsername != options.Username)
return true;
// ReSharper disable once ConvertIfStatementToReturnStatement
if (!string.IsNullOrEmpty(options.TargetPassword) && options.TargetPassword != options.Password)
return true;
// Everything's the same, we can use the same channel
return false;
}
private static IConnection GetTargetConnection(ShovelOptions options)
{
var factory = new ConnectionFactory
{
HostName = !string.IsNullOrEmpty(options.TargetHost) ? options.TargetHost : options.Host,
Port = options.TargetPort ?? options.Port,
VirtualHost = !string.IsNullOrEmpty(options.TargetVirtualHost) ? options.TargetVirtualHost : options.VirtualHost,
UserName = !string.IsNullOrEmpty(options.TargetUsername) ? options.TargetUsername : options.Username,
Password = !string.IsNullOrEmpty(options.TargetPassword) ? options.TargetPassword : options.Password
};
return factory.CreateConnection();
}
private static void RunPurge(PurgeOptions options)
{
if (!options.Confirm)
{
Console.Write($"Do you want to purge the queue '{options.QueueName}'? (Y/N) ");
var answer = Console.ReadLine();
if (string.IsNullOrEmpty(answer) || !answer.Equals("Y", StringComparison.CurrentCultureIgnoreCase))
return;
}
uint messageCount;
using (var connection = GetConnection(options))
using (var channel = connection.CreateModel())
{
messageCount = channel.QueuePurge(options.QueueName);
}
Console.WriteLine($"{messageCount} message{(messageCount != 1 ? "s" : "")} purged from '{options.QueueName}'.");
}
private static void RunDeclareQueue(DeclareQueueOptions options)
{
// Parse early to fail early
var bindings = ParseBindings(options.Bindings);
using (var connection = GetConnection(options))
using (var channel = connection.CreateModel())
{
channel.QueueDeclare(options.QueueName, true, false, false);
foreach (var (exchange, routingKey) in bindings)
channel.QueueBind(options.QueueName, exchange, routingKey);
}
Console.WriteLine($"Queue {options.QueueName} declared with {bindings.Length} binding{(bindings.Length != 1 ? "s" : "")}.");
}
private static void RunRemoveQueue(RemoveQueueOptions options)
{
if (!options.Confirm)
{
Console.Write($"Do you want to remove the queue '{options.QueueName}'? (Y/N) ");
var answer = Console.ReadLine();
if (string.IsNullOrEmpty(answer) || !answer.Equals("Y", StringComparison.CurrentCultureIgnoreCase))
return;
}
uint messageCount;
try
{
using (var connection = GetConnection(options))
using (var channel = connection.CreateModel())
{
messageCount = channel.QueueDelete(options.QueueName, true, true);
}
}
catch (OperationInterruptedException e)
{
if (e.ShutdownReason.ReplyCode == 406)
{
if (!options.ConfirmPurge)
{
Console.Write($"There are messages remaining. Do you want to purge the queue '{options.QueueName}'? (Y/N) ");
var answer = Console.ReadLine();
if (string.IsNullOrEmpty(answer) || !answer.Equals("Y", StringComparison.CurrentCultureIgnoreCase))
return;
}
using (var connection = GetConnection(options))
using (var channel = connection.CreateModel())
{
messageCount = channel.QueueDelete(options.QueueName, true, false);
}
}
else
throw;
}
Console.WriteLine(messageCount == 0
? $"Empty or non-existent queue '{options.QueueName}' removed."
: $"{messageCount} message{(messageCount != 1 ? "s" : "")} purged while removing '{options.QueueName}'.");
}
private static void RunBindQueue(BindQueueOptions options)
{
var bindings = ParseBindings(options.Bindings);
using (var connection = GetConnection(options))
using (var channel = connection.CreateModel())
{
foreach (var (exchange, routingKey) in bindings)
channel.QueueBind(options.QueueName, exchange, routingKey);
}
Console.WriteLine($"{bindings.Length} binding{(bindings.Length != 1 ? "s" : "")} added to queue {options.QueueName}.");
}
private static void RunUnbindQueue(UnbindQueueOptions options)
{
var bindings = ParseBindings(options.Bindings);
using (var connection = GetConnection(options))
using (var channel = connection.CreateModel())
{
foreach (var (exchange, routingKey) in bindings)
channel.QueueUnbind(options.QueueName, exchange, routingKey);
}
Console.WriteLine($"{bindings.Length} binding{(bindings.Length != 1 ? "s" : "")} removed from queue {options.QueueName}.");
}
private static Tuple<string, string>[] ParseBindings(IEnumerable<string> bindings)
{
return bindings
.Select(b =>
{
var parts = b.Split(':');
if (parts.Length != 2)
throw new InvalidOperationException($"Invalid binding format: {b}");
return new Tuple<string, string>(parts[0], parts[1]);
})
.ToArray(); .ToArray();
CommandLine.Parser.Default.ParseArguments(args, verbTypes.ToArray())
.WithParsed(o =>
{
try
{
var executableVerbAttribute = o.GetType().GetCustomAttribute<ExecutableVerbAttribute>();
var executer = Activator.CreateInstance(executableVerbAttribute.VerbExecuter, o) as IVerbExecuter;
// Should have been validated by the ExecutableVerbAttribute
Debug.Assert(executer != null, nameof(executer) + " != null");
using var consoleWrapper = new ConsoleWrapper();
executer.Execute(consoleWrapper);
exitCode = 0;
}
catch (Exception e)
{
Console.WriteLine(e.Message);
DebugConfirmClose();
}
})
.WithNotParsed(_ =>
{
DebugConfirmClose();
});
return exitCode;
}
private static void DebugConfirmClose()
{
if (!Debugger.IsAttached)
return;
Console.WriteLine("Press any Enter key to continue...");
Console.ReadLine();
} }
} }
} }

80
Tapeti.Cmd/RateLimiter/BatchSizeRateLimiter.cs Normal file
View File

@ -0,0 +1,80 @@
using System;
using System.Threading;
using Tapeti.Cmd.ConsoleHelper;
namespace Tapeti.Cmd.RateLimiter
{
public abstract class BaseBatchSizeRateLimiter : IRateLimiter
{
private readonly IConsole console;
private readonly IRateLimiter decoratedRateLimiter;
private readonly int batchSize;
private int batchCount;
protected BaseBatchSizeRateLimiter(IConsole console, IRateLimiter decoratedRateLimiter, int batchSize)
{
this.console = console;
this.decoratedRateLimiter = decoratedRateLimiter;
this.batchSize = batchSize;
}
public void Execute(Action action)
{
batchCount++;
if (batchCount > batchSize)
{
Pause(console);
batchCount = 0;
}
decoratedRateLimiter.Execute(action);
}
protected abstract void Pause(IConsole console);
}
public class ManualBatchSizeRateLimiter : BaseBatchSizeRateLimiter
{
public ManualBatchSizeRateLimiter(IConsole console, IRateLimiter decoratedRateLimiter, int batchSize) : base(console, decoratedRateLimiter, batchSize)
{
}
protected override void Pause(IConsole console)
{
using var consoleWriter = console.GetTemporaryWriter();
consoleWriter.Confirm("Press any key to continue with the next batch...");
}
}
public class TimedBatchSizeRateLimiter : BaseBatchSizeRateLimiter
{
private readonly int batchPauseTime;
public TimedBatchSizeRateLimiter(IConsole console, IRateLimiter decoratedRateLimiter, int batchSize, int batchPauseTime) : base(console, decoratedRateLimiter, batchSize)
{
this.batchPauseTime = batchPauseTime;
}
protected override void Pause(IConsole console)
{
using var consoleWriter = console.GetTemporaryWriter();
var remaining = batchPauseTime;
while (remaining > 0 && !console.Cancelled)
{
consoleWriter.WriteLine($"Next batch in {remaining} second{(remaining != 1 ? "s" : "")}...");
Thread.Sleep(1000);
remaining--;
}
}
}
}

29
Tapeti.Cmd/RateLimiter/RateLimiterFactory.cs Normal file
View File

@ -0,0 +1,29 @@
using System;
using Tapeti.Cmd.ConsoleHelper;
namespace Tapeti.Cmd.RateLimiter
{
public static class RateLimiterFactory
{
public static IRateLimiter Create(IConsole console, int? maxRate, int? batchSize, int? batchPauseTime)
{
IRateLimiter rateLimiter;
if (maxRate > 0)
rateLimiter = new SpreadRateLimiter(maxRate.Value, TimeSpan.FromSeconds(1));
else
rateLimiter = new NoRateLimiter();
// ReSharper disable once InvertIf - I don't like the readability of that flow
if (batchSize > 0)
{
if (batchPauseTime > 0)
rateLimiter = new TimedBatchSizeRateLimiter(console, rateLimiter, batchSize.Value, batchPauseTime.Value);
else
rateLimiter = new ManualBatchSizeRateLimiter(console, rateLimiter, batchSize.Value);
}
return rateLimiter;
}
}
}

19
Tapeti.Cmd/Serialization/EasyNetQMessageSerializer.cs
View File

@ -11,22 +11,23 @@ namespace Tapeti.Cmd.Serialization
{ {
public class EasyNetQMessageSerializer : IMessageSerializer public class EasyNetQMessageSerializer : IMessageSerializer
{ {
private static readonly Regex InvalidCharRegex = new Regex(@"[\\\/:\*\?\""\<\>|]", RegexOptions.Compiled); private static readonly Regex InvalidCharRegex = new(@"[\\\/:\*\?\""\<\>|]", RegexOptions.Compiled);
private readonly string path;
private readonly Lazy<string> writablePath; private readonly Lazy<string> writablePath;
private int messageCount; private int messageCount;
private readonly Lazy<string[]> files;
public EasyNetQMessageSerializer(string path) public EasyNetQMessageSerializer(string path)
{ {
this.path = path;
writablePath = new Lazy<string>(() => writablePath = new Lazy<string>(() =>
{ {
Directory.CreateDirectory(path); Directory.CreateDirectory(path);
return path; return path;
}); });
files = new Lazy<string[]>(() => Directory.GetFiles(path, "*.*.message.txt"));
} }
@ -60,9 +61,15 @@ namespace Tapeti.Cmd.Serialization
} }
public int GetMessageCount()
{
return files.Value.Length;
}
public IEnumerable<Message> Deserialize(IModel channel) public IEnumerable<Message> Deserialize(IModel channel)
{ {
foreach (var file in Directory.GetFiles(path, "*.*.message.txt")) foreach (var file in files.Value)
{ {
const string messageTag = ".message."; const string messageTag = ".message.";
@ -303,7 +310,7 @@ namespace Tapeti.Cmd.Serialization
public Message ToMessage() public Message ToMessage()
{ {
return new Message return new()
{ {
//ConsumerTag = //ConsumerTag =
DeliveryTag = DeliverTag, DeliveryTag = DeliverTag,

2
Tapeti.Cmd/Serialization/IMessageSerializer.cs
View File

@ -19,6 +19,8 @@ namespace Tapeti.Cmd.Serialization
public interface IMessageSerializer : IDisposable public interface IMessageSerializer : IDisposable
{ {
void Serialize(Message message); void Serialize(Message message);
int GetMessageCount();
IEnumerable<Message> Deserialize(IModel channel); IEnumerable<Message> Deserialize(IModel channel);
} }
} }

54
Tapeti.Cmd/Serialization/SingleFileJSONMessageSerializer.cs
View File

@ -14,8 +14,11 @@ namespace Tapeti.Cmd.Serialization
private readonly bool disposeStream; private readonly bool disposeStream;
private readonly Encoding encoding; private readonly Encoding encoding;
// StreamReader.DefaultBufferSize is private :-/
private const int DefaultBufferSize = 1024;
private static readonly JsonSerializerSettings SerializerSettings = new JsonSerializerSettings
private static readonly JsonSerializerSettings SerializerSettings = new()
{ {
NullValueHandling = NullValueHandling.Ignore NullValueHandling = NullValueHandling.Ignore
}; };
@ -40,23 +43,48 @@ namespace Tapeti.Cmd.Serialization
exportFile.Value.WriteLine(serialized); exportFile.Value.WriteLine(serialized);
} }
public int GetMessageCount()
{
if (!stream.CanSeek)
return 0;
var position = stream.Position;
try
{
var lineCount = 0;
using var reader = new StreamReader(stream, encoding, true, DefaultBufferSize, true);
while (!reader.EndOfStream)
{
if (!string.IsNullOrEmpty(reader.ReadLine()))
lineCount++;
}
return lineCount;
}
finally
{
stream.Position = position;
}
}
public IEnumerable<Message> Deserialize(IModel channel) public IEnumerable<Message> Deserialize(IModel channel)
{ {
using (var reader = new StreamReader(stream, encoding)) using var reader = new StreamReader(stream, encoding, true, DefaultBufferSize, true);
while (!reader.EndOfStream)
{ {
while (!reader.EndOfStream) var serialized = reader.ReadLine();
{ if (string.IsNullOrEmpty(serialized))
var serialized = reader.ReadLine(); continue;
if (string.IsNullOrEmpty(serialized))
continue;
var serializableMessage = JsonConvert.DeserializeObject<SerializableMessage>(serialized); var serializableMessage = JsonConvert.DeserializeObject<SerializableMessage>(serialized);
if (serializableMessage == null) if (serializableMessage == null)
continue; continue;
yield return serializableMessage.ToMessage(channel); yield return serializableMessage.ToMessage(channel);
}
} }
} }
@ -135,7 +163,7 @@ namespace Tapeti.Cmd.Serialization
public Message ToMessage(IModel channel) public Message ToMessage(IModel channel)
{ {
return new Message return new()
{ {
DeliveryTag = DeliveryTag, DeliveryTag = DeliveryTag,
Redelivered = Redelivered, Redelivered = Redelivered,

1
Tapeti.Cmd/Tapeti.Cmd.csproj
View File

@ -12,6 +12,7 @@
<PackageProjectUrl>https://github.com/MvRens/Tapeti</PackageProjectUrl> <PackageProjectUrl>https://github.com/MvRens/Tapeti</PackageProjectUrl>
<Version>2.0.0</Version> <Version>2.0.0</Version>
<Product>Tapeti Command-line Utility</Product> <Product>Tapeti Command-line Utility</Product>
<LangVersion>latest</LangVersion>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>

22
Tapeti.Cmd/Verbs/BaseConnectionOptions.cs Normal file
View File

@ -0,0 +1,22 @@
using CommandLine;
namespace Tapeti.Cmd.Verbs
{
public class BaseConnectionOptions
{
[Option('h', "host", HelpText = "Hostname of the RabbitMQ server.", Default = "localhost")]
public string Host { get; set; }
[Option("port", HelpText = "AMQP port of the RabbitMQ server.", Default = 5672)]
public int Port { get; set; }
[Option('v', "virtualhost", HelpText = "Virtual host used for the RabbitMQ connection.", Default = "/")]
public string VirtualHost { get; set; }
[Option('u', "username", HelpText = "Username used to connect to the RabbitMQ server.", Default = "guest")]
public string Username { get; set; }
[Option('p', "password", HelpText = "Password used to connect to the RabbitMQ server.", Default = "guest")]
public string Password { get; set; }
}
}

17
Tapeti.Cmd/Verbs/BaseMessageSerializerOptions.cs Normal file
View File

@ -0,0 +1,17 @@
using CommandLine;
namespace Tapeti.Cmd.Verbs
{
public enum SerializationMethod
{
SingleFileJSON,
EasyNetQHosepipe
}
public class BaseMessageSerializerOptions : BaseConnectionOptions
{
[Option('s', "serialization", HelpText = "The method used to serialize the message for import or export. Valid options: SingleFileJSON, EasyNetQHosepipe.", Default = SerializationMethod.SingleFileJSON)]
public SerializationMethod SerializationMethod { get; set; }
}
}

56
Tapeti.Cmd/Verbs/BindQueueVerb.cs Normal file
View File

@ -0,0 +1,56 @@
using System;
using System.Collections.Generic;
using CommandLine;
using RabbitMQ.Client;
using Tapeti.Cmd.ConsoleHelper;
using Tapeti.Cmd.Parser;
namespace Tapeti.Cmd.Verbs
{
[Verb("bindqueue", HelpText = "Add a binding to an existing queue.")]
[ExecutableVerb(typeof(BindQueueVerb))]
public class BindQueueOptions : BaseConnectionOptions
{
[Option('q', "queue", Required = true, HelpText = "The name of the queue to add the binding(s) to.")]
public string QueueName { get; set; }
[Option('b', "bindings", Required = false, HelpText = "One or more bindings to add to the queue. Format: <exchange>:<routingKey>")]
public IEnumerable<string> Bindings { get; set; }
}
public class BindQueueVerb : IVerbExecuter
{
private readonly BindQueueOptions options;
public BindQueueVerb(BindQueueOptions options)
{
this.options = options;
}
public void Execute(IConsole console)
{
var consoleWriter = console.GetPermanentWriter();
var bindings = BindingParser.Parse(options.Bindings);
var factory = new ConnectionFactory
{
HostName = options.Host,
Port = options.Port,
VirtualHost = options.VirtualHost,
UserName = options.Username,
Password = options.Password
};
using var connection = factory.CreateConnection();
using var channel = connection.CreateModel();
foreach (var (exchange, routingKey) in bindings)
channel.QueueBind(options.QueueName, exchange, routingKey);
consoleWriter.WriteLine($"{bindings.Length} binding{(bindings.Length != 1 ? "s" : "")} added to queue {options.QueueName}.");
}
}
}

60
Tapeti.Cmd/Verbs/DeclareQueueVerb.cs Normal file
View File

@ -0,0 +1,60 @@
using System;
using System.Collections.Generic;
using CommandLine;
using RabbitMQ.Client;
using Tapeti.Cmd.ConsoleHelper;
using Tapeti.Cmd.Parser;
namespace Tapeti.Cmd.Verbs
{
[Verb("declarequeue", HelpText = "Declares a durable queue without arguments.")]
[ExecutableVerb(typeof(DeclareQueueVerb))]
public class DeclareQueueOptions : BaseConnectionOptions
{
[Option('q', "queue", Required = true, HelpText = "The name of the queue to declare.")]
public string QueueName { get; set; }
[Option('b', "bindings", Required = false, HelpText = "One or more bindings to add to the queue. Format: <exchange>:<routingKey>")]
public IEnumerable<string> Bindings { get; set; }
}
public class DeclareQueueVerb : IVerbExecuter
{
private readonly DeclareQueueOptions options;
public DeclareQueueVerb(DeclareQueueOptions options)
{
this.options = options;
}
public void Execute(IConsole console)
{
var consoleWriter = console.GetPermanentWriter();
// Parse early to fail early
var bindings = BindingParser.Parse(options.Bindings);
var factory = new ConnectionFactory
{
HostName = options.Host,
Port = options.Port,
VirtualHost = options.VirtualHost,
UserName = options.Username,
Password = options.Password
};
using var connection = factory.CreateConnection();
using var channel = connection.CreateModel();
channel.QueueDeclare(options.QueueName, true, false, false);
foreach (var (exchange, routingKey) in bindings)
channel.QueueBind(options.QueueName, exchange, routingKey);
consoleWriter.WriteLine($"Queue {options.QueueName} declared with {bindings.Length} binding{(bindings.Length != 1 ? "s" : "")}.");
}
}
}

54
Tapeti.Cmd/Verbs/ExampleVerb.cs Normal file
View File

@ -0,0 +1,54 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Text;
using CommandLine;
using RabbitMQ.Client;
using Tapeti.Cmd.ConsoleHelper;
using Tapeti.Cmd.Mock;
using Tapeti.Cmd.Serialization;
namespace Tapeti.Cmd.Verbs
{
[Verb("example", HelpText = "Output an example SingleFileJSON formatted message.")]
[ExecutableVerb(typeof(ExampleVerb))]
public class ExampleOptions
{
}
public class ExampleVerb : IVerbExecuter
{
public ExampleVerb(ExampleOptions options)
{
// Prevent compiler warnings, the parameter is expected by the Activator
Debug.Assert(options != null);
}
public void Execute(IConsole console)
{
using var messageSerializer = new SingleFileJSONMessageSerializer(Console.OpenStandardOutput(), false, new UTF8Encoding(false));
messageSerializer.Serialize(new Message
{
Exchange = "example",
Queue = "example.queue",
RoutingKey = "example.routing.key",
DeliveryTag = 42,
Properties = new MockBasicProperties
{
ContentType = "application/json",
DeliveryMode = 2,
Headers = new Dictionary<string, object>
{
{ "classType", Encoding.UTF8.GetBytes("Tapeti.Cmd.Example:Tapeti.Cmd") }
},
ReplyTo = "reply.queue",
Timestamp = new AmqpTimestamp(new DateTimeOffset(DateTime.UtcNow).ToUnixTimeSeconds())
},
Body = Encoding.UTF8.GetBytes("{ \"Hello\": \"world!\" }")
});
}
}
}

31
Tapeti.Cmd/Verbs/ExecutableVerbAttribute.cs Normal file
View File

@ -0,0 +1,31 @@
using System;
using Tapeti.Cmd.ConsoleHelper;
namespace Tapeti.Cmd.Verbs
{
/// <remarks>
/// Implementations are expected to have a constructor which accepts the options class
/// associated with the ExecutableVerb attribute.
/// </remarks>
public interface IVerbExecuter
{
void Execute(IConsole console);
}
[AttributeUsage(AttributeTargets.Class)]
public class ExecutableVerbAttribute : Attribute
{
public Type VerbExecuter { get; }
public ExecutableVerbAttribute(Type verbExecuter)
{
if (!typeof(IVerbExecuter).IsAssignableFrom(verbExecuter))
throw new InvalidCastException("Type must support IVerbExecuter");
VerbExecuter = verbExecuter;
}
}
}

115
Tapeti.Cmd/Verbs/ExportVerb.cs Normal file
View File

@ -0,0 +1,115 @@
using System;
using System.IO;
using System.Text;
using CommandLine;
using RabbitMQ.Client;
using Tapeti.Cmd.ConsoleHelper;
using Tapeti.Cmd.Serialization;
namespace Tapeti.Cmd.Verbs
{
[Verb("export", HelpText = "Fetch messages from a queue and write it to disk.")]
[ExecutableVerb(typeof(ExportVerb))]
public class ExportOptions : BaseMessageSerializerOptions
{
[Option('q', "queue", Required = true, HelpText = "The queue to read the messages from.")]
public string QueueName { get; set; }
[Option('o', "output", Required = true, HelpText = "Path or filename (depending on the chosen serialization method) where the messages will be output to.")]
public string OutputPath { get; set; }
[Option('r', "remove", HelpText = "If specified messages are acknowledged and removed from the queue. If not messages are kept.")]
public bool RemoveMessages { get; set; }
[Option('n', "maxcount", HelpText = "(Default: all) Maximum number of messages to retrieve from the queue.")]
public int? MaxCount { get; set; }
}
public class ExportVerb : IVerbExecuter
{
private readonly ExportOptions options;
public ExportVerb(ExportOptions options)
{
this.options = options;
}
public void Execute(IConsole console)
{
var consoleWriter = console.GetPermanentWriter();
var factory = new ConnectionFactory
{
HostName = options.Host,
Port = options.Port,
VirtualHost = options.VirtualHost,
UserName = options.Username,
Password = options.Password
};
using var messageSerializer = GetMessageSerializer(options);
using var connection = factory.CreateConnection();
using var channel = connection.CreateModel();
var totalCount = (int)channel.MessageCount(options.QueueName);
if (options.MaxCount.HasValue && options.MaxCount.Value < totalCount)
totalCount = options.MaxCount.Value;
consoleWriter.WriteLine($"Exporting {totalCount} message{(totalCount != 1 ? "s" : "")} (actual number may differ if queue has active consumers or publishers)");
var messageCount = 0;
using (var progressBar = new ProgressBar(console, totalCount))
{
while (!console.Cancelled && (!options.MaxCount.HasValue || messageCount < options.MaxCount.Value))
{
var result = channel.BasicGet(options.QueueName, false);
if (result == null)
// No more messages on the queue
break;
messageCount++;
messageSerializer.Serialize(new Message
{
DeliveryTag = result.DeliveryTag,
Redelivered = result.Redelivered,
Exchange = result.Exchange,
RoutingKey = result.RoutingKey,
Queue = options.QueueName,
Properties = result.BasicProperties,
Body = result.Body.ToArray()
});
if (options.RemoveMessages)
channel.BasicAck(result.DeliveryTag, false);
progressBar.Report(messageCount);
}
}
consoleWriter.WriteLine($"{messageCount} message{(messageCount != 1 ? "s" : "")} exported.");
}
private static IMessageSerializer GetMessageSerializer(ExportOptions options)
{
switch (options.SerializationMethod)
{
case SerializationMethod.SingleFileJSON:
return new SingleFileJSONMessageSerializer(new FileStream(options.OutputPath, FileMode.Create, FileAccess.Write, FileShare.Read), true, Encoding.UTF8);
case SerializationMethod.EasyNetQHosepipe:
if (string.IsNullOrEmpty(options.OutputPath))
throw new ArgumentException("An output path must be provided when using EasyNetQHosepipe serialization");
return new EasyNetQMessageSerializer(options.OutputPath);
default:
throw new ArgumentOutOfRangeException(nameof(options.SerializationMethod), options.SerializationMethod, "Invalid SerializationMethod");
}
}
}
}

144
Tapeti.Cmd/Verbs/ImportVerb.cs Normal file
View File

@ -0,0 +1,144 @@
using System;
using System.IO;
using System.Text;
using CommandLine;
using RabbitMQ.Client;
using Tapeti.Cmd.ConsoleHelper;
using Tapeti.Cmd.RateLimiter;
using Tapeti.Cmd.Serialization;
namespace Tapeti.Cmd.Verbs
{
[Verb("import", HelpText = "Read messages from disk as previously exported and publish them to a queue.")]
[ExecutableVerb(typeof(ImportVerb))]
public class ImportOptions : BaseMessageSerializerOptions
{
[Option('i', "input", Group = "Input", HelpText = "Path or filename (depending on the chosen serialization method) where the messages will be read from.")]
public string InputFile { get; set; }
[Option('m', "message", Group = "Input", HelpText = "Single message to be sent, in the same format as used for SingleFileJSON. Serialization argument has no effect when using this input.")]
public string InputMessage { get; set; }
[Option('c', "pipe", Group = "Input", HelpText = "Messages are read from the standard input pipe, in the same format as used for SingleFileJSON. Serialization argument has no effect when using this input.")]
public bool InputPipe { get; set; }
[Option('e', "exchange", HelpText = "If specified publishes to the originating exchange using the original routing key. By default these are ignored and the message is published directly to the originating queue.")]
public bool PublishToExchange { get; set; }
[Option("maxrate", HelpText = "The maximum amount of messages per second to import.")]
public int? MaxRate { get; set; }
[Option("batchsize", HelpText = "How many messages to import before pausing. Will wait for manual confirmation unless batchpausetime is specified.")]
public int? BatchSize { get; set; }
[Option("batchpausetime", HelpText = "How many seconds to wait before starting the next batch if batchsize is specified.")]
public int? BatchPauseTime { get; set; }
}
public class ImportVerb : IVerbExecuter
{
private readonly ImportOptions options;
public ImportVerb(ImportOptions options)
{
this.options = options;
}
public void Execute(IConsole console)
{
var consoleWriter = console.GetPermanentWriter();
var factory = new ConnectionFactory
{
HostName = options.Host,
Port = options.Port,
VirtualHost = options.VirtualHost,
UserName = options.Username,
Password = options.Password
};
using var messageSerializer = GetMessageSerializer(options);
using var connection = factory.CreateConnection();
using var channel = connection.CreateModel();
var rateLimiter = RateLimiterFactory.Create(console, options.MaxRate, options.BatchSize, options.BatchPauseTime);
var totalCount = messageSerializer.GetMessageCount();
var messageCount = 0;
ProgressBar progress = null;
if (totalCount > 0)
progress = new ProgressBar(console, totalCount);
try
{
foreach (var message in messageSerializer.Deserialize(channel))
{
if (console.Cancelled)
break;
rateLimiter.Execute(() =>
{
if (console.Cancelled)
return;
var exchange = options.PublishToExchange ? message.Exchange : "";
var routingKey = options.PublishToExchange ? message.RoutingKey : message.Queue;
// ReSharper disable AccessToDisposedClosure
channel.BasicPublish(exchange, routingKey, message.Properties, message.Body);
messageCount++;
progress?.Report(messageCount);
// ReSharper restore AccessToDisposedClosure
});
}
}
finally
{
progress?.Dispose();
}
consoleWriter.WriteLine($"{messageCount} message{(messageCount != 1 ? "s" : "")} published.");
}
private static IMessageSerializer GetMessageSerializer(ImportOptions options)
{
switch (options.SerializationMethod)
{
case SerializationMethod.SingleFileJSON:
return new SingleFileJSONMessageSerializer(GetInputStream(options, out var disposeStream), disposeStream, Encoding.UTF8);
case SerializationMethod.EasyNetQHosepipe:
if (string.IsNullOrEmpty(options.InputFile))
throw new ArgumentException("An input path must be provided when using EasyNetQHosepipe serialization");
return new EasyNetQMessageSerializer(options.InputFile);
default:
throw new ArgumentOutOfRangeException(nameof(options.SerializationMethod), options.SerializationMethod, "Invalid SerializationMethod");
}
}
private static Stream GetInputStream(ImportOptions options, out bool disposeStream)
{
if (options.InputPipe)
{
disposeStream = false;
return Console.OpenStandardInput();
}
if (!string.IsNullOrEmpty(options.InputMessage))
{
disposeStream = true;
return new MemoryStream(Encoding.UTF8.GetBytes(options.InputMessage));
}
disposeStream = true;
return new FileStream(options.InputFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
}
}
}

58
Tapeti.Cmd/Verbs/PurgeVerb.cs Normal file
View File

@ -0,0 +1,58 @@
using System;
using CommandLine;
using RabbitMQ.Client;
using Tapeti.Cmd.ConsoleHelper;
namespace Tapeti.Cmd.Verbs
{
[Verb("purge", HelpText = "Removes all messages from a queue destructively.")]
[ExecutableVerb(typeof(PurgeVerb))]
public class PurgeOptions : BaseConnectionOptions
{
[Option('q', "queue", Required = true, HelpText = "The queue to purge.")]
public string QueueName { get; set; }
[Option("confirm", HelpText = "Confirms the purging of the specified queue. If not provided, an interactive prompt will ask for confirmation.", Default = false)]
public bool Confirm { get; set; }
}
public class PurgeVerb : IVerbExecuter
{
private readonly PurgeOptions options;
public PurgeVerb(PurgeOptions options)
{
this.options = options;
}
public void Execute(IConsole console)
{
var consoleWriter = console.GetPermanentWriter();
if (!options.Confirm)
{
if (!consoleWriter.ConfirmYesNo($"Do you want to purge the queue '{options.QueueName}'?"))
return;
}
var factory = new ConnectionFactory
{
HostName = options.Host,
Port = options.Port,
VirtualHost = options.VirtualHost,
UserName = options.Username,
Password = options.Password
};
using var connection = factory.CreateConnection();
using var channel = connection.CreateModel();
var messageCount = channel.QueuePurge(options.QueueName);
consoleWriter.WriteLine($"{messageCount} message{(messageCount != 1 ? "s" : "")} purged from '{options.QueueName}'.");
}
}
}

87
Tapeti.Cmd/Verbs/RemoveQueueVerb.cs Normal file
View File

@ -0,0 +1,87 @@
using System;
using CommandLine;
using RabbitMQ.Client;
using RabbitMQ.Client.Exceptions;
using Tapeti.Cmd.ConsoleHelper;
namespace Tapeti.Cmd.Verbs
{
[Verb("removequeue", HelpText = "Removes a durable queue.")]
[ExecutableVerb(typeof(RemoveQueueVerb))]
public class RemoveQueueOptions : BaseConnectionOptions
{
[Option('q', "queue", Required = true, HelpText = "The name of the queue to remove.")]
public string QueueName { get; set; }
[Option("confirm", HelpText = "Confirms the removal of the specified queue. If not provided, an interactive prompt will ask for confirmation.", Default = false)]
public bool Confirm { get; set; }
[Option("confirmpurge", HelpText = "Confirms the removal of the specified queue even if there still are messages in the queue. If not provided, an interactive prompt will ask for confirmation.", Default = false)]
public bool ConfirmPurge { get; set; }
}
public class RemoveQueueVerb : IVerbExecuter
{
private readonly RemoveQueueOptions options;
public RemoveQueueVerb(RemoveQueueOptions options)
{
this.options = options;
}
public void Execute(IConsole console)
{
var consoleWriter = console.GetPermanentWriter();
if (!options.Confirm)
{
if (!consoleWriter.ConfirmYesNo($"Do you want to remove the queue '{options.QueueName}'?"))
return;
}
var factory = new ConnectionFactory
{
HostName = options.Host,
Port = options.Port,
VirtualHost = options.VirtualHost,
UserName = options.Username,
Password = options.Password
};
uint messageCount;
try
{
using var connection = factory.CreateConnection();
using var channel = connection.CreateModel();
messageCount = channel.QueueDelete(options.QueueName, true, true);
}
catch (OperationInterruptedException e)
{
if (e.ShutdownReason.ReplyCode == 406)
{
if (!options.ConfirmPurge)
{
if (!consoleWriter.ConfirmYesNo($"There are messages remaining. Do you want to purge the queue '{options.QueueName}'?"))
return;
}
using var connection = factory.CreateConnection();
using var channel = connection.CreateModel();
messageCount = channel.QueueDelete(options.QueueName, true, false);
}
else
throw;
}
consoleWriter.WriteLine(messageCount == 0
? $"Empty or non-existent queue '{options.QueueName}' removed."
: $"{messageCount} message{(messageCount != 1 ? "s" : "")} purged while removing '{options.QueueName}'.");
}
}
}

179
Tapeti.Cmd/Verbs/ShovelVerb.cs Normal file
View File

@ -0,0 +1,179 @@
using System;
using CommandLine;
using RabbitMQ.Client;
using Tapeti.Cmd.ConsoleHelper;
using Tapeti.Cmd.RateLimiter;
namespace Tapeti.Cmd.Verbs
{
[Verb("shovel", HelpText = "Reads messages from a queue and publishes them to another queue, optionally to another RabbitMQ server.")]
[ExecutableVerb(typeof(ShovelVerb))]
public class ShovelOptions : BaseConnectionOptions
{
[Option('q', "queue", Required = true, HelpText = "The queue to read the messages from.")]
public string QueueName { get; set; }
[Option('t', "targetqueue", HelpText = "The target queue to publish the messages to. Defaults to the source queue if a different target host, port or virtualhost is specified. Otherwise it must be different from the source queue.")]
public string TargetQueueName { get; set; }
[Option('r', "remove", HelpText = "If specified messages are acknowledged and removed from the source queue. If not messages are kept.")]
public bool RemoveMessages { get; set; }
[Option('n', "maxcount", HelpText = "(Default: all) Maximum number of messages to retrieve from the queue.")]
public int? MaxCount { get; set; }
[Option("targethost", HelpText = "Hostname of the target RabbitMQ server. Defaults to the source host. Note that you may still specify a different targetusername for example.")]
public string TargetHost { get; set; }
[Option("targetport", HelpText = "AMQP port of the target RabbitMQ server. Defaults to the source port.")]
public int? TargetPort { get; set; }
[Option("targetvirtualhost", HelpText = "Virtual host used for the target RabbitMQ connection. Defaults to the source virtualhost.")]
public string TargetVirtualHost { get; set; }
[Option("targetusername", HelpText = "Username used to connect to the target RabbitMQ server. Defaults to the source username.")]
public string TargetUsername { get; set; }
[Option("targetpassword", HelpText = "Password used to connect to the target RabbitMQ server. Defaults to the source password.")]
public string TargetPassword { get; set; }
[Option("maxrate", HelpText = "The maximum amount of messages per second to shovel.")]
public int? MaxRate { get; set; }
[Option("batchsize", HelpText = "How many messages to shovel before pausing. Will wait for manual confirmation unless batchpausetime is specified.")]
public int? BatchSize { get; set; }
[Option("batchpausetime", HelpText = "How many seconds to wait before starting the next batch if batchsize is specified.")]
public int? BatchPauseTime { get; set; }
}
public class ShovelVerb : IVerbExecuter
{
private readonly ShovelOptions options;
public ShovelVerb(ShovelOptions options)
{
this.options = options;
}
public void Execute(IConsole console)
{
var sourceFactory = new ConnectionFactory
{
HostName = options.Host,
Port = options.Port,
VirtualHost = options.VirtualHost,
UserName = options.Username,
Password = options.Password
};
using var sourceConnection = sourceFactory.CreateConnection();
using var sourceChannel = sourceConnection.CreateModel();
if (RequiresSecondConnection(options))
{
var targetFactory = new ConnectionFactory
{
HostName = !string.IsNullOrEmpty(options.TargetHost) ? options.TargetHost : options.Host,
Port = options.TargetPort ?? options.Port,
VirtualHost = !string.IsNullOrEmpty(options.TargetVirtualHost) ? options.TargetVirtualHost : options.VirtualHost,
UserName = !string.IsNullOrEmpty(options.TargetUsername) ? options.TargetUsername : options.Username,
Password = !string.IsNullOrEmpty(options.TargetPassword) ? options.TargetPassword : options.Password
};
using var targetConnection = targetFactory.CreateConnection();
using var targetChannel = targetConnection.CreateModel();
Shovel(console, options, sourceChannel, targetChannel);
}
else
Shovel(console, options, sourceChannel, sourceChannel);
}
private static void Shovel(IConsole console, ShovelOptions options, IModel sourceChannel, IModel targetChannel)
{
var consoleWriter = console.GetPermanentWriter();
var rateLimiter = RateLimiterFactory.Create(console, options.MaxRate, options.BatchSize, options.BatchPauseTime);
var targetQueueName = !string.IsNullOrEmpty(options.TargetQueueName) ? options.TargetQueueName : options.QueueName;
var totalCount = (int)sourceChannel.MessageCount(options.QueueName);
if (options.MaxCount.HasValue && options.MaxCount.Value < totalCount)
totalCount = options.MaxCount.Value;
consoleWriter.WriteLine($"Shoveling {totalCount} message{(totalCount != 1 ? "s" : "")} (actual number may differ if queue has active consumers or publishers)");
var messageCount = 0;
using (var progressBar = new ProgressBar(console, totalCount))
{
var hasMessage = true;
while (!console.Cancelled && hasMessage && (!options.MaxCount.HasValue || messageCount < options.MaxCount.Value))
{
rateLimiter.Execute(() =>
{
if (console.Cancelled)
return;
var result = sourceChannel.BasicGet(options.QueueName, false);
if (result == null)
{
// No more messages on the queue
hasMessage = false;
return;
}
// Since RabbitMQ client 6 we need to copy the body before calling another channel method
// like BasicPublish, or the published body will be corrupted if sourceChannel and targetChannel are the same
var bodyCopy = result.Body.ToArray();
targetChannel.BasicPublish("", targetQueueName, result.BasicProperties, bodyCopy);
messageCount++;
if (options.RemoveMessages)
sourceChannel.BasicAck(result.DeliveryTag, false);
// ReSharper disable once AccessToDisposedClosure
progressBar.Report(messageCount);
});
}
}
consoleWriter.WriteLine($"{messageCount} message{(messageCount != 1 ? "s" : "")} shoveled.");
}
private static bool RequiresSecondConnection(ShovelOptions options)
{
if (!string.IsNullOrEmpty(options.TargetHost) && options.TargetHost != options.Host)
return true;
if (options.TargetPort.HasValue && options.TargetPort.Value != options.Port)
return true;
if (!string.IsNullOrEmpty(options.TargetVirtualHost) && options.TargetVirtualHost != options.VirtualHost)
return true;
// All relevant target host parameters are either omitted or the same. This means the queue must be different
// to prevent an infinite loop.
if (string.IsNullOrEmpty(options.TargetQueueName) || options.TargetQueueName == options.QueueName)
throw new ArgumentException("Target queue must be different from the source queue when shoveling within the same (virtual) host");
if (!string.IsNullOrEmpty(options.TargetUsername) && options.TargetUsername != options.Username)
return true;
// ReSharper disable once ConvertIfStatementToReturnStatement
if (!string.IsNullOrEmpty(options.TargetPassword) && options.TargetPassword != options.Password)
return true;
// Everything's the same, we can use the same channel
return false;
}
}
}

56
Tapeti.Cmd/Verbs/UnbindQueueVerb.cs Normal file
View File

@ -0,0 +1,56 @@
using System;
using System.Collections.Generic;
using CommandLine;
using RabbitMQ.Client;
using Tapeti.Cmd.ConsoleHelper;
using Tapeti.Cmd.Parser;
namespace Tapeti.Cmd.Verbs
{
[Verb("unbindqueue", HelpText = "Remove a binding from a queue.")]
[ExecutableVerb(typeof(UnbindQueueVerb))]
public class UnbindQueueOptions : BaseConnectionOptions
{
[Option('q', "queue", Required = true, HelpText = "The name of the queue to remove the binding(s) from.")]
public string QueueName { get; set; }
[Option('b', "bindings", Required = false, HelpText = "One or more bindings to remove from the queue. Format: <exchange>:<routingKey>")]
public IEnumerable<string> Bindings { get; set; }
}
public class UnbindQueueVerb : IVerbExecuter
{
private readonly UnbindQueueOptions options;
public UnbindQueueVerb(UnbindQueueOptions options)
{
this.options = options;
}
public void Execute(IConsole console)
{
var consoleWriter = console.GetPermanentWriter();
var bindings = BindingParser.Parse(options.Bindings);
var factory = new ConnectionFactory
{
HostName = options.Host,
Port = options.Port,
VirtualHost = options.VirtualHost,
UserName = options.Username,
Password = options.Password
};
using var connection = factory.CreateConnection();
using var channel = connection.CreateModel();
foreach (var (exchange, routingKey) in bindings)
channel.QueueUnbind(options.QueueName, exchange, routingKey);
consoleWriter.WriteLine($"{bindings.Length} binding{(bindings.Length != 1 ? "s" : "")} removed from queue {options.QueueName}.");
}
}
}

20
Tapeti.Flow/ContextItems.cs
View File

@ -1,20 +0,0 @@
namespace Tapeti.Flow
{
/// <summary>
/// Key names as used in the message context store. For internal use.
/// </summary>
public static class ContextItems
{
/// <summary>
/// Key given to the FlowContext object as stored in the message context.
/// </summary>
public const string FlowContext = "Tapeti.Flow.FlowContext";
/// <summary>
/// Indicates if the current message handler is the last one to be called before a
/// parallel flow is done and the convergeMethod will be called.
/// Temporarily disables storing the flow state.
/// </summary>
public const string FlowIsConverging = "Tapeti.Flow.IsConverging";
}
}

6
Tapeti.Flow/Default/FlowBindingMiddleware.cs
View File

@ -74,16 +74,16 @@ namespace Tapeti.Flow.Default
} }
private static Task HandleYieldPoint(IControllerMessageContext context, IYieldPoint yieldPoint) private static Task HandleYieldPoint(IMessageContext context, IYieldPoint yieldPoint)
{ {
var flowHandler = context.Config.DependencyResolver.Resolve<IFlowHandler>(); var flowHandler = context.Config.DependencyResolver.Resolve<IFlowHandler>();
return flowHandler.Execute(new FlowHandlerContext(context), yieldPoint); return flowHandler.Execute(new FlowHandlerContext(context), yieldPoint);
} }
private static Task HandleParallelResponse(IControllerMessageContext context) private static Task HandleParallelResponse(IMessageContext context)
{ {
if (context.Get<object>(ContextItems.FlowIsConverging, out _)) if (context.TryGet<FlowMessageContextPayload>(out var flowPayload) && flowPayload.FlowIsConverging)
return Task.CompletedTask; return Task.CompletedTask;
var flowHandler = context.Config.DependencyResolver.Resolve<IFlowHandler>(); var flowHandler = context.Config.DependencyResolver.Resolve<IFlowHandler>();

62
Tapeti.Flow/Default/FlowContinuationMiddleware.cs
View File

@ -12,24 +12,31 @@ namespace Tapeti.Flow.Default
/// </summary> /// </summary>
internal class FlowContinuationMiddleware : IControllerFilterMiddleware, IControllerMessageMiddleware, IControllerCleanupMiddleware internal class FlowContinuationMiddleware : IControllerFilterMiddleware, IControllerMessageMiddleware, IControllerCleanupMiddleware
{ {
public async Task Filter(IControllerMessageContext context, Func<Task> next) public async Task Filter(IMessageContext context, Func<Task> next)
{ {
if (!context.TryGet<ControllerMessageContextPayload>(out var controllerPayload))
return;
var flowContext = await EnrichWithFlowContext(context); var flowContext = await EnrichWithFlowContext(context);
if (flowContext?.ContinuationMetadata == null) if (flowContext?.ContinuationMetadata == null)
return; return;
if (flowContext.ContinuationMetadata.MethodName != MethodSerializer.Serialize(context.Binding.Method)) if (flowContext.ContinuationMetadata.MethodName != MethodSerializer.Serialize(controllerPayload.Binding.Method))
return; return;
await next(); await next();
} }
public async Task Handle(IControllerMessageContext context, Func<Task> next) public async Task Handle(IMessageContext context, Func<Task> next)
{ {
if (context.Get(ContextItems.FlowContext, out FlowContext flowContext)) if (!context.TryGet<ControllerMessageContextPayload>(out var controllerPayload))
{ return;
Newtonsoft.Json.JsonConvert.PopulateObject(flowContext.FlowState.Data, context.Controller);
if (context.TryGet<FlowMessageContextPayload>(out var flowPayload))
{
var flowContext = flowPayload.FlowContext;
Newtonsoft.Json.JsonConvert.PopulateObject(flowContext.FlowState.Data, controllerPayload.Controller);
// Remove Continuation now because the IYieldPoint result handler will store the new state // Remove Continuation now because the IYieldPoint result handler will store the new state
flowContext.FlowState.Continuations.Remove(flowContext.ContinuationID); flowContext.FlowState.Continuations.Remove(flowContext.ContinuationID);
@ -38,28 +45,33 @@ namespace Tapeti.Flow.Default
if (converge) if (converge)
// Indicate to the FlowBindingMiddleware that the state must not to be stored // Indicate to the FlowBindingMiddleware that the state must not to be stored
context.Store(ContextItems.FlowIsConverging, null); flowPayload.FlowIsConverging = true;
await next(); await next();
if (converge) if (converge)
await CallConvergeMethod(context, await CallConvergeMethod(context, controllerPayload,
flowContext.ContinuationMetadata.ConvergeMethodName, flowContext.ContinuationMetadata.ConvergeMethodName,
flowContext.ContinuationMetadata.ConvergeMethodSync); flowContext.ContinuationMetadata.ConvergeMethodSync);
} }
else else
await next(); await next();
} }
public async Task Cleanup(IControllerMessageContext context, ConsumeResult consumeResult, Func<Task> next) public async Task Cleanup(IMessageContext context, ConsumeResult consumeResult, Func<Task> next)
{ {
await next(); await next();
if (!context.Get(ContextItems.FlowContext, out FlowContext flowContext)) if (!context.TryGet<ControllerMessageContextPayload>(out var controllerPayload))
return; return;
if (flowContext.ContinuationMetadata.MethodName != MethodSerializer.Serialize(context.Binding.Method)) if (!context.TryGet<FlowMessageContextPayload>(out var flowPayload))
return;
var flowContext = flowPayload.FlowContext;
if (flowContext.ContinuationMetadata.MethodName != MethodSerializer.Serialize(controllerPayload.Binding.Method))
// Do not call when the controller method was filtered, if the same message has two methods // Do not call when the controller method was filtered, if the same message has two methods
return; return;
@ -76,10 +88,10 @@ namespace Tapeti.Flow.Default
private static async Task<FlowContext> EnrichWithFlowContext(IControllerMessageContext context) private static async Task<FlowContext> EnrichWithFlowContext(IMessageContext context)
{ {
if (context.Get(ContextItems.FlowContext, out FlowContext flowContext)) if (context.TryGet<FlowMessageContextPayload>(out var flowPayload))
return flowContext; return flowPayload.FlowContext;
if (context.Properties.CorrelationId == null) if (context.Properties.CorrelationId == null)
@ -100,7 +112,7 @@ namespace Tapeti.Flow.Default
if (flowState == null) if (flowState == null)
return null; return null;
flowContext = new FlowContext var flowContext = new FlowContext
{ {
HandlerContext = new FlowHandlerContext(context), HandlerContext = new FlowHandlerContext(context),
@ -112,26 +124,28 @@ namespace Tapeti.Flow.Default
}; };
// IDisposable items in the IMessageContext are automatically disposed // IDisposable items in the IMessageContext are automatically disposed
context.Store(ContextItems.FlowContext, flowContext); context.Store(new FlowMessageContextPayload(flowContext));
return flowContext; return flowContext;
} }
private static async Task CallConvergeMethod(IControllerMessageContext context, string methodName, bool sync) private static async Task CallConvergeMethod(IMessageContext context, ControllerMessageContextPayload controllerPayload, string methodName, bool sync)
{ {
IYieldPoint yieldPoint; IYieldPoint yieldPoint;
var method = context.Controller.GetType().GetMethod(methodName, BindingFlags.NonPublic | BindingFlags.Instance); var method = controllerPayload.Controller.GetType().GetMethod(methodName, BindingFlags.NonPublic | BindingFlags.Instance);
if (method == null) if (method == null)
throw new ArgumentException($"Unknown converge method in controller {context.Controller.GetType().Name}: {methodName}"); throw new ArgumentException($"Unknown converge method in controller {controllerPayload.Controller.GetType().Name}: {methodName}");
if (sync) if (sync)
yieldPoint = (IYieldPoint)method.Invoke(context.Controller, new object[] {}); yieldPoint = (IYieldPoint)method.Invoke(controllerPayload.Controller, new object[] {});
else else
yieldPoint = await (Task<IYieldPoint>)method.Invoke(context.Controller, new object[] { }); yieldPoint = await (Task<IYieldPoint>)method.Invoke(controllerPayload.Controller, new object[] { });
if (yieldPoint == null) if (yieldPoint == null)
throw new YieldPointException($"Yield point is required in controller {context.Controller.GetType().Name} for converge method {methodName}"); throw new YieldPointException($"Yield point is required in controller {controllerPayload.Controller.GetType().Name} for converge method {methodName}");
var flowHandler = context.Config.DependencyResolver.Resolve<IFlowHandler>(); var flowHandler = context.Config.DependencyResolver.Resolve<IFlowHandler>();
await flowHandler.Execute(new FlowHandlerContext(context), yieldPoint); await flowHandler.Execute(new FlowHandlerContext(context), yieldPoint);

13
Tapeti.Flow/Default/FlowHandlerContext.cs
View File

@ -18,15 +18,18 @@ namespace Tapeti.Flow.Default
/// <summary> /// <summary>
/// </summary> /// </summary>
public FlowHandlerContext(IControllerMessageContext source) public FlowHandlerContext(IMessageContext source)
{ {
if (source == null) if (source == null)
return; return;
if (!source.TryGet<ControllerMessageContextPayload>(out var controllerPayload))
return;
Config = source.Config; Config = source.Config;
Controller = source.Controller; Controller = controllerPayload.Controller;
Method = source.Binding.Method; Method = controllerPayload.Binding.Method;
ControllerMessageContext = source; MessageContext = source;
} }
@ -45,6 +48,6 @@ namespace Tapeti.Flow.Default
public MethodInfo Method { get; set; } public MethodInfo Method { get; set; }
/// <inheritdoc /> /// <inheritdoc />
public IControllerMessageContext ControllerMessageContext { get; set; } public IMessageContext MessageContext { get; set; }
} }
} }

16
Tapeti.Flow/Default/FlowProvider.cs
View File

@ -162,16 +162,16 @@ namespace Tapeti.Flow.Default
private static ReplyMetadata GetReply(IFlowHandlerContext context) private static ReplyMetadata GetReply(IFlowHandlerContext context)
{ {
var requestAttribute = context.ControllerMessageContext?.Message?.GetType().GetCustomAttribute<RequestAttribute>(); var requestAttribute = context.MessageContext?.Message?.GetType().GetCustomAttribute<RequestAttribute>();
if (requestAttribute?.Response == null) if (requestAttribute?.Response == null)
return null; return null;
return new ReplyMetadata return new ReplyMetadata
{ {
CorrelationId = context.ControllerMessageContext.Properties.CorrelationId, CorrelationId = context.MessageContext.Properties.CorrelationId,
ReplyTo = context.ControllerMessageContext.Properties.ReplyTo, ReplyTo = context.MessageContext.Properties.ReplyTo,
ResponseTypeName = requestAttribute.Response.FullName, ResponseTypeName = requestAttribute.Response.FullName,
Mandatory = context.ControllerMessageContext.Properties.Persistent.GetValueOrDefault(true) Mandatory = context.MessageContext.Properties.Persistent.GetValueOrDefault(true)
}; };
} }
@ -206,8 +206,8 @@ namespace Tapeti.Flow.Default
try try
{ {
var messageContext = context.ControllerMessageContext; var messageContext = context.MessageContext;
if (messageContext == null || !messageContext.Get(ContextItems.FlowContext, out flowContext)) if (messageContext == null || !messageContext.TryGet<FlowMessageContextPayload>(out var flowPayload))
{ {
flowContext = new FlowContext flowContext = new FlowContext
{ {
@ -218,7 +218,9 @@ namespace Tapeti.Flow.Default
// in the messageContext as the yield point is the last to execute. // in the messageContext as the yield point is the last to execute.
disposeFlowContext = true; disposeFlowContext = true;
} }
else
flowContext = flowPayload.FlowContext;
try try
{ {
await executableYieldPoint.Execute(flowContext); await executableYieldPoint.Execute(flowContext);

33
Tapeti.Flow/FlowMessageContextPayload.cs Normal file
View File

@ -0,0 +1,33 @@
using System;
using Tapeti.Config;
using Tapeti.Flow.Default;
namespace Tapeti.Flow
{
/// <summary>
/// Contains information about the flow for the current message. For internal use.
/// </summary>
internal class FlowMessageContextPayload : IMessageContextPayload, IDisposable
{
public FlowContext FlowContext { get; }
/// <summary>
/// Indicates if the current message handler is the last one to be called before a
/// parallel flow is done and the convergeMethod will be called.
/// Temporarily disables storing the flow state.
/// </summary>
public bool FlowIsConverging { get; set; }
public FlowMessageContextPayload(FlowContext flowContext)
{
FlowContext = flowContext;
}
public void Dispose()
{
FlowContext?.Dispose();
}
}
}

4
Tapeti.Flow/IFlowHandlerContext.cs
View File

@ -29,9 +29,9 @@ namespace Tapeti.Flow
/// <summary> /// <summary>
/// Access to the controller message context if this is a continuated flow. /// Access to the message context if this is a continuated flow.
/// Will be null when in a starting flow. /// Will be null when in a starting flow.
/// </summary> /// </summary>
IControllerMessageContext ControllerMessageContext { get; } IMessageContext MessageContext { get; }
} }
} }

44
Tapeti.Serilog/TapetiSeriLogger.cs
View File

@ -1,5 +1,6 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Text;
using Tapeti.Config; using Tapeti.Config;
using ISerilogLogger = Serilog.ILogger; using ISerilogLogger = Serilog.ILogger;
@ -12,6 +13,21 @@ namespace Tapeti.Serilog
/// </summary> /// </summary>
public class TapetiSeriLogger: IBindingLogger public class TapetiSeriLogger: IBindingLogger
{ {
/// <summary>
/// Implements the Tapeti ILogger interface for Serilog output. This version
/// includes the message body and information if available when an error occurs.
/// </summary>
public class WithMessageLogging : TapetiSeriLogger
{
/// <inheritdoc />
public WithMessageLogging(ISerilogLogger seriLogger) : base(seriLogger) { }
internal override bool IncludeMessageInfo() => true;
}
private readonly ISerilogLogger seriLogger; private readonly ISerilogLogger seriLogger;
@ -69,20 +85,38 @@ namespace Tapeti.Serilog
/// <inheritdoc /> /// <inheritdoc />
public void ConsumeException(Exception exception, IMessageContext messageContext, ConsumeResult consumeResult) public void ConsumeException(Exception exception, IMessageContext messageContext, ConsumeResult consumeResult)
{ {
var message = new StringBuilder("Tapeti: exception in message handler");
var messageParams = new List<object>();
var contextLogger = seriLogger var contextLogger = seriLogger
.ForContext("consumeResult", consumeResult) .ForContext("consumeResult", consumeResult)
.ForContext("exchange", messageContext.Exchange) .ForContext("exchange", messageContext.Exchange)
.ForContext("queue", messageContext.Queue) .ForContext("queue", messageContext.Queue)
.ForContext("routingKey", messageContext.RoutingKey); .ForContext("routingKey", messageContext.RoutingKey);
if (messageContext is IControllerMessageContext controllerMessageContext) if (messageContext.TryGet<ControllerMessageContextPayload>(out var controllerPayload))
{ {
contextLogger = contextLogger contextLogger = contextLogger
.ForContext("controller", controllerMessageContext.Binding.Controller.FullName) .ForContext("controller", controllerPayload.Binding.Controller.FullName)
.ForContext("method", controllerMessageContext.Binding.Method.Name); .ForContext("method", controllerPayload.Binding.Method.Name);
message.Append(" {controller}.{method}");
messageParams.Add(controllerPayload.Binding.Controller.FullName);
messageParams.Add(controllerPayload.Binding.Method.Name);
}
if (IncludeMessageInfo())
{
message.Append(" on exchange {exchange}, queue {queue}, routingKey {routingKey}, replyTo {replyTo}, correlationId {correlationId} with body {body}");
messageParams.Add(messageContext.Exchange);
messageParams.Add(messageContext.Queue);
messageParams.Add(messageContext.RoutingKey);
messageParams.Add(messageContext.Properties.ReplyTo);
messageParams.Add(messageContext.Properties.CorrelationId);
messageParams.Add(messageContext.RawBody != null ? Encoding.UTF8.GetString(messageContext.RawBody) : null);
} }
contextLogger.Error(exception, "Tapeti: exception in message handler"); contextLogger.Error(exception, message.ToString(), messageParams.ToArray());
} }
/// <inheritdoc /> /// <inheritdoc />
@ -134,5 +168,7 @@ namespace Tapeti.Serilog
else else
seriLogger.Information("Tapeti: obsolete queue {queue} has been unbound but not yet deleted, {messageCount} messages remaining", queueName, messageCount); seriLogger.Information("Tapeti: obsolete queue {queue} has been unbound but not yet deleted, {messageCount} messages remaining", queueName, messageCount);
} }
internal virtual bool IncludeMessageInfo() => false;
} }
} }

32
Tapeti/Config/ControllerMessageContextPayload.cs Normal file
View File

@ -0,0 +1,32 @@
namespace Tapeti.Config
{
/// <inheritdoc />
/// <summary>
/// Extends the message context with information about the controller.
/// </summary>
public class ControllerMessageContextPayload : IMessageContextPayload
{
/// <summary>
/// An instance of the controller referenced by the binding. Note: can be null during Cleanup.
/// </summary>
public object Controller { get; }
/// <remarks>
/// Provides access to the binding which is currently processing the message.
/// </remarks>
public IControllerMethodBinding Binding { get; }
/// <summary>
/// Constructs the payload to enrich the message context with information about the controller.
/// </summary>
/// <param name="controller">An instance of the controller referenced by the binding</param>
/// <param name="binding">The binding which is currently processing the message</param>
public ControllerMessageContextPayload(object controller, IControllerMethodBinding binding)
{
Controller = controller;
Binding = binding;
}
}
}

4
Tapeti/Config/IControllerBindingContext.cs
View File

@ -11,7 +11,7 @@ namespace Tapeti.Config
/// Injects a value for a controller method parameter. /// Injects a value for a controller method parameter.
/// </summary> /// </summary>
/// <param name="context"></param> /// <param name="context"></param>
public delegate object ValueFactory(IControllerMessageContext context); public delegate object ValueFactory(IMessageContext context);
/// <summary> /// <summary>
@ -19,7 +19,7 @@ namespace Tapeti.Config
/// </summary> /// </summary>
/// <param name="context"></param> /// <param name="context"></param>
/// <param name="value"></param> /// <param name="value"></param>
public delegate Task ResultHandler(IControllerMessageContext context, object value); public delegate Task ResultHandler(IMessageContext context, object value);
/// <summary> /// <summary>

2
Tapeti/Config/IControllerCleanupMiddleware.cs
View File

@ -14,6 +14,6 @@ namespace Tapeti.Config
/// <param name="context"></param> /// <param name="context"></param>
/// <param name="consumeResult"></param> /// <param name="consumeResult"></param>
/// <param name="next">Always call to allow the next in the chain to clean up</param> /// <param name="next">Always call to allow the next in the chain to clean up</param>
Task Cleanup(IControllerMessageContext context, ConsumeResult consumeResult, Func<Task> next); Task Cleanup(IMessageContext context, ConsumeResult consumeResult, Func<Task> next);
} }
} }

2
Tapeti/Config/IControllerFilterMiddleware.cs
View File

@ -15,6 +15,6 @@ namespace Tapeti.Config
/// <param name="context"></param> /// <param name="context"></param>
/// <param name="next"></param> /// <param name="next"></param>
/// <returns></returns> /// <returns></returns>
Task Filter(IControllerMessageContext context, Func<Task> next); Task Filter(IMessageContext context, Func<Task> next);
} }
} }

20
Tapeti/Config/IControllerMessageContext.cs
View File

@ -1,20 +0,0 @@
namespace Tapeti.Config
{
/// <inheritdoc />
/// <summary>
/// Extends the message context with information about the controller.
/// </summary>
public interface IControllerMessageContext : IMessageContext
{
/// <summary>
/// An instance of the controller referenced by the binding. Note: is null during Cleanup.
/// </summary>
object Controller { get; }
/// <remarks>
/// Provides access to the binding which is currently processing the message.
/// </remarks>
new IControllerMethodBinding Binding { get; }
}
}

2
Tapeti/Config/IControllerMessageMiddleware.cs
View File

@ -14,6 +14,6 @@ namespace Tapeti.Config
/// </summary> /// </summary>
/// <param name="context"></param> /// <param name="context"></param>
/// <param name="next">Call to pass the message to the next handler in the chain or call the controller method</param> /// <param name="next">Call to pass the message to the next handler in the chain or call the controller method</param>
Task Handle(IControllerMessageContext context, Func<Task> next); Task Handle(IMessageContext context, Func<Task> next);
} }
} }

50
Tapeti/Config/IMessageContext.cs
View File

@ -1,5 +1,7 @@
using System; using System;
// ReSharper disable UnusedMemberInSuper.Global - public API
namespace Tapeti.Config namespace Tapeti.Config
{ {
/// <summary> /// <summary>
@ -27,6 +29,11 @@ namespace Tapeti.Config
/// </summary> /// </summary>
string RoutingKey { get; } string RoutingKey { get; }
/// <summary>
/// Contains the raw body of the message.
/// </summary>
byte[] RawBody { get; }
/// <summary> /// <summary>
/// Contains the decoded message instance. /// Contains the decoded message instance.
/// </summary> /// </summary>
@ -42,6 +49,36 @@ namespace Tapeti.Config
/// </remarks> /// </remarks>
IBinding Binding { get; } IBinding Binding { get; }
/// <summary>
/// Stores additional properties in the message context which can be passed between middleware stages.
/// </summary>
/// <remarks>
/// Only one instance of type T is stored, if Enrich was called before for this type an InvalidOperationException will be thrown.
/// </remarks>
/// <param name="payload">A class implementing IMessageContextPayload</param>
void Store<T>(T payload) where T : IMessageContextPayload;
/// <summary>
/// Stored a new payload, or updates an existing one.
/// </summary>
/// <param name="onAdd">A method returning the new payload to be stored</param>
/// <param name="onUpdate">A method called when the payload exists</param>
/// <typeparam name="T">The payload type as passed to Enrich</typeparam>
void StoreOrUpdate<T>(Func<T> onAdd, Action<T> onUpdate) where T : IMessageContextPayload;
/// <summary>
/// Returns the properties as previously stored with Enrich. Throws a KeyNotFoundException
/// if the payload is not stored in this message context.
/// </summary>
/// <typeparam name="T">The payload type as passed to Enrich</typeparam>
T Get<T>() where T : IMessageContextPayload;
/// <summary>
/// Returns true and the payload value if this message context was previously enriched with the payload T.
/// </summary>
/// <typeparam name="T">The payload type as passed to Enrich</typeparam>
bool TryGet<T>(out T payload) where T : IMessageContextPayload;
/// <summary> /// <summary>
/// Stores a key-value pair in the context for passing information between the various /// Stores a key-value pair in the context for passing information between the various
@ -49,6 +86,7 @@ namespace Tapeti.Config
/// </summary> /// </summary>
/// <param name="key">A unique key. It is recommended to prefix it with the package name which hosts the middleware to prevent conflicts</param> /// <param name="key">A unique key. It is recommended to prefix it with the package name which hosts the middleware to prevent conflicts</param>
/// <param name="value">Will be disposed if the value implements IDisposable or IAsyncDisposable</param> /// <param name="value">Will be disposed if the value implements IDisposable or IAsyncDisposable</param>
[Obsolete("For backwards compatibility only. Use Store<T> payload for typed properties instead")]
void Store(string key, object value); void Store(string key, object value);
/// <summary> /// <summary>
@ -57,6 +95,18 @@ namespace Tapeti.Config
/// <param name="key"></param> /// <param name="key"></param>
/// <param name="value"></param> /// <param name="value"></param>
/// <returns>True if the value was found, False otherwise</returns> /// <returns>True if the value was found, False otherwise</returns>
[Obsolete("For backwards compatibility only. Use Get<T> payload overload for typed properties instead")]
bool Get<T>(string key, out T value) where T : class; bool Get<T>(string key, out T value) where T : class;
} }
/// <summary>
/// Base interface for additional properties added to the message context.
/// </summary>
/// <remarks>
/// Descendants implementing IDisposable or IAsyncDisposable will be disposed along with the message context.
/// </remarks>
public interface IMessageContextPayload
{
}
} }

4
Tapeti/Connection/TapetiConsumer.cs
View File

@ -57,6 +57,7 @@ namespace Tapeti.Connection
return await DispatchMessage(message, new MessageContextData return await DispatchMessage(message, new MessageContextData
{ {
RawBody = body,
Exchange = exchange, Exchange = exchange,
RoutingKey = routingKey, RoutingKey = routingKey,
Properties = properties Properties = properties
@ -70,6 +71,7 @@ namespace Tapeti.Connection
Queue = queueName, Queue = queueName,
Exchange = exchange, Exchange = exchange,
RoutingKey = routingKey, RoutingKey = routingKey,
RawBody = body,
Message = message, Message = message,
Properties = properties, Properties = properties,
Binding = null Binding = null
@ -112,6 +114,7 @@ namespace Tapeti.Connection
Queue = queueName, Queue = queueName,
Exchange = messageContextData.Exchange, Exchange = messageContextData.Exchange,
RoutingKey = messageContextData.RoutingKey, RoutingKey = messageContextData.RoutingKey,
RawBody = messageContextData.RawBody,
Message = message, Message = message,
Properties = messageContextData.Properties, Properties = messageContextData.Properties,
Binding = binding Binding = binding
@ -174,6 +177,7 @@ namespace Tapeti.Connection
private struct MessageContextData private struct MessageContextData
{ {
public byte[] RawBody;
public string Exchange; public string Exchange;
public string RoutingKey; public string RoutingKey;
public IMessageProperties Properties; public IMessageProperties Properties;

35
Tapeti/Default/ConsoleLogger.cs
View File

@ -11,6 +11,19 @@ namespace Tapeti.Default
/// </summary> /// </summary>
public class ConsoleLogger : IBindingLogger public class ConsoleLogger : IBindingLogger
{ {
/// <summary>
/// Default ILogger implementation for console applications. This version
/// includes the message body if available when an error occurs.
/// </summary>
public class WithMessageLogging : ConsoleLogger
{
/// <inheritdoc />
public WithMessageLogging() : base() { }
internal override bool IncludeMessageBody() => true;
}
/// <inheritdoc /> /// <inheritdoc />
public void Connect(IConnectContext connectContext) public void Connect(IConnectContext connectContext)
{ {
@ -39,17 +52,23 @@ namespace Tapeti.Default
public void ConsumeException(Exception exception, IMessageContext messageContext, ConsumeResult consumeResult) public void ConsumeException(Exception exception, IMessageContext messageContext, ConsumeResult consumeResult)
{ {
Console.WriteLine("[Tapeti] Exception while handling message"); Console.WriteLine("[Tapeti] Exception while handling message");
Console.WriteLine($" Result : {consumeResult}"); Console.WriteLine($" Result : {consumeResult}");
Console.WriteLine($" Exchange : {messageContext.Exchange}"); Console.WriteLine($" Exchange : {messageContext.Exchange}");
Console.WriteLine($" Queue : {messageContext.Queue}"); Console.WriteLine($" Queue : {messageContext.Queue}");
Console.WriteLine($" RoutingKey : {messageContext.RoutingKey}"); Console.WriteLine($" RoutingKey : {messageContext.RoutingKey}");
Console.WriteLine($" ReplyTo : {messageContext.Properties.ReplyTo}");
Console.WriteLine($" CorrelationId : {messageContext.Properties.CorrelationId}");
if (messageContext is IControllerMessageContext controllerMessageContext) if (messageContext.TryGet<ControllerMessageContextPayload>(out var controllerPayload))
{ {
Console.WriteLine($" Controller : {controllerMessageContext.Binding.Controller.FullName}"); Console.WriteLine($" Controller : {controllerPayload.Binding.Controller.FullName}");
Console.WriteLine($" Method : {controllerMessageContext.Binding.Method.Name}"); Console.WriteLine($" Method : {controllerPayload.Binding.Method.Name}");
} }
if (IncludeMessageBody())
Console.WriteLine($" Body : {(messageContext.RawBody != null ? Encoding.UTF8.GetString(messageContext.RawBody) : "<null>")}");
Console.WriteLine(); Console.WriteLine();
Console.WriteLine(exception); Console.WriteLine(exception);
} }
@ -102,5 +121,7 @@ namespace Tapeti.Default
? $"[Tapeti] Obsolete queue was deleted: {queueName}" ? $"[Tapeti] Obsolete queue was deleted: {queueName}"
: $"[Tapeti] Obsolete queue bindings removed: {queueName}, {messageCount} messages remaining"); : $"[Tapeti] Obsolete queue bindings removed: {queueName}, {messageCount} messages remaining");
} }
internal virtual bool IncludeMessageBody() => false;
} }
} }

71
Tapeti/Default/ControllerMessageContext.cs
View File

@ -1,71 +0,0 @@
using System.Threading.Tasks;
using Tapeti.Config;
namespace Tapeti.Default
{
internal class ControllerMessageContext : IControllerMessageContext
{
private readonly IMessageContext decoratedContext;
/// <inheritdoc />
public object Controller { get; set; }
/// <inheritdoc />
public ITapetiConfig Config => decoratedContext.Config;
/// <inheritdoc />
public string Queue => decoratedContext.Queue;
/// <inheritdoc />
public string Exchange => decoratedContext.Exchange;
/// <inheritdoc />
public string RoutingKey => decoratedContext.RoutingKey;
/// <inheritdoc />
public object Message => decoratedContext.Message;
/// <inheritdoc />
public IMessageProperties Properties => decoratedContext.Properties;
IBinding IMessageContext.Binding => decoratedContext.Binding;
IControllerMethodBinding IControllerMessageContext.Binding => decoratedContext.Binding as IControllerMethodBinding;
public ControllerMessageContext(IMessageContext decoratedContext)
{
this.decoratedContext = decoratedContext;
}
/// <inheritdoc />
public void Dispose()
{
// Do not call decoratedContext.Dispose - by design
}
/// <inheritdoc />
public ValueTask DisposeAsync()
{
// Do not call decoratedContext.DisposeAsync - by design
return default;
}
/// <inheritdoc />
public void Store(string key, object value)
{
decoratedContext.Store(key, value);
}
/// <inheritdoc />
public bool Get<T>(string key, out T value) where T : class
{
return decoratedContext.Get(key, out value);
}
}
}

38
Tapeti/Default/ControllerMethodBinding.cs
View File

@ -160,39 +160,30 @@ namespace Tapeti.Default
public async Task Invoke(IMessageContext context) public async Task Invoke(IMessageContext context)
{ {
var controller = dependencyResolver.Resolve(bindingInfo.ControllerType); var controller = dependencyResolver.Resolve(bindingInfo.ControllerType);
context.Store(new ControllerMessageContextPayload(controller, context.Binding as IControllerMethodBinding));
await using var controllerContext = new ControllerMessageContext(context)
{
Controller = controller
};
if (!await FilterAllowed(controllerContext)) if (!await FilterAllowed(context))
return; return;
await MiddlewareHelper.GoAsync( await MiddlewareHelper.GoAsync(
bindingInfo.MessageMiddleware, bindingInfo.MessageMiddleware,
async (handler, next) => await handler.Handle(controllerContext, next), async (handler, next) => await handler.Handle(context, next),
async () => await messageHandler(controllerContext)); async () => await messageHandler(context));
} }
/// <inheritdoc /> /// <inheritdoc />
public async Task Cleanup(IMessageContext context, ConsumeResult consumeResult) public async Task Cleanup(IMessageContext context, ConsumeResult consumeResult)
{ {
await using var controllerContext = new ControllerMessageContext(context)
{
Controller = null
};
await MiddlewareHelper.GoAsync( await MiddlewareHelper.GoAsync(
bindingInfo.CleanupMiddleware, bindingInfo.CleanupMiddleware,
async (handler, next) => await handler.Cleanup(controllerContext, consumeResult, next), async (handler, next) => await handler.Cleanup(context, consumeResult, next),
() => Task.CompletedTask); () => Task.CompletedTask);
} }
private async Task<bool> FilterAllowed(IControllerMessageContext context) private async Task<bool> FilterAllowed(IMessageContext context)
{ {
var allowed = false; var allowed = false;
await MiddlewareHelper.GoAsync( await MiddlewareHelper.GoAsync(
@ -208,7 +199,7 @@ namespace Tapeti.Default
} }
private delegate Task MessageHandlerFunc(IControllerMessageContext context); private delegate Task MessageHandlerFunc(IMessageContext context);
private MessageHandlerFunc WrapMethod(MethodInfo method, IEnumerable<ValueFactory> parameterFactories, ResultHandler resultHandler) private MessageHandlerFunc WrapMethod(MethodInfo method, IEnumerable<ValueFactory> parameterFactories, ResultHandler resultHandler)
@ -233,9 +224,10 @@ namespace Tapeti.Default
{ {
return context => return context =>
{ {
var controllerPayload = context.Get<ControllerMessageContextPayload>();
try try
{ {
var result = method.Invoke(context.Controller, parameterFactories.Select(p => p(context)).ToArray()); var result = method.Invoke(controllerPayload.Controller, parameterFactories.Select(p => p(context)).ToArray());
return resultHandler(context, result); return resultHandler(context, result);
} }
catch (Exception e) catch (Exception e)
@ -250,9 +242,10 @@ namespace Tapeti.Default
{ {
return context => return context =>
{ {
var controllerPayload = context.Get<ControllerMessageContextPayload>();
try try
{ {
method.Invoke(context.Controller, parameterFactories.Select(p => p(context)).ToArray()); method.Invoke(controllerPayload.Controller, parameterFactories.Select(p => p(context)).ToArray());
return Task.CompletedTask; return Task.CompletedTask;
} }
catch (Exception e) catch (Exception e)
@ -268,9 +261,10 @@ namespace Tapeti.Default
{ {
return context => return context =>
{ {
var controllerPayload = context.Get<ControllerMessageContextPayload>();
try try
{ {
return (Task) method.Invoke(context.Controller, parameterFactories.Select(p => p(context)).ToArray()); return (Task) method.Invoke(controllerPayload.Controller, parameterFactories.Select(p => p(context)).ToArray());
} }
catch (Exception e) catch (Exception e)
{ {
@ -285,9 +279,10 @@ namespace Tapeti.Default
{ {
return context => return context =>
{ {
var controllerPayload = context.Get<ControllerMessageContextPayload>();
try try
{ {
return (Task<object>)method.Invoke(context.Controller, parameterFactories.Select(p => p(context)).ToArray()); return (Task<object>)method.Invoke(controllerPayload.Controller, parameterFactories.Select(p => p(context)).ToArray());
} }
catch (Exception e) catch (Exception e)
{ {
@ -302,9 +297,10 @@ namespace Tapeti.Default
{ {
return context => return context =>
{ {
var controllerPayload = context.Get<ControllerMessageContextPayload>();
try try
{ {
return Task.FromResult(method.Invoke(context.Controller, parameterFactories.Select(p => p(context)).ToArray())); return Task.FromResult(method.Invoke(controllerPayload.Controller, parameterFactories.Select(p => p(context)).ToArray()));
} }
catch (Exception e) catch (Exception e)
{ {

95
Tapeti/Default/MessageContext.cs
View File

@ -7,7 +7,7 @@ namespace Tapeti.Default
{ {
internal class MessageContext : IMessageContext internal class MessageContext : IMessageContext
{ {
private readonly Dictionary<string, object> items = new(); private readonly Dictionary<Type, IMessageContextPayload> payloads = new();
/// <inheritdoc /> /// <inheritdoc />
@ -22,6 +22,9 @@ namespace Tapeti.Default
/// <inheritdoc /> /// <inheritdoc />
public string RoutingKey { get; set; } public string RoutingKey { get; set; }
/// <inheritdoc />
public byte[] RawBody { get; set; }
/// <inheritdoc /> /// <inheritdoc />
public object Message { get; set; } public object Message { get; set; }
@ -31,21 +34,52 @@ namespace Tapeti.Default
/// <inheritdoc /> /// <inheritdoc />
public IBinding Binding { get; set; } public IBinding Binding { get; set; }
public void Store<T>(T payload) where T : IMessageContextPayload
{
payloads.Add(typeof(T), payload);
}
public void StoreOrUpdate<T>(Func<T> onAdd, Action<T> onUpdate) where T : IMessageContextPayload
{
if (payloads.TryGetValue(typeof(T), out var payload))
onUpdate((T)payload);
else
payloads.Add(typeof(T), onAdd());
}
public T Get<T>() where T : IMessageContextPayload
{
return (T)payloads[typeof(T)];
}
public bool TryGet<T>(out T payload) where T : IMessageContextPayload
{
if (payloads.TryGetValue(typeof(T), out var payloadValue))
{
payload = (T)payloadValue;
return true;
}
payload = default;
return false;
}
/// <inheritdoc /> /// <inheritdoc />
public void Dispose() public void Dispose()
{ {
foreach (var item in items.Values) foreach (var payload in payloads.Values)
(item as IDisposable)?.Dispose(); (payload as IDisposable)?.Dispose();
} }
/// <inheritdoc /> /// <inheritdoc />
public async ValueTask DisposeAsync() public async ValueTask DisposeAsync()
{ {
foreach (var item in items.Values) foreach (var payload in payloads.Values)
{ {
if (item is IAsyncDisposable asyncDisposable) if (payload is IAsyncDisposable asyncDisposable)
await asyncDisposable.DisposeAsync(); await asyncDisposable.DisposeAsync();
} }
} }
@ -55,21 +89,66 @@ namespace Tapeti.Default
/// <inheritdoc /> /// <inheritdoc />
public void Store(string key, object value) public void Store(string key, object value)
{ {
items.Add(key, value); StoreOrUpdate(
() => new KeyValuePayload(key, value),
payload => payload.Add(key, value));
} }
/// <inheritdoc /> /// <inheritdoc />
public bool Get<T>(string key, out T value) where T : class public bool Get<T>(string key, out T value) where T : class
{ {
if (!items.TryGetValue(key, out var objectValue)) if (!TryGet<KeyValuePayload>(out var payload) ||
!payload.TryGetValue(key, out var objectValue))
{ {
value = default(T); value = null;
return false; return false;
} }
value = (T)objectValue; value = (T)objectValue;
return true; return true;
} }
// ReSharper disable once InconsistentNaming
public class KeyValuePayload : IMessageContextPayload, IDisposable, IAsyncDisposable
{
private readonly Dictionary<string, object> items = new();
public KeyValuePayload(string key, object value)
{
Add(key, value);
}
public void Add(string key, object value)
{
items.Add(key, value);
}
public bool TryGetValue(string key, out object value)
{
return items.TryGetValue(key, out value);
}
public void Dispose()
{
foreach (var item in items.Values)
(item as IDisposable)?.Dispose();
}
public async ValueTask DisposeAsync()
{
foreach (var item in items.Values)
{
if (item is IAsyncDisposable asyncDisposable)
await asyncDisposable.DisposeAsync();
}
}
}
} }
} }

109
docs/tapeticmd.rst
View File

@ -1,7 +1,9 @@
Tapeti.Cmd Tapeti.Cmd
========== ==========
The Tapeti command-line tool provides various operations for managing messages. It tries to be compatible with all type of messages, but has been tested only against JSON messages, specifically those sent by Tapeti. The Tapeti command-line tool provides various operations for managing messages and queues.
Some operations, like shovel, are compatible with all types of messages. However, commands like import and export can assume JSON messages, specifically those sent by Tapeti, so your results may vary.
Common parameters Common parameters
@ -83,6 +85,12 @@ Read messages from disk as previously exported and publish them to a queue.
--maxrate <messages per second> --maxrate <messages per second>
The maximum amount of messages per second to import. The maximum amount of messages per second to import.
--batchsize <messages per batch>
How many messages to import before pausing. Will wait for manual confirmation unless batchpausetime is specified.
--batchpausetime <seconds>
How many seconds to wait before starting the next batch if batchsize is specified.
Either input, message or pipe is required. Either input, message or pipe is required.
@ -128,6 +136,12 @@ Reads messages from a queue and publishes them to another queue, optionally to a
--maxrate <messages per second> --maxrate <messages per second>
The maximum amount of messages per second to shovel. The maximum amount of messages per second to shovel.
--batchsize <messages per batch>
How many messages to shovel before pausing. Will wait for manual confirmation unless batchpausetime is specified.
--batchpausetime <seconds>
How many seconds to wait before starting the next batch if batchsize is specified.
Example: Example:
:: ::
@ -135,6 +149,99 @@ Example:
.\Tapeti.Cmd.exe shovel -q tapeti.example.01 -t tapeti.example.06 .\Tapeti.Cmd.exe shovel -q tapeti.example.01 -t tapeti.example.06
Purge
-----
Removes all messages from a queue destructively.
-q <queue>, --queue <queue>
*Required*. The queue to purge.
--confirm
Confirms the purging of the specified queue. If not provided, an interactive prompt will ask for confirmation.
Example:
::
.\Tapeti.Cmd.exe purge -q tapeti.example.01
Declare queue
-------------
Declares a durable queue without arguments.
-q <queue>, --queue <queue>
*Required*. The queue to declare.
-b <bindings>, --bindings <bindings>
One or more bindings to add to the queue. Format: <exchange>:<routingKey>
Example:
::
.\Tapeti.Cmd.exe declarequeue -q tapeti.cmd.example -b myexchange:example.message myexchange:another.message
Bind queue
----------
Add a binding to an existing queue.
-q <queue>, --queue <queue>
*Required*. The name of the queue to add the binding(s) to.
-b <bindings>, --bindings <bindings>
One or more bindings to add to the queue. Format: <exchange>:<routingKey>
Example:
::
.\Tapeti.Cmd.exe bindqueue -q tapeti.cmd.example -b myexchange:example.message myexchange:another.message
Unbind queue
------------
Remove a binding from a queue.
-q <queue>, --queue <queue>
*Required*. The name of the queue to remove the binding(s) from.
-b <bindings>, --bindings <bindings>
One or more bindings to remove from the queue. Format: <exchange>:<routingKey>
Example:
::
.\Tapeti.Cmd.exe unbindqueue -q tapeti.cmd.example -b myexchange:example.message myexchange:another.message
Remove queue
------------
Removes a durable queue.
-q <queue>, --queue <queue>
*Required*. The name of the queue to remove.
--confirm
Confirms the removal of the specified queue. If not provided, an interactive prompt will ask for confirmation.
--confirmpurge
Confirms the removal of the specified queue even if there still are messages in the queue. If not provided, an interactive prompt will ask for confirmation.
Example:
::
.\Tapeti.Cmd.exe removequeue -q tapeti.cmd.example
Serialization methods Serialization methods
--------------------- ---------------------