From e157598fa77566c363a9efd2b40fce13b3e40803 Mon Sep 17 00:00:00 2001 From: Mark van Renswoude Date: Sat, 4 Sep 2021 14:01:03 +0200 Subject: [PATCH] Implemented #32: Progress and batching for Tapeti.Cmd import Refactored console interaction to support this feature Updated documentation with recently added verbs --- Tapeti.Cmd/ConsoleHelper/ConsoleWrapper.cs | 259 ++++++++++++++++++ Tapeti.Cmd/ConsoleHelper/IConsole.cs | 44 +++ .../{ASCII => ConsoleHelper}/ProgressBar.cs | 31 +-- Tapeti.Cmd/Program.cs | 7 +- .../RateLimiter/BatchSizeRateLimiter.cs | 80 ++++++ Tapeti.Cmd/RateLimiter/RateLimiterFactory.cs | 29 ++ Tapeti.Cmd/Verbs/BindQueueVerb.cs | 6 +- Tapeti.Cmd/Verbs/DeclareQueueVerb.cs | 9 +- Tapeti.Cmd/Verbs/ExampleVerb.cs | 3 +- Tapeti.Cmd/Verbs/ExecutableVerbAttribute.cs | 3 +- Tapeti.Cmd/Verbs/ExportVerb.cs | 20 +- Tapeti.Cmd/Verbs/ImportVerb.cs | 37 ++- Tapeti.Cmd/Verbs/PurgeVerb.cs | 12 +- Tapeti.Cmd/Verbs/RemoveQueueVerb.cs | 17 +- Tapeti.Cmd/Verbs/ShovelVerb.cs | 68 +++-- Tapeti.Cmd/Verbs/UnbindQueueVerb.cs | 6 +- docs/tapeticmd.rst | 109 +++++++- 17 files changed, 621 insertions(+), 119 deletions(-) create mode 100644 Tapeti.Cmd/ConsoleHelper/ConsoleWrapper.cs create mode 100644 Tapeti.Cmd/ConsoleHelper/IConsole.cs rename Tapeti.Cmd/{ASCII => ConsoleHelper}/ProgressBar.cs (73%) create mode 100644 Tapeti.Cmd/RateLimiter/BatchSizeRateLimiter.cs create mode 100644 Tapeti.Cmd/RateLimiter/RateLimiterFactory.cs diff --git a/Tapeti.Cmd/ConsoleHelper/ConsoleWrapper.cs b/Tapeti.Cmd/ConsoleHelper/ConsoleWrapper.cs new file mode 100644 index 0000000..05f8541 --- /dev/null +++ b/Tapeti.Cmd/ConsoleHelper/ConsoleWrapper.cs @@ -0,0 +1,259 @@ +using System; +using System.Collections.Generic; +using System.Threading; + +namespace Tapeti.Cmd.ConsoleHelper +{ + public class ConsoleWrapper : IConsole + { + private readonly List temporaryWriters = new(); + private bool temporaryActive; + + private int temporaryCursorTop; + + + public ConsoleWrapper() + { + temporaryCursorTop = Console.CursorTop; + + Console.CancelKeyPress += (_, args) => + { + if (Cancelled) + return; + + using var consoleWriter = GetPermanentWriter(); + consoleWriter.WriteLine("Cancelling..."); + + args.Cancel = true; + Cancelled = true; + }; + } + + + public void Dispose() + { + foreach (var writer in temporaryWriters) + writer.Dispose(); + + Console.CursorVisible = true; + GC.SuppressFinalize(this); + } + + + public bool Cancelled { get; private set; } + + public IConsoleWriter GetPermanentWriter() + { + return new PermanentWriter(this); + } + + + public IConsoleWriter GetTemporaryWriter() + { + var writer = new TemporaryWriter(this, temporaryWriters.Count); + temporaryWriters.Add(writer); + + return writer; + } + + + private void AcquirePermanent() + { + if (!temporaryActive) + return; + + foreach (var writer in temporaryWriters) + { + Console.SetCursorPosition(0, temporaryCursorTop + writer.RelativePosition); + writer.Clear(); + } + + Console.SetCursorPosition(0, temporaryCursorTop); + Console.CursorVisible = true; + temporaryActive = false; + } + + + private void ReleasePermanent() + { + if (temporaryWriters.Count == 0) + { + temporaryCursorTop = Console.CursorTop; + return; + } + + foreach (var writer in temporaryWriters) + { + writer.Restore(); + Console.WriteLine(); + } + + // Store the cursor position afterwards to account for buffer scrolling + temporaryCursorTop = Console.CursorTop - temporaryWriters.Count; + Console.CursorVisible = false; + temporaryActive = true; + } + + + private void AcquireTemporary(TemporaryWriter writer) + { + Console.SetCursorPosition(0, temporaryCursorTop + writer.RelativePosition); + + if (temporaryActive) + return; + + Console.CursorVisible = false; + temporaryActive = true; + } + + + private void DisposeWriter(BaseWriter writer) + { + if (writer is not TemporaryWriter temporaryWriter) + return; + + Console.SetCursorPosition(0, temporaryCursorTop + temporaryWriter.RelativePosition); + temporaryWriter.Clear(); + + temporaryWriters.Remove(temporaryWriter); + } + + + private abstract class BaseWriter : IConsoleWriter + { + protected readonly ConsoleWrapper Owner; + + + protected BaseWriter(ConsoleWrapper owner) + { + Owner = owner; + } + + + public virtual void Dispose() + { + Owner.DisposeWriter(this); + GC.SuppressFinalize(this); + } + + public abstract bool Enabled { get; } + + public abstract void WriteLine(string value); + + + public void Confirm(string message) + { + WriteLine(message); + TryReadKey(false, out var _); + } + + + public bool ConfirmYesNo(string message) + { + WriteLine($"{message} (Y/N) "); + if (!TryReadKey(true, out var key)) + return false; + + return key.KeyChar == 'y' || key.KeyChar == 'Y'; + } + + + private bool TryReadKey(bool showKeyOutput, out ConsoleKeyInfo keyInfo) + { + while (!Owner.Cancelled && !Console.KeyAvailable) + Thread.Sleep(50); + + if (Owner.Cancelled) + { + keyInfo = default; + return false; + } + + keyInfo = Console.ReadKey(!showKeyOutput); + return true; + } + } + + + private class PermanentWriter : BaseWriter + { + public PermanentWriter(ConsoleWrapper owner) : base(owner) + { + } + + + public override bool Enabled => true; + + + public override void WriteLine(string value) + { + Owner.AcquirePermanent(); + try + { + Console.WriteLine(value); + } + finally + { + Owner.ReleasePermanent(); + } + } + } + + + private class TemporaryWriter : BaseWriter + { + public int RelativePosition { get; } + + private bool isActive; + private string storedValue; + + + public TemporaryWriter(ConsoleWrapper owner, int relativePosition) : base(owner) + { + RelativePosition = relativePosition; + } + + + public override bool Enabled => !Console.IsOutputRedirected; + + + public override void WriteLine(string value) + { + if (!Enabled) + return; + + Owner.AcquireTemporary(this); + Console.Write(value); + + if (!string.IsNullOrEmpty(storedValue) && storedValue.Length > value.Length) + // Clear characters remaining from the previous value + Console.Write(new string(' ', storedValue.Length - value.Length)); + + storedValue = value; + isActive = true; + } + + + public void Clear() + { + if (!isActive) + return; + + if (!string.IsNullOrEmpty(storedValue)) + Console.Write(new string(' ', storedValue.Length)); + + isActive = false; + } + + + public void Restore() + { + if (string.IsNullOrEmpty(storedValue)) + return; + + Console.Write(storedValue); + isActive = true; + } + } + } +} diff --git a/Tapeti.Cmd/ConsoleHelper/IConsole.cs b/Tapeti.Cmd/ConsoleHelper/IConsole.cs new file mode 100644 index 0000000..d563d1e --- /dev/null +++ b/Tapeti.Cmd/ConsoleHelper/IConsole.cs @@ -0,0 +1,44 @@ +using System; + +namespace Tapeti.Cmd.ConsoleHelper +{ + /// + /// Wraps access to the console to provide cooperation between temporary outputs like the + /// progress bar and batch confirmations. Temporary outputs hide the cursor and will be + /// automatically be erased and restored when a permanent writer is called. + /// + /// + /// Temporary outputs are automatically supressed when the console output is redirected. + /// The Enabled property will reflect this. + /// + public interface IConsole : IDisposable + { + bool Cancelled { get; } + + IConsoleWriter GetPermanentWriter(); + IConsoleWriter GetTemporaryWriter(); + } + + + /// + /// For simplicity outputs only support one line of text. + /// For temporary writers, each call to WriteLine will overwrite the previous and clear any + /// extra characters if the previous value was longer. + /// + public interface IConsoleWriter : IDisposable + { + bool Enabled { get; } + + void WriteLine(string value); + + /// + /// Waits for any user input. + /// + void Confirm(string message); + + /// + /// Waits for user confirmation (Y/N). + /// + bool ConfirmYesNo(string message); + } +} diff --git a/Tapeti.Cmd/ASCII/ProgressBar.cs b/Tapeti.Cmd/ConsoleHelper/ProgressBar.cs similarity index 73% rename from Tapeti.Cmd/ASCII/ProgressBar.cs rename to Tapeti.Cmd/ConsoleHelper/ProgressBar.cs index f357457..2d08be8 100644 --- a/Tapeti.Cmd/ASCII/ProgressBar.cs +++ b/Tapeti.Cmd/ConsoleHelper/ProgressBar.cs @@ -1,12 +1,13 @@ using System; using System.Text; -namespace Tapeti.Cmd.ASCII +namespace Tapeti.Cmd.ConsoleHelper { public class ProgressBar : IDisposable, IProgress { private static readonly TimeSpan UpdateInterval = TimeSpan.FromMilliseconds(20); + private readonly IConsoleWriter consoleWriter; private readonly int max; private readonly int width; private readonly bool showPosition; @@ -14,39 +15,33 @@ namespace Tapeti.Cmd.ASCII private readonly bool enabled; private DateTime lastUpdate = DateTime.MinValue; - private int lastOutputLength; - public ProgressBar(int max, int width = 10, bool showPosition = true) + public ProgressBar(IConsole console, int max, int width = 10, bool showPosition = true) { if (width <= 0) throw new ArgumentOutOfRangeException(nameof(width), "Width must be greater than zero"); if (max <= 0) throw new ArgumentOutOfRangeException(nameof(max), "Max must be greater than zero"); - + + consoleWriter = console.GetTemporaryWriter(); + this.max = max; this.width = width; this.showPosition = showPosition; - enabled = !Console.IsOutputRedirected; + enabled = consoleWriter.Enabled; if (!enabled) return; - Console.CursorVisible = false; Redraw(); } public void Dispose() { - if (!enabled || lastOutputLength <= 0) - return; - - Console.CursorLeft = 0; - Console.Write(new string(' ', lastOutputLength)); - Console.CursorLeft = 0; - Console.CursorVisible = true; + consoleWriter.Dispose(); } @@ -89,15 +84,7 @@ namespace Tapeti.Cmd.ASCII else output.Append(" ").Append((int)Math.Truncate((decimal)position / max * 100)).Append("%"); - - var newLength = output.Length; - if (newLength < lastOutputLength) - output.Append(new string(' ', lastOutputLength - output.Length)); - - Console.CursorLeft = 0; - Console.Write(output); - - lastOutputLength = newLength; + consoleWriter.WriteLine(output.ToString()); } } } diff --git a/Tapeti.Cmd/Program.cs b/Tapeti.Cmd/Program.cs index b1f563a..787d326 100644 --- a/Tapeti.Cmd/Program.cs +++ b/Tapeti.Cmd/Program.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.Linq; using System.Reflection; using CommandLine; +using Tapeti.Cmd.ConsoleHelper; using Tapeti.Cmd.Verbs; namespace Tapeti.Cmd @@ -26,8 +27,10 @@ namespace Tapeti.Cmd // Should have been validated by the ExecutableVerbAttribute Debug.Assert(executer != null, nameof(executer) + " != null"); - - executer.Execute(); + + using var consoleWrapper = new ConsoleWrapper(); + + executer.Execute(consoleWrapper); exitCode = 0; } catch (Exception e) diff --git a/Tapeti.Cmd/RateLimiter/BatchSizeRateLimiter.cs b/Tapeti.Cmd/RateLimiter/BatchSizeRateLimiter.cs new file mode 100644 index 0000000..c9e8ed9 --- /dev/null +++ b/Tapeti.Cmd/RateLimiter/BatchSizeRateLimiter.cs @@ -0,0 +1,80 @@ +using System; +using System.Threading; +using Tapeti.Cmd.ConsoleHelper; + +namespace Tapeti.Cmd.RateLimiter +{ + public abstract class BaseBatchSizeRateLimiter : IRateLimiter + { + private readonly IConsole console; + private readonly IRateLimiter decoratedRateLimiter; + private readonly int batchSize; + private int batchCount; + + + protected BaseBatchSizeRateLimiter(IConsole console, IRateLimiter decoratedRateLimiter, int batchSize) + { + this.console = console; + this.decoratedRateLimiter = decoratedRateLimiter; + this.batchSize = batchSize; + } + + + public void Execute(Action action) + { + batchCount++; + if (batchCount > batchSize) + { + Pause(console); + batchCount = 0; + } + + decoratedRateLimiter.Execute(action); + } + + + protected abstract void Pause(IConsole console); + } + + + public class ManualBatchSizeRateLimiter : BaseBatchSizeRateLimiter + { + public ManualBatchSizeRateLimiter(IConsole console, IRateLimiter decoratedRateLimiter, int batchSize) : base(console, decoratedRateLimiter, batchSize) + { + } + + + protected override void Pause(IConsole console) + { + using var consoleWriter = console.GetTemporaryWriter(); + consoleWriter.Confirm("Press any key to continue with the next batch..."); + } + } + + + public class TimedBatchSizeRateLimiter : BaseBatchSizeRateLimiter + { + private readonly int batchPauseTime; + + + public TimedBatchSizeRateLimiter(IConsole console, IRateLimiter decoratedRateLimiter, int batchSize, int batchPauseTime) : base(console, decoratedRateLimiter, batchSize) + { + this.batchPauseTime = batchPauseTime; + } + + + protected override void Pause(IConsole console) + { + using var consoleWriter = console.GetTemporaryWriter(); + + var remaining = batchPauseTime; + while (remaining > 0 && !console.Cancelled) + { + consoleWriter.WriteLine($"Next batch in {remaining} second{(remaining != 1 ? "s" : "")}..."); + + Thread.Sleep(1000); + remaining--; + } + } + } +} diff --git a/Tapeti.Cmd/RateLimiter/RateLimiterFactory.cs b/Tapeti.Cmd/RateLimiter/RateLimiterFactory.cs new file mode 100644 index 0000000..45ed0e2 --- /dev/null +++ b/Tapeti.Cmd/RateLimiter/RateLimiterFactory.cs @@ -0,0 +1,29 @@ +using System; +using Tapeti.Cmd.ConsoleHelper; + +namespace Tapeti.Cmd.RateLimiter +{ + public static class RateLimiterFactory + { + public static IRateLimiter Create(IConsole console, int? maxRate, int? batchSize, int? batchPauseTime) + { + IRateLimiter rateLimiter; + + if (maxRate > 0) + rateLimiter = new SpreadRateLimiter(maxRate.Value, TimeSpan.FromSeconds(1)); + else + rateLimiter = new NoRateLimiter(); + + // ReSharper disable once InvertIf - I don't like the readability of that flow + if (batchSize > 0) + { + if (batchPauseTime > 0) + rateLimiter = new TimedBatchSizeRateLimiter(console, rateLimiter, batchSize.Value, batchPauseTime.Value); + else + rateLimiter = new ManualBatchSizeRateLimiter(console, rateLimiter, batchSize.Value); + } + + return rateLimiter; + } + } +} diff --git a/Tapeti.Cmd/Verbs/BindQueueVerb.cs b/Tapeti.Cmd/Verbs/BindQueueVerb.cs index 368b7df..fc4416d 100644 --- a/Tapeti.Cmd/Verbs/BindQueueVerb.cs +++ b/Tapeti.Cmd/Verbs/BindQueueVerb.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using CommandLine; using RabbitMQ.Client; +using Tapeti.Cmd.ConsoleHelper; using Tapeti.Cmd.Parser; namespace Tapeti.Cmd.Verbs @@ -29,8 +30,9 @@ namespace Tapeti.Cmd.Verbs } - public void Execute() + public void Execute(IConsole console) { + var consoleWriter = console.GetPermanentWriter(); var bindings = BindingParser.Parse(options.Bindings); var factory = new ConnectionFactory @@ -48,7 +50,7 @@ namespace Tapeti.Cmd.Verbs foreach (var (exchange, routingKey) in bindings) channel.QueueBind(options.QueueName, exchange, routingKey); - Console.WriteLine($"{bindings.Length} binding{(bindings.Length != 1 ? "s" : "")} added to queue {options.QueueName}."); + consoleWriter.WriteLine($"{bindings.Length} binding{(bindings.Length != 1 ? "s" : "")} added to queue {options.QueueName}."); } } } diff --git a/Tapeti.Cmd/Verbs/DeclareQueueVerb.cs b/Tapeti.Cmd/Verbs/DeclareQueueVerb.cs index 5ebd72f..a119233 100644 --- a/Tapeti.Cmd/Verbs/DeclareQueueVerb.cs +++ b/Tapeti.Cmd/Verbs/DeclareQueueVerb.cs @@ -2,11 +2,12 @@ using System.Collections.Generic; using CommandLine; using RabbitMQ.Client; +using Tapeti.Cmd.ConsoleHelper; using Tapeti.Cmd.Parser; namespace Tapeti.Cmd.Verbs { - [Verb("declarequeue", HelpText = "Declares a durable queue without arguments, compatible with Tapeti.")] + [Verb("declarequeue", HelpText = "Declares a durable queue without arguments.")] [ExecutableVerb(typeof(DeclareQueueVerb))] public class DeclareQueueOptions : BaseConnectionOptions { @@ -29,8 +30,10 @@ namespace Tapeti.Cmd.Verbs } - public void Execute() + public void Execute(IConsole console) { + var consoleWriter = console.GetPermanentWriter(); + // Parse early to fail early var bindings = BindingParser.Parse(options.Bindings); @@ -51,7 +54,7 @@ namespace Tapeti.Cmd.Verbs foreach (var (exchange, routingKey) in bindings) channel.QueueBind(options.QueueName, exchange, routingKey); - Console.WriteLine($"Queue {options.QueueName} declared with {bindings.Length} binding{(bindings.Length != 1 ? "s" : "")}."); + consoleWriter.WriteLine($"Queue {options.QueueName} declared with {bindings.Length} binding{(bindings.Length != 1 ? "s" : "")}."); } } } diff --git a/Tapeti.Cmd/Verbs/ExampleVerb.cs b/Tapeti.Cmd/Verbs/ExampleVerb.cs index 6998e70..b7fcb6e 100644 --- a/Tapeti.Cmd/Verbs/ExampleVerb.cs +++ b/Tapeti.Cmd/Verbs/ExampleVerb.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.Text; using CommandLine; using RabbitMQ.Client; +using Tapeti.Cmd.ConsoleHelper; using Tapeti.Cmd.Mock; using Tapeti.Cmd.Serialization; @@ -25,7 +26,7 @@ namespace Tapeti.Cmd.Verbs } - public void Execute() + public void Execute(IConsole console) { using var messageSerializer = new SingleFileJSONMessageSerializer(Console.OpenStandardOutput(), false, new UTF8Encoding(false)); diff --git a/Tapeti.Cmd/Verbs/ExecutableVerbAttribute.cs b/Tapeti.Cmd/Verbs/ExecutableVerbAttribute.cs index aaeec9e..b172dd2 100644 --- a/Tapeti.Cmd/Verbs/ExecutableVerbAttribute.cs +++ b/Tapeti.Cmd/Verbs/ExecutableVerbAttribute.cs @@ -1,4 +1,5 @@ using System; +using Tapeti.Cmd.ConsoleHelper; namespace Tapeti.Cmd.Verbs { @@ -8,7 +9,7 @@ namespace Tapeti.Cmd.Verbs /// public interface IVerbExecuter { - void Execute(); + void Execute(IConsole console); } diff --git a/Tapeti.Cmd/Verbs/ExportVerb.cs b/Tapeti.Cmd/Verbs/ExportVerb.cs index 33af936..ac687cd 100644 --- a/Tapeti.Cmd/Verbs/ExportVerb.cs +++ b/Tapeti.Cmd/Verbs/ExportVerb.cs @@ -3,7 +3,7 @@ using System.IO; using System.Text; using CommandLine; using RabbitMQ.Client; -using Tapeti.Cmd.ASCII; +using Tapeti.Cmd.ConsoleHelper; using Tapeti.Cmd.Serialization; namespace Tapeti.Cmd.Verbs @@ -37,8 +37,9 @@ namespace Tapeti.Cmd.Verbs } - public void Execute() + public void Execute(IConsole console) { + var consoleWriter = console.GetPermanentWriter(); var factory = new ConnectionFactory { HostName = options.Host, @@ -56,19 +57,12 @@ namespace Tapeti.Cmd.Verbs if (options.MaxCount.HasValue && options.MaxCount.Value < totalCount) totalCount = options.MaxCount.Value; - Console.WriteLine($"Exporting {totalCount} message{(totalCount != 1 ? "s" : "")} (actual number may differ if queue has active consumers or publishers)"); + consoleWriter.WriteLine($"Exporting {totalCount} message{(totalCount != 1 ? "s" : "")} (actual number may differ if queue has active consumers or publishers)"); var messageCount = 0; - var cancelled = false; - - Console.CancelKeyPress += (_, args) => - { - args.Cancel = true; - cancelled = true; - }; - using (var progressBar = new ProgressBar(totalCount)) + using (var progressBar = new ProgressBar(console, totalCount)) { - while (!cancelled && (!options.MaxCount.HasValue || messageCount < options.MaxCount.Value)) + while (!console.Cancelled && (!options.MaxCount.HasValue || messageCount < options.MaxCount.Value)) { var result = channel.BasicGet(options.QueueName, false); if (result == null) @@ -96,7 +90,7 @@ namespace Tapeti.Cmd.Verbs } } - Console.WriteLine($"{messageCount} message{(messageCount != 1 ? "s" : "")} exported."); + consoleWriter.WriteLine($"{messageCount} message{(messageCount != 1 ? "s" : "")} exported."); } diff --git a/Tapeti.Cmd/Verbs/ImportVerb.cs b/Tapeti.Cmd/Verbs/ImportVerb.cs index 1040220..cb2eb73 100644 --- a/Tapeti.Cmd/Verbs/ImportVerb.cs +++ b/Tapeti.Cmd/Verbs/ImportVerb.cs @@ -3,7 +3,7 @@ using System.IO; using System.Text; using CommandLine; using RabbitMQ.Client; -using Tapeti.Cmd.ASCII; +using Tapeti.Cmd.ConsoleHelper; using Tapeti.Cmd.RateLimiter; using Tapeti.Cmd.Serialization; @@ -27,6 +27,12 @@ namespace Tapeti.Cmd.Verbs [Option("maxrate", HelpText = "The maximum amount of messages per second to import.")] public int? MaxRate { get; set; } + + [Option("batchsize", HelpText = "How many messages to import before pausing. Will wait for manual confirmation unless batchpausetime is specified.")] + public int? BatchSize { get; set; } + + [Option("batchpausetime", HelpText = "How many seconds to wait before starting the next batch if batchsize is specified.")] + public int? BatchPauseTime { get; set; } } @@ -41,8 +47,9 @@ namespace Tapeti.Cmd.Verbs } - public void Execute() + public void Execute(IConsole console) { + var consoleWriter = console.GetPermanentWriter(); var factory = new ConnectionFactory { HostName = options.Host, @@ -55,30 +62,27 @@ namespace Tapeti.Cmd.Verbs using var messageSerializer = GetMessageSerializer(options); using var connection = factory.CreateConnection(); using var channel = connection.CreateModel(); - var rateLimiter = GetRateLimiter(options.MaxRate); + var rateLimiter = RateLimiterFactory.Create(console, options.MaxRate, options.BatchSize, options.BatchPauseTime); var totalCount = messageSerializer.GetMessageCount(); var messageCount = 0; - var cancelled = false; - Console.CancelKeyPress += (_, args) => - { - args.Cancel = true; - cancelled = true; - }; ProgressBar progress = null; if (totalCount > 0) - progress = new ProgressBar(totalCount); + progress = new ProgressBar(console, totalCount); try { foreach (var message in messageSerializer.Deserialize(channel)) { - if (cancelled) + if (console.Cancelled) break; rateLimiter.Execute(() => { + if (console.Cancelled) + return; + var exchange = options.PublishToExchange ? message.Exchange : ""; var routingKey = options.PublishToExchange ? message.RoutingKey : message.Queue; @@ -96,7 +100,7 @@ namespace Tapeti.Cmd.Verbs progress?.Dispose(); } - Console.WriteLine($"{messageCount} message{(messageCount != 1 ? "s" : "")} published."); + consoleWriter.WriteLine($"{messageCount} message{(messageCount != 1 ? "s" : "")} published."); } @@ -136,14 +140,5 @@ namespace Tapeti.Cmd.Verbs disposeStream = true; return new FileStream(options.InputFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); } - - - private static IRateLimiter GetRateLimiter(int? maxRate) - { - if (!maxRate.HasValue || maxRate.Value <= 0) - return new NoRateLimiter(); - - return new SpreadRateLimiter(maxRate.Value, TimeSpan.FromSeconds(1)); - } } } diff --git a/Tapeti.Cmd/Verbs/PurgeVerb.cs b/Tapeti.Cmd/Verbs/PurgeVerb.cs index 30c6d00..bd99cd2 100644 --- a/Tapeti.Cmd/Verbs/PurgeVerb.cs +++ b/Tapeti.Cmd/Verbs/PurgeVerb.cs @@ -1,6 +1,7 @@ using System; using CommandLine; using RabbitMQ.Client; +using Tapeti.Cmd.ConsoleHelper; namespace Tapeti.Cmd.Verbs { @@ -27,14 +28,13 @@ namespace Tapeti.Cmd.Verbs } - public void Execute() + public void Execute(IConsole console) { + var consoleWriter = console.GetPermanentWriter(); + if (!options.Confirm) { - Console.Write($"Do you want to purge the queue '{options.QueueName}'? (Y/N) "); - var answer = Console.ReadLine(); - - if (string.IsNullOrEmpty(answer) || !answer.Equals("Y", StringComparison.CurrentCultureIgnoreCase)) + if (!consoleWriter.ConfirmYesNo($"Do you want to purge the queue '{options.QueueName}'?")) return; } @@ -52,7 +52,7 @@ namespace Tapeti.Cmd.Verbs var messageCount = channel.QueuePurge(options.QueueName); - Console.WriteLine($"{messageCount} message{(messageCount != 1 ? "s" : "")} purged from '{options.QueueName}'."); + consoleWriter.WriteLine($"{messageCount} message{(messageCount != 1 ? "s" : "")} purged from '{options.QueueName}'."); } } } diff --git a/Tapeti.Cmd/Verbs/RemoveQueueVerb.cs b/Tapeti.Cmd/Verbs/RemoveQueueVerb.cs index f83b4d1..59066c3 100644 --- a/Tapeti.Cmd/Verbs/RemoveQueueVerb.cs +++ b/Tapeti.Cmd/Verbs/RemoveQueueVerb.cs @@ -2,6 +2,7 @@ using CommandLine; using RabbitMQ.Client; using RabbitMQ.Client.Exceptions; +using Tapeti.Cmd.ConsoleHelper; namespace Tapeti.Cmd.Verbs { @@ -31,14 +32,13 @@ namespace Tapeti.Cmd.Verbs } - public void Execute() + public void Execute(IConsole console) { + var consoleWriter = console.GetPermanentWriter(); + if (!options.Confirm) { - Console.Write($"Do you want to remove the queue '{options.QueueName}'? (Y/N) "); - var answer = Console.ReadLine(); - - if (string.IsNullOrEmpty(answer) || !answer.Equals("Y", StringComparison.CurrentCultureIgnoreCase)) + if (!consoleWriter.ConfirmYesNo($"Do you want to remove the queue '{options.QueueName}'?")) return; } @@ -66,10 +66,7 @@ namespace Tapeti.Cmd.Verbs { if (!options.ConfirmPurge) { - Console.Write($"There are messages remaining. Do you want to purge the queue '{options.QueueName}'? (Y/N) "); - var answer = Console.ReadLine(); - - if (string.IsNullOrEmpty(answer) || !answer.Equals("Y", StringComparison.CurrentCultureIgnoreCase)) + if (!consoleWriter.ConfirmYesNo($"There are messages remaining. Do you want to purge the queue '{options.QueueName}'?")) return; } @@ -82,7 +79,7 @@ namespace Tapeti.Cmd.Verbs throw; } - Console.WriteLine(messageCount == 0 + consoleWriter.WriteLine(messageCount == 0 ? $"Empty or non-existent queue '{options.QueueName}' removed." : $"{messageCount} message{(messageCount != 1 ? "s" : "")} purged while removing '{options.QueueName}'."); } diff --git a/Tapeti.Cmd/Verbs/ShovelVerb.cs b/Tapeti.Cmd/Verbs/ShovelVerb.cs index c601b5a..08b96a3 100644 --- a/Tapeti.Cmd/Verbs/ShovelVerb.cs +++ b/Tapeti.Cmd/Verbs/ShovelVerb.cs @@ -1,7 +1,7 @@ using System; using CommandLine; using RabbitMQ.Client; -using Tapeti.Cmd.ASCII; +using Tapeti.Cmd.ConsoleHelper; using Tapeti.Cmd.RateLimiter; namespace Tapeti.Cmd.Verbs @@ -39,6 +39,12 @@ namespace Tapeti.Cmd.Verbs [Option("maxrate", HelpText = "The maximum amount of messages per second to shovel.")] public int? MaxRate { get; set; } + + [Option("batchsize", HelpText = "How many messages to shovel before pausing. Will wait for manual confirmation unless batchpausetime is specified.")] + public int? BatchSize { get; set; } + + [Option("batchpausetime", HelpText = "How many seconds to wait before starting the next batch if batchsize is specified.")] + public int? BatchPauseTime { get; set; } } @@ -53,7 +59,7 @@ namespace Tapeti.Cmd.Verbs } - public void Execute() + public void Execute(IConsole console) { var sourceFactory = new ConnectionFactory { @@ -81,48 +87,49 @@ namespace Tapeti.Cmd.Verbs using var targetConnection = targetFactory.CreateConnection(); using var targetChannel = targetConnection.CreateModel(); - Shovel(options, sourceChannel, targetChannel); + Shovel(console, options, sourceChannel, targetChannel); } else - Shovel(options, sourceChannel, sourceChannel); + Shovel(console, options, sourceChannel, sourceChannel); } - private static void Shovel(ShovelOptions options, IModel sourceChannel, IModel targetChannel) + private static void Shovel(IConsole console, ShovelOptions options, IModel sourceChannel, IModel targetChannel) { - var rateLimiter = GetRateLimiter(options.MaxRate); + var consoleWriter = console.GetPermanentWriter(); + var rateLimiter = RateLimiterFactory.Create(console, options.MaxRate, options.BatchSize, options.BatchPauseTime); var targetQueueName = !string.IsNullOrEmpty(options.TargetQueueName) ? options.TargetQueueName : options.QueueName; var totalCount = (int)sourceChannel.MessageCount(options.QueueName); if (options.MaxCount.HasValue && options.MaxCount.Value < totalCount) totalCount = options.MaxCount.Value; - Console.WriteLine($"Shoveling {totalCount} message{(totalCount != 1 ? "s" : "")} (actual number may differ if queue has active consumers or publishers)"); + consoleWriter.WriteLine($"Shoveling {totalCount} message{(totalCount != 1 ? "s" : "")} (actual number may differ if queue has active consumers or publishers)"); var messageCount = 0; - var cancelled = false; - Console.CancelKeyPress += (_, args) => + using (var progressBar = new ProgressBar(console, totalCount)) { - args.Cancel = true; - cancelled = true; - }; + var hasMessage = true; - using (var progressBar = new ProgressBar(totalCount)) - { - while (!cancelled && (!options.MaxCount.HasValue || messageCount < options.MaxCount.Value)) + while (!console.Cancelled && hasMessage && (!options.MaxCount.HasValue || messageCount < options.MaxCount.Value)) { - var result = sourceChannel.BasicGet(options.QueueName, false); - if (result == null) - // No more messages on the queue - break; - - // Since RabbitMQ client 6 we need to copy the body before calling another channel method - // like BasicPublish, or the published body will be corrupted if sourceChannel and targetChannel are the same - var bodyCopy = result.Body.ToArray(); - - rateLimiter.Execute(() => { + if (console.Cancelled) + return; + + var result = sourceChannel.BasicGet(options.QueueName, false); + if (result == null) + { + // No more messages on the queue + hasMessage = false; + return; + } + + // Since RabbitMQ client 6 we need to copy the body before calling another channel method + // like BasicPublish, or the published body will be corrupted if sourceChannel and targetChannel are the same + var bodyCopy = result.Body.ToArray(); + targetChannel.BasicPublish("", targetQueueName, result.BasicProperties, bodyCopy); messageCount++; @@ -135,7 +142,7 @@ namespace Tapeti.Cmd.Verbs } } - Console.WriteLine($"{messageCount} message{(messageCount != 1 ? "s" : "")} shoveled."); + consoleWriter.WriteLine($"{messageCount} message{(messageCount != 1 ? "s" : "")} shoveled."); } @@ -168,14 +175,5 @@ namespace Tapeti.Cmd.Verbs // Everything's the same, we can use the same channel return false; } - - - private static IRateLimiter GetRateLimiter(int? maxRate) - { - if (!maxRate.HasValue || maxRate.Value <= 0) - return new NoRateLimiter(); - - return new SpreadRateLimiter(maxRate.Value, TimeSpan.FromSeconds(1)); - } } } diff --git a/Tapeti.Cmd/Verbs/UnbindQueueVerb.cs b/Tapeti.Cmd/Verbs/UnbindQueueVerb.cs index 12c91f2..6dbcdeb 100644 --- a/Tapeti.Cmd/Verbs/UnbindQueueVerb.cs +++ b/Tapeti.Cmd/Verbs/UnbindQueueVerb.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using CommandLine; using RabbitMQ.Client; +using Tapeti.Cmd.ConsoleHelper; using Tapeti.Cmd.Parser; namespace Tapeti.Cmd.Verbs @@ -29,8 +30,9 @@ namespace Tapeti.Cmd.Verbs } - public void Execute() + public void Execute(IConsole console) { + var consoleWriter = console.GetPermanentWriter(); var bindings = BindingParser.Parse(options.Bindings); var factory = new ConnectionFactory @@ -48,7 +50,7 @@ namespace Tapeti.Cmd.Verbs foreach (var (exchange, routingKey) in bindings) channel.QueueUnbind(options.QueueName, exchange, routingKey); - Console.WriteLine($"{bindings.Length} binding{(bindings.Length != 1 ? "s" : "")} removed from queue {options.QueueName}."); + consoleWriter.WriteLine($"{bindings.Length} binding{(bindings.Length != 1 ? "s" : "")} removed from queue {options.QueueName}."); } } } diff --git a/docs/tapeticmd.rst b/docs/tapeticmd.rst index af3c77b..a474baf 100644 --- a/docs/tapeticmd.rst +++ b/docs/tapeticmd.rst @@ -1,7 +1,9 @@ Tapeti.Cmd ========== -The Tapeti command-line tool provides various operations for managing messages. It tries to be compatible with all type of messages, but has been tested only against JSON messages, specifically those sent by Tapeti. +The Tapeti command-line tool provides various operations for managing messages and queues. + +Some operations, like shovel, are compatible with all types of messages. However, commands like import and export can assume JSON messages, specifically those sent by Tapeti, so your results may vary. Common parameters @@ -83,6 +85,12 @@ Read messages from disk as previously exported and publish them to a queue. --maxrate The maximum amount of messages per second to import. +--batchsize + How many messages to import before pausing. Will wait for manual confirmation unless batchpausetime is specified. + +--batchpausetime + How many seconds to wait before starting the next batch if batchsize is specified. + Either input, message or pipe is required. @@ -128,6 +136,12 @@ Reads messages from a queue and publishes them to another queue, optionally to a --maxrate The maximum amount of messages per second to shovel. +--batchsize + How many messages to shovel before pausing. Will wait for manual confirmation unless batchpausetime is specified. + +--batchpausetime + How many seconds to wait before starting the next batch if batchsize is specified. + Example: :: @@ -135,6 +149,99 @@ Example: .\Tapeti.Cmd.exe shovel -q tapeti.example.01 -t tapeti.example.06 +Purge +----- + +Removes all messages from a queue destructively. + +-q , --queue + *Required*. The queue to purge. + +--confirm + Confirms the purging of the specified queue. If not provided, an interactive prompt will ask for confirmation. + + +Example: +:: + + .\Tapeti.Cmd.exe purge -q tapeti.example.01 + + +Declare queue +------------- + +Declares a durable queue without arguments. + +-q , --queue + *Required*. The queue to declare. + +-b , --bindings + One or more bindings to add to the queue. Format: : + + +Example: +:: + + .\Tapeti.Cmd.exe declarequeue -q tapeti.cmd.example -b myexchange:example.message myexchange:another.message + + +Bind queue +---------- + +Add a binding to an existing queue. + +-q , --queue + *Required*. The name of the queue to add the binding(s) to. + +-b , --bindings + One or more bindings to add to the queue. Format: : + + +Example: +:: + + .\Tapeti.Cmd.exe bindqueue -q tapeti.cmd.example -b myexchange:example.message myexchange:another.message + + +Unbind queue +------------ + +Remove a binding from a queue. + +-q , --queue + *Required*. The name of the queue to remove the binding(s) from. + +-b , --bindings + One or more bindings to remove from the queue. Format: : + + +Example: +:: + + .\Tapeti.Cmd.exe unbindqueue -q tapeti.cmd.example -b myexchange:example.message myexchange:another.message + + +Remove queue +------------ + +Removes a durable queue. + +-q , --queue + *Required*. The name of the queue to remove. + +--confirm + Confirms the removal of the specified queue. If not provided, an interactive prompt will ask for confirmation. + +--confirmpurge + Confirms the removal of the specified queue even if there still are messages in the queue. If not provided, an interactive prompt will ask for confirmation. + + +Example: +:: + + .\Tapeti.Cmd.exe removequeue -q tapeti.cmd.example + + Serialization methods ---------------------