Implemented a rate limiting option for Tapeti.Cmd

Moved example 06 to the "examples" sub-folder

06-StatelessRequestResponse/06-StatelessRequestResponse.csproj

@@ -1,20 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
-    <RootNamespace>_06_StatelessRequestResponse</RootNamespace>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="SimpleInjector" Version="4.9.0" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\Examples\ExampleLib\ExampleLib.csproj" />
-    <ProjectReference Include="..\Examples\Messaging.TapetiExample\Messaging.TapetiExample.csproj" />
-    <ProjectReference Include="..\Tapeti.DataAnnotations\Tapeti.DataAnnotations.csproj" />
-    <ProjectReference Include="..\Tapeti.SimpleInjector\Tapeti.SimpleInjector.csproj" />
-  </ItemGroup>
-
-</Project>

Examples/06-StatelessRequestResponse/06-StatelessRequestResponse.csproj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <RootNamespace>_06_StatelessRequestResponse</RootNamespace>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="SimpleInjector" Version="4.9.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ExampleLib\ExampleLib.csproj" />
+    <ProjectReference Include="..\Messaging.TapetiExample\Messaging.TapetiExample.csproj" />
+    <ProjectReference Include="..\..\Tapeti.DataAnnotations\Tapeti.DataAnnotations.csproj" />
+    <ProjectReference Include="..\..\Tapeti.SimpleInjector\Tapeti.SimpleInjector.csproj" />
+  </ItemGroup>
+
+</Project>

Tapeti.Cmd/Commands/ImportCommand.cs

@@ -1,4 +1,5 @@
 using RabbitMQ.Client;
+using Tapeti.Cmd.RateLimiter;
 using Tapeti.Cmd.Serialization;
 
 namespace Tapeti.Cmd.Commands
@@ -10,17 +11,20 @@ namespace Tapeti.Cmd.Commands
         public bool DirectToQueue { get; set; }
 
 
-        public int Execute(IModel channel)
+        public int Execute(IModel channel, IRateLimiter rateLimiter)
         {
             var messageCount = 0;
 
             foreach (var message in MessageSerializer.Deserialize())
             {
-                var exchange = DirectToQueue ? "" : message.Exchange;
+                rateLimiter.Execute(() =>
-                var routingKey = DirectToQueue ? message.Queue : message.RoutingKey;
+                {
+                    var exchange = DirectToQueue ? "" : message.Exchange;
+                    var routingKey = DirectToQueue ? message.Queue : message.RoutingKey;
 
-                channel.BasicPublish(exchange, routingKey, message.Properties, message.Body);
+                    channel.BasicPublish(exchange, routingKey, message.Properties, message.Body);
-                messageCount++;
+                    messageCount++;
+                });
             }
 
             return messageCount;

Tapeti.Cmd/Commands/ShovelCommand.cs

@@ -1,4 +1,5 @@
 using RabbitMQ.Client;
+using Tapeti.Cmd.RateLimiter;
 
 namespace Tapeti.Cmd.Commands
 {
@@ -10,7 +11,7 @@ namespace Tapeti.Cmd.Commands
         public int? MaxCount { get; set; }
 
 
-        public int Execute(IModel sourceChannel, IModel targetChannel)
+        public int Execute(IModel sourceChannel, IModel targetChannel, IRateLimiter rateLimiter)
         {
             var messageCount = 0;
 
@@ -22,13 +23,14 @@ namespace Tapeti.Cmd.Commands
                     break;
 
 
-                targetChannel.BasicPublish("", TargetQueueName, result.BasicProperties, result.Body);
+                rateLimiter.Execute(() =>
+                {
+                    targetChannel.BasicPublish("", TargetQueueName, result.BasicProperties, result.Body);
+                    messageCount++;
 
-
+                    if (RemoveMessages)
-                messageCount++;
+                        sourceChannel.BasicAck(result.DeliveryTag, false);
-
+                });
-                if (RemoveMessages)
-                    sourceChannel.BasicAck(result.DeliveryTag, false);
             }
 
             return messageCount;

Tapeti.Cmd/Program.cs

@@ -4,10 +4,10 @@ using System.Diagnostics;
 using System.IO;
 using System.Text;
 using CommandLine;
-using CommandLine.Text;
 using RabbitMQ.Client;
 using RabbitMQ.Client.Framing;
 using Tapeti.Cmd.Commands;
+using Tapeti.Cmd.RateLimiter;
 using Tapeti.Cmd.Serialization;
 
 namespace Tapeti.Cmd
@@ -79,6 +79,9 @@ namespace Tapeti.Cmd
 
             [Option('e', "exchange", HelpText = "If specified publishes to the originating exchange using the original routing key. By default these are ignored and the message is published directly to the originating queue.")]
             public bool PublishToExchange { get; set; }
+
+            [Option("maxrate", HelpText = "The maximum amount of messages per second to import.")]
+            public int? MaxRate { get; set; }
         }
 
 
@@ -111,6 +114,9 @@ namespace Tapeti.Cmd
 
             [Option("targetpassword", HelpText = "Password used to connect to the target RabbitMQ server. Defaults to the source password.")]
             public string TargetPassword { get; set; }
+
+            [Option("maxrate", HelpText = "The maximum amount of messages per second to shovel.")]
+            public int? MaxRate { get; set; }
         }
 
 
@@ -236,6 +242,15 @@ namespace Tapeti.Cmd
         }
 
 
+        private static IRateLimiter GetRateLimiter(int? maxRate)
+        {
+            if (maxRate.GetValueOrDefault() <= 0)
+                return new NoRateLimiter();
+
+            return new SpreadRateLimiter(maxRate.Value, TimeSpan.FromSeconds(1));
+        }
+
+
         private static void RunExport(ExportOptions options)
         {
             int messageCount;
@@ -271,7 +286,7 @@ namespace Tapeti.Cmd
                     MessageSerializer = messageSerializer,
 
                     DirectToQueue = !options.PublishToExchange
-                }.Execute(channel);
+                }.Execute(channel, GetRateLimiter(options.MaxRate));
             }
 
             Console.WriteLine($"{messageCount} message{(messageCount != 1 ? "s" : "")} published.");
@@ -299,11 +314,11 @@ namespace Tapeti.Cmd
                     using (var targetConnection = GetTargetConnection(options))
                     using (var targetChannel = targetConnection.CreateModel())
                     {
-                        messageCount = shovelCommand.Execute(sourceChannel, targetChannel);
+                        messageCount = shovelCommand.Execute(sourceChannel, targetChannel, GetRateLimiter(options.MaxRate));
                     }
                 }
                 else
-                    messageCount = shovelCommand.Execute(sourceChannel, sourceChannel);
+                    messageCount = shovelCommand.Execute(sourceChannel, sourceChannel, GetRateLimiter(options.MaxRate));
             }
 
             Console.WriteLine($"{messageCount} message{(messageCount != 1 ? "s" : "")} shoveled.");

Tapeti.Cmd/RateLimiter/IRateLimiter.cs

@@ -0,0 +1,9 @@
+using System;
+
+namespace Tapeti.Cmd.RateLimiter
+{
+    public interface IRateLimiter
+    {
+        void Execute(Action action);
+    }
+}

Tapeti.Cmd/RateLimiter/NoRateLimiter.cs

@@ -0,0 +1,12 @@
+using System;
+
+namespace Tapeti.Cmd.RateLimiter
+{
+    public class NoRateLimiter : IRateLimiter
+    {
+        public void Execute(Action action)
+        {
+            action();
+        }
+    }
+}

Tapeti.Cmd/RateLimiter/SpreadRateLimiter.cs

@@ -0,0 +1,30 @@
+using System;
+using System.Threading;
+
+namespace Tapeti.Cmd.RateLimiter
+{
+    public class SpreadRateLimiter : IRateLimiter
+    {
+        private readonly TimeSpan delay;
+        private DateTime lastExecute = DateTime.MinValue;
+
+        public SpreadRateLimiter(int amount, TimeSpan perTimespan)
+        {
+            delay = TimeSpan.FromMilliseconds(perTimespan.TotalMilliseconds / amount);
+        }
+
+
+        public void Execute(Action action)
+        {
+            // Very simple implementation; the time between actions must be at least the delay.
+            // This prevents bursts followed by nothing which are common with normal rate limiter implementations.
+            var remainingWaitTime = delay - (DateTime.Now - lastExecute);
+
+            if (remainingWaitTime.TotalMilliseconds > 0)
+                Thread.Sleep(remainingWaitTime);
+
+            action();
+            lastExecute = DateTime.Now;
+        }
+    }
+}

Tapeti.Cmd/build-release.bat

@@ -1,3 +1,4 @@
+@Echo Off
 mkdir publish
 
 REM Executable is generated using self-contained=true, which is just a wrapper for "dotnet Tapeti.Cmd.dll".

Tapeti.sln

@@ -57,7 +57,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{62002327
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tapeti.Cmd", "Tapeti.Cmd\Tapeti.Cmd.csproj", "{C8728BFC-7F97-41BC-956B-690A57B634EC}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "06-StatelessRequestResponse", "06-StatelessRequestResponse\06-StatelessRequestResponse.csproj", "{152227AA-3165-4550-8997-6EA80C84516E}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "06-StatelessRequestResponse", "Examples\06-StatelessRequestResponse\06-StatelessRequestResponse.csproj", "{152227AA-3165-4550-8997-6EA80C84516E}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution

docs/tapeticmd.rst

@@ -80,6 +80,10 @@ Read messages from disk as previously exported and publish them to a queue.
 -s <method>, --serialization <method>
   The method used to serialize the message for import or export. Valid options: SingleFileJSON, EasyNetQHosepipe. Defaults to SingleFileJSON. See Serialization methods below for more information.
 
+--maxrate <messages per second>
+  The maximum amount of messages per second to import.
+
+
 Either input, message or pipe is required.
 
 Example:
@@ -121,6 +125,8 @@ Reads messages from a queue and publishes them to another queue, optionally to a
 --targetpassword <password>
   Password used to connect to the target RabbitMQ server. Defaults to the source password.
 
+--maxrate <messages per second>
+  The maximum amount of messages per second to shovel.
 
 
 Example: