GitHub-Mirror/Tapeti
1
0
Fork 0
mirror of synced 2024-11-05 02:59:16 +00:00
Code Issues Projects Releases Wiki Activity

Implemented a rate limiting option for Tapeti.Cmd

Browse Source 
Moved example 06 to the "examples" sub-folder
This commit is contained in:
Mark van Renswoude 2020-07-03 15:51:41 +02:00
parent 266c4475a6
commit e31d638a11
14 changed files with 234 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
06-StatelessRequestResponse/06-StatelessRequestResponse.csproj
View File

@ -1,20 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp2.1</TargetFramework>
<RootNamespace>_06_StatelessRequestResponse</RootNamespace>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="SimpleInjector" Version="4.9.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Examples\ExampleLib\ExampleLib.csproj" />
<ProjectReference Include="..\Examples\Messaging.TapetiExample\Messaging.TapetiExample.csproj" />
<ProjectReference Include="..\Tapeti.DataAnnotations\Tapeti.DataAnnotations.csproj" />
<ProjectReference Include="..\Tapeti.SimpleInjector\Tapeti.SimpleInjector.csproj" />
</ItemGroup>
</Project>

20
Examples/06-StatelessRequestResponse/06-StatelessRequestResponse.csproj Normal file
View File

@ -0,0 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp2.1</TargetFramework>
<RootNamespace>_06_StatelessRequestResponse</RootNamespace>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="SimpleInjector" Version="4.9.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\ExampleLib\ExampleLib.csproj" />
<ProjectReference Include="..\Messaging.TapetiExample\Messaging.TapetiExample.csproj" />
<ProjectReference Include="..\..\Tapeti.DataAnnotations\Tapeti.DataAnnotations.csproj" />
<ProjectReference Include="..\..\Tapeti.SimpleInjector\Tapeti.SimpleInjector.csproj" />
</ItemGroup>
</Project>

56
06-StatelessRequestResponse/ExampleMessageController.cs → Examples/06-StatelessRequestResponse/ExampleMessageController.cs
View File

@ -1,28 +1,28 @@
using System;
using ExampleLib;
using Messaging.TapetiExample;
using Tapeti.Annotations;
namespace _06_StatelessRequestResponse
{
[MessageController]
[DynamicQueue("tapeti.example.06")]
public class ExampleMessageController
{
private readonly IExampleState exampleState;
public ExampleMessageController(IExampleState exampleState)
{
this.exampleState = exampleState;
}
[ResponseHandler]
public void HandleQuoteResponse(QuoteResponseMessage message)
{
Console.WriteLine("Received response: " + message.Quote);
exampleState.Done();
}
}
}
using System;
using ExampleLib;
using Messaging.TapetiExample;
using Tapeti.Annotations;
namespace _06_StatelessRequestResponse
{
[MessageController]
[DynamicQueue("tapeti.example.06")]
public class ExampleMessageController
{
private readonly IExampleState exampleState;
public ExampleMessageController(IExampleState exampleState)
{
this.exampleState = exampleState;
}
[ResponseHandler]
public void HandleQuoteResponse(QuoteResponseMessage message)
{
Console.WriteLine("Received response: " + message.Quote);
exampleState.Done();
}
}
}

102
06-StatelessRequestResponse/Program.cs → Examples/06-StatelessRequestResponse/Program.cs
View File

@ -1,51 +1,51 @@
using System;
using System.Threading.Tasks;
using ExampleLib;
using Messaging.TapetiExample;
using SimpleInjector;
using Tapeti;
using Tapeti.DataAnnotations;
using Tapeti.Default;
using Tapeti.SimpleInjector;
namespace _06_StatelessRequestResponse
{
public class Program
{
public static void Main(string[] args)
{
var container = new Container();
var dependencyResolver = new SimpleInjectorDependencyResolver(container);
container.Register<ILogger, ConsoleLogger>();
var helper = new ExampleConsoleApp(dependencyResolver);
helper.Run(MainAsync);
}
internal static async Task MainAsync(IDependencyResolver dependencyResolver, Func<Task> waitForDone)
{
var config = new TapetiConfig(dependencyResolver)
.WithDataAnnotations()
.RegisterAllControllers()
.Build();
using (var connection = new TapetiConnection(config))
{
await connection.Subscribe();
var publisher = dependencyResolver.Resolve<IPublisher>();
await publisher.PublishRequest<ExampleMessageController, QuoteRequestMessage, QuoteResponseMessage>(
new QuoteRequestMessage
{
Amount = 1
},
c => c.HandleQuoteResponse);
await waitForDone();
}
}
}
}
using System;
using System.Threading.Tasks;
using ExampleLib;
using Messaging.TapetiExample;
using SimpleInjector;
using Tapeti;
using Tapeti.DataAnnotations;
using Tapeti.Default;
using Tapeti.SimpleInjector;
namespace _06_StatelessRequestResponse
{
public class Program
{
public static void Main(string[] args)
{
var container = new Container();
var dependencyResolver = new SimpleInjectorDependencyResolver(container);
container.Register<ILogger, ConsoleLogger>();
var helper = new ExampleConsoleApp(dependencyResolver);
helper.Run(MainAsync);
}
internal static async Task MainAsync(IDependencyResolver dependencyResolver, Func<Task> waitForDone)
{
var config = new TapetiConfig(dependencyResolver)
.WithDataAnnotations()
.RegisterAllControllers()
.Build();
using (var connection = new TapetiConnection(config))
{
await connection.Subscribe();
var publisher = dependencyResolver.Resolve<IPublisher>();
await publisher.PublishRequest<ExampleMessageController, QuoteRequestMessage, QuoteResponseMessage>(
new QuoteRequestMessage
{
Amount = 1
},
c => c.HandleQuoteResponse);
await waitForDone();
}
}
}
}

76
06-StatelessRequestResponse/ReceivingMessageController.cs → Examples/06-StatelessRequestResponse/ReceivingMessageController.cs
View File

@ -1,38 +1,38 @@
using Messaging.TapetiExample;
using Tapeti.Annotations;
namespace _06_StatelessRequestResponse
{
[MessageController]
[DynamicQueue("tapeti.example.06.receiver")]
public class ReceivingMessageController
{
// No publisher required, responses can simply be returned
public QuoteResponseMessage HandleQuoteRequest(QuoteRequestMessage message)
{
string quote;
switch (message.Amount)
{
case 1:
// Well, they asked for it... :-)
quote = "'";
break;
case 2:
quote = "\"";
break;
default:
// We have to return a response.
quote = null;
break;
}
return new QuoteResponseMessage
{
Quote = quote
};
}
}
}
using Messaging.TapetiExample;
using Tapeti.Annotations;
namespace _06_StatelessRequestResponse
{
[MessageController]
[DynamicQueue("tapeti.example.06.receiver")]
public class ReceivingMessageController
{
// No publisher required, responses can simply be returned
public QuoteResponseMessage HandleQuoteRequest(QuoteRequestMessage message)
{
string quote;
switch (message.Amount)
{
case 1:
// Well, they asked for it... :-)
quote = "'";
break;
case 2:
quote = "\"";
break;
default:
// We have to return a response.
quote = null;
break;
}
return new QuoteResponseMessage
{
Quote = quote
};
}
}
}

14
Tapeti.Cmd/Commands/ImportCommand.cs
View File

@ -1,4 +1,5 @@
using RabbitMQ.Client;
using Tapeti.Cmd.RateLimiter;
using Tapeti.Cmd.Serialization;
namespace Tapeti.Cmd.Commands
@ -10,17 +11,20 @@ namespace Tapeti.Cmd.Commands
public bool DirectToQueue { get; set; }
public int Execute(IModel channel)
public int Execute(IModel channel, IRateLimiter rateLimiter)
{
var messageCount = 0;
foreach (var message in MessageSerializer.Deserialize())
{
var exchange = DirectToQueue ? "" : message.Exchange;
var routingKey = DirectToQueue ? message.Queue : message.RoutingKey;
rateLimiter.Execute(() =>
{
var exchange = DirectToQueue ? "" : message.Exchange;
var routingKey = DirectToQueue ? message.Queue : message.RoutingKey;
channel.BasicPublish(exchange, routingKey, message.Properties, message.Body);
messageCount++;
channel.BasicPublish(exchange, routingKey, message.Properties, message.Body);
messageCount++;
});
}
return messageCount;

18
Tapeti.Cmd/Commands/ShovelCommand.cs
View File

@ -1,5 +1,6 @@
using RabbitMQ.Client;
using Tapeti.Cmd.RateLimiter;
namespace Tapeti.Cmd.Commands
{
public class ShovelCommand
@ -10,7 +11,7 @@ namespace Tapeti.Cmd.Commands
public int? MaxCount { get; set; }
public int Execute(IModel sourceChannel, IModel targetChannel)
public int Execute(IModel sourceChannel, IModel targetChannel, IRateLimiter rateLimiter)
{
var messageCount = 0;
@ -22,13 +23,14 @@ namespace Tapeti.Cmd.Commands
break;
targetChannel.BasicPublish("", TargetQueueName, result.BasicProperties, result.Body);
rateLimiter.Execute(() =>
{
targetChannel.BasicPublish("", TargetQueueName, result.BasicProperties, result.Body);
messageCount++;
messageCount++;
if (RemoveMessages)
sourceChannel.BasicAck(result.DeliveryTag, false);
if (RemoveMessages)
sourceChannel.BasicAck(result.DeliveryTag, false);
});
}
return messageCount;

23
Tapeti.Cmd/Program.cs
View File

@ -4,10 +4,10 @@ using System.Diagnostics;
using System.IO;
using System.Text;
using CommandLine;
using CommandLine.Text;
using RabbitMQ.Client;
using RabbitMQ.Client.Framing;
using Tapeti.Cmd.Commands;
using Tapeti.Cmd.RateLimiter;
using Tapeti.Cmd.Serialization;
namespace Tapeti.Cmd
@ -79,6 +79,9 @@ namespace Tapeti.Cmd
[Option('e', "exchange", HelpText = "If specified publishes to the originating exchange using the original routing key. By default these are ignored and the message is published directly to the originating queue.")]
public bool PublishToExchange { get; set; }
[Option("maxrate", HelpText = "The maximum amount of messages per second to import.")]
public int? MaxRate { get; set; }
}
@ -111,6 +114,9 @@ namespace Tapeti.Cmd
[Option("targetpassword", HelpText = "Password used to connect to the target RabbitMQ server. Defaults to the source password.")]
public string TargetPassword { get; set; }
[Option("maxrate", HelpText = "The maximum amount of messages per second to shovel.")]
public int? MaxRate { get; set; }
}
@ -236,6 +242,15 @@ namespace Tapeti.Cmd
}
private static IRateLimiter GetRateLimiter(int? maxRate)
{
if (maxRate.GetValueOrDefault() <= 0)
return new NoRateLimiter();
return new SpreadRateLimiter(maxRate.Value, TimeSpan.FromSeconds(1));
}
private static void RunExport(ExportOptions options)
{
int messageCount;
@ -271,7 +286,7 @@ namespace Tapeti.Cmd
MessageSerializer = messageSerializer,
DirectToQueue = !options.PublishToExchange
}.Execute(channel);
}.Execute(channel, GetRateLimiter(options.MaxRate));
}
Console.WriteLine($"{messageCount} message{(messageCount != 1 ? "s" : "")} published.");
@ -299,11 +314,11 @@ namespace Tapeti.Cmd
using (var targetConnection = GetTargetConnection(options))
using (var targetChannel = targetConnection.CreateModel())
{
messageCount = shovelCommand.Execute(sourceChannel, targetChannel);
messageCount = shovelCommand.Execute(sourceChannel, targetChannel, GetRateLimiter(options.MaxRate));
}
}
else
messageCount = shovelCommand.Execute(sourceChannel, sourceChannel);
messageCount = shovelCommand.Execute(sourceChannel, sourceChannel, GetRateLimiter(options.MaxRate));
}
Console.WriteLine($"{messageCount} message{(messageCount != 1 ? "s" : "")} shoveled.");

9
Tapeti.Cmd/RateLimiter/IRateLimiter.cs Normal file
View File

@ -0,0 +1,9 @@
using System;
namespace Tapeti.Cmd.RateLimiter
{
public interface IRateLimiter
{
void Execute(Action action);
}
}

12
Tapeti.Cmd/RateLimiter/NoRateLimiter.cs Normal file
View File

@ -0,0 +1,12 @@
using System;
namespace Tapeti.Cmd.RateLimiter
{
public class NoRateLimiter : IRateLimiter
{
public void Execute(Action action)
{
action();
}
}
}

30
Tapeti.Cmd/RateLimiter/SpreadRateLimiter.cs Normal file
View File

@ -0,0 +1,30 @@
using System;
using System.Threading;
namespace Tapeti.Cmd.RateLimiter
{
public class SpreadRateLimiter : IRateLimiter
{
private readonly TimeSpan delay;
private DateTime lastExecute = DateTime.MinValue;
public SpreadRateLimiter(int amount, TimeSpan perTimespan)
{
delay = TimeSpan.FromMilliseconds(perTimespan.TotalMilliseconds / amount);
}
public void Execute(Action action)
{
// Very simple implementation; the time between actions must be at least the delay.
// This prevents bursts followed by nothing which are common with normal rate limiter implementations.
var remainingWaitTime = delay - (DateTime.Now - lastExecute);
if (remainingWaitTime.TotalMilliseconds > 0)
Thread.Sleep(remainingWaitTime);
action();
lastExecute = DateTime.Now;
}
}
}

1
Tapeti.Cmd/build-release.bat
View File

@ -1,3 +1,4 @@
@Echo Off
mkdir publish
REM Executable is generated using self-contained=true, which is just a wrapper for "dotnet Tapeti.Cmd.dll".

2
Tapeti.sln
View File

@ -57,7 +57,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{62002327
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tapeti.Cmd", "Tapeti.Cmd\Tapeti.Cmd.csproj", "{C8728BFC-7F97-41BC-956B-690A57B634EC}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "06-StatelessRequestResponse", "06-StatelessRequestResponse\06-StatelessRequestResponse.csproj", "{152227AA-3165-4550-8997-6EA80C84516E}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "06-StatelessRequestResponse", "Examples\06-StatelessRequestResponse\06-StatelessRequestResponse.csproj", "{152227AA-3165-4550-8997-6EA80C84516E}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution

6
docs/tapeticmd.rst
View File

@ -80,6 +80,10 @@ Read messages from disk as previously exported and publish them to a queue.
-s <method>, --serialization <method>
The method used to serialize the message for import or export. Valid options: SingleFileJSON, EasyNetQHosepipe. Defaults to SingleFileJSON. See Serialization methods below for more information.
--maxrate <messages per second>
The maximum amount of messages per second to import.
Either input, message or pipe is required.
Example:
@ -121,6 +125,8 @@ Reads messages from a queue and publishes them to another queue, optionally to a
--targetpassword <password>
Password used to connect to the target RabbitMQ server. Defaults to the source password.
--maxrate <messages per second>
The maximum amount of messages per second to shovel.
Example: