Add parallel exporting

Closes #264

DiscordChatExporter.Cli/Commands/ExportCommandBase.cs

@@ -53,6 +53,7 @@ namespace DiscordChatExporter.Cli.Commands
                 After, Before, progress);
 
             console.Output.WriteLine();
+            console.Output.WriteLine("Done.");
         }
 
         protected async ValueTask ExportAsync(IConsole console, Channel channel)

DiscordChatExporter.Cli/Commands/ExportDirectMessagesCommand.cs

@@ -1,16 +1,13 @@
 using System.Linq;
-using System.Net;
 using System.Threading.Tasks;
 using CliFx;
 using CliFx.Attributes;
-using DiscordChatExporter.Core.Models.Exceptions;
 using DiscordChatExporter.Core.Services;
-using DiscordChatExporter.Core.Services.Exceptions;
 
 namespace DiscordChatExporter.Cli.Commands
 {
     [Command("exportdm", Description = "Export all direct message channels.")]
-    public class ExportDirectMessagesCommand : ExportCommandBase
+    public class ExportDirectMessagesCommand : ExportMultipleCommandBase
     {
         public ExportDirectMessagesCommand(SettingsService settingsService, DataService dataService, ExportService exportService)
             : base(settingsService, dataService, exportService)
@@ -19,32 +16,10 @@ namespace DiscordChatExporter.Cli.Commands
 
         public override async ValueTask ExecuteAsync(IConsole console)
         {
-            // Get channels
+            var directMessageChannels = await DataService.GetDirectMessageChannelsAsync(Token);
-            var channels = await DataService.GetDirectMessageChannelsAsync(Token);
+            var channels = directMessageChannels.OrderBy(c => c.Name).ToArray();
 
-            // Order channels
+            await ExportMultipleAsync(console, channels);
-            channels = channels.OrderBy(c => c.Name).ToArray();
-
-            // Loop through channels
-            foreach (var channel in channels)
-            {
-                try
-                {
-                    await ExportAsync(console, channel);
-                }
-                catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.Forbidden)
-                {
-                    console.Error.WriteLine("You don't have access to this channel.");
-                }
-                catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
-                {
-                    console.Error.WriteLine("This channel doesn't exist.");
-                }
-                catch (DomainException ex)
-                {
-                    console.Error.WriteLine(ex.Message);
-                }
-            }
         }
     }
 }

DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs

@@ -1,17 +1,14 @@
 using System.Linq;
-using System.Net;
 using System.Threading.Tasks;
 using CliFx;
 using CliFx.Attributes;
 using DiscordChatExporter.Core.Models;
-using DiscordChatExporter.Core.Models.Exceptions;
 using DiscordChatExporter.Core.Services;
-using DiscordChatExporter.Core.Services.Exceptions;
 
 namespace DiscordChatExporter.Cli.Commands
 {
     [Command("exportguild", Description = "Export all channels within specified guild.")]
-    public class ExportGuildCommand : ExportCommandBase
+    public class ExportGuildCommand : ExportMultipleCommandBase
     {
         [CommandOption("guild", 'g', IsRequired = true, Description = "Guild ID.")]
         public string GuildId { get; set; } = "";
@@ -23,32 +20,14 @@ namespace DiscordChatExporter.Cli.Commands
 
         public override async ValueTask ExecuteAsync(IConsole console)
         {
-            // Get channels
+            var guildChannels = await DataService.GetGuildChannelsAsync(Token, GuildId);
-            var channels = await DataService.GetGuildChannelsAsync(Token, GuildId);
 
-            // Filter and order channels
+            var channels = guildChannels
-            channels = channels.Where(c => c.Type.IsExportable()).OrderBy(c => c.Name).ToArray();
+                .Where(c => c.Type.IsExportable())
+                .OrderBy(c => c.Name)
+                .ToArray();
 
-            // Loop through channels
+            await ExportMultipleAsync(console, channels);
-            foreach (var channel in channels)
-            {
-                try
-                {
-                    await ExportAsync(console, channel);
-                }
-                catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.Forbidden)
-                {
-                    console.Error.WriteLine("You don't have access to this channel.");
-                }
-                catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
-                {
-                    console.Error.WriteLine("This channel doesn't exist.");
-                }
-                catch (DomainException ex)
-                {
-                    console.Error.WriteLine(ex.Message);
-                }
-            }
         }
     }
 }

DiscordChatExporter.Cli/Commands/ExportMultipleCommandBase.cs

@@ -0,0 +1,92 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Threading;
+using System.Threading.Tasks;
+using CliFx;
+using CliFx.Attributes;
+using CliFx.Utilities;
+using DiscordChatExporter.Core.Models;
+using DiscordChatExporter.Core.Models.Exceptions;
+using DiscordChatExporter.Core.Services;
+using DiscordChatExporter.Core.Services.Exceptions;
+using Gress;
+using Tyrrrz.Extensions;
+
+namespace DiscordChatExporter.Cli.Commands
+{
+    public abstract class ExportMultipleCommandBase : ExportCommandBase
+    {
+        [CommandOption("parallel", Description = "Export this number of separate channels in parallel.")]
+        public int ParallelLimit { get; set; } = 1;
+
+        protected ExportMultipleCommandBase(SettingsService settingsService, DataService dataService, ExportService exportService)
+            : base(settingsService, dataService, exportService)
+        {
+        }
+
+        protected async ValueTask ExportMultipleAsync(IConsole console, IReadOnlyList<Channel> channels)
+        {
+            // This uses a separate route from ExportCommandBase because the progress ticker is not thread-safe
+            // Ugly code ahead. Will need to refactor.
+
+            if (!string.IsNullOrWhiteSpace(DateFormat))
+                SettingsService.DateFormat = DateFormat;
+
+            // Progress
+            console.Output.Write($"Exporting {channels.Count} channels... ");
+            var ticker = console.CreateProgressTicker();
+
+            // TODO: refactor this after improving Gress
+            var progressManager = new ProgressManager();
+            progressManager.PropertyChanged += (sender, args) => ticker.Report(progressManager.Progress);
+
+            var operations = progressManager.CreateOperations(channels.Count);
+
+            // Export channels
+            using var semaphore = new SemaphoreSlim(ParallelLimit.ClampMin(1));
+
+            var errors = new List<string>();
+
+            await Task.WhenAll(channels.Select(async (channel, i) =>
+            {
+                var operation = operations[i];
+                await semaphore.WaitAsync();
+
+                var guild = await DataService.GetGuildAsync(Token, channel.GuildId);
+
+                try
+                {
+                    await ExportService.ExportChatLogAsync(Token, guild, channel,
+                        OutputPath, ExportFormat, PartitionLimit,
+                        After, Before, operation);
+                }
+                catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.Forbidden)
+                {
+                    errors.Add("You don't have access to this channel.");
+                }
+                catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
+                {
+                    errors.Add("This channel doesn't exist.");
+                }
+                catch (DomainException ex)
+                {
+                    errors.Add(ex.Message);
+                }
+                finally
+                {
+                    semaphore.Release();
+                    operation.Dispose();
+                }
+            }));
+
+            ticker.Report(1);
+            console.Output.WriteLine();
+
+            foreach (var error in errors)
+                console.Error.WriteLine(error);
+
+            console.Output.WriteLine("Done.");
+        }
+    }
+}

DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs

@@ -20,13 +20,13 @@ namespace DiscordChatExporter.Cli.Commands
 
         public override async ValueTask ExecuteAsync(IConsole console)
         {
-            // Get channels
+            var guildChannels = await DataService.GetGuildChannelsAsync(Token, GuildId);
-            var channels = await DataService.GetGuildChannelsAsync(Token, GuildId);
 
-            // Filter and order channels
+            var channels = guildChannels
-            channels = channels.Where(c => c.Type.IsExportable()).OrderBy(c => c.Name).ToArray();
+                .Where(c => c.Type.IsExportable())
+                .OrderBy(c => c.Name)
+                .ToArray();
 
-            // Print result
             foreach (var channel in channels)
                 console.Output.WriteLine($"{channel.Id} | {channel.Name}");
         }

DiscordChatExporter.Cli/Commands/GetDirectMessageChannelsCommand.cs

@@ -16,13 +16,9 @@ namespace DiscordChatExporter.Cli.Commands
 
         public override async ValueTask ExecuteAsync(IConsole console)
         {
-            // Get channels
+            var directMessageChannels = await DataService.GetDirectMessageChannelsAsync(Token);
-            var channels = await DataService.GetDirectMessageChannelsAsync(Token);
+            var channels = directMessageChannels.OrderBy(c => c.Name).ToArray();
 
-            // Order channels
-            channels = channels.OrderBy(c => c.Name).ToArray();
-
-            // Print result
             foreach (var channel in channels)
                 console.Output.WriteLine($"{channel.Id} | {channel.Name}");
         }

DiscordChatExporter.Cli/Commands/GetGuildsCommand.cs

@@ -16,14 +16,9 @@ namespace DiscordChatExporter.Cli.Commands
 
         public override async ValueTask ExecuteAsync(IConsole console)
         {
-            // Get guilds
             var guilds = await DataService.GetUserGuildsAsync(Token);
 
-            // Order guilds
+            foreach (var guild in guilds.OrderBy(g => g.Name))
-            guilds = guilds.OrderBy(g => g.Name).ToArray();
-
-            // Print result
-            foreach (var guild in guilds)
                 console.Output.WriteLine($"{guild.Id} | {guild.Name}");
         }
     }

DiscordChatExporter.Cli/DiscordChatExporter.Cli.csproj

@@ -8,6 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="CliFx" Version="1.0.0" />
+    <PackageReference Include="Gress" Version="1.1.1" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="3.1.1" />
     <PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" />
   </ItemGroup>

DiscordChatExporter.Core.Services/SettingsService.cs

@@ -11,6 +11,8 @@ namespace DiscordChatExporter.Core.Services
 
         public bool IsTokenPersisted { get; set; } = true;
 
+        public int ParallelLimit { get; set; } = 1;
+
         public AuthToken? LastToken { get; set; }
 
         public ExportFormat LastExportFormat { get; set; } = ExportFormat.HtmlDark;

DiscordChatExporter.Gui/ViewModels/Dialogs/SettingsViewModel.cs

@@ -1,5 +1,6 @@
 using DiscordChatExporter.Core.Services;
 using DiscordChatExporter.Gui.ViewModels.Framework;
+using Tyrrrz.Extensions;
 
 namespace DiscordChatExporter.Gui.ViewModels.Dialogs
 {
@@ -25,6 +26,12 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs
             set => _settingsService.IsTokenPersisted = value;
         }
 
+        public int ParallelLimit
+        {
+            get => _settingsService.ParallelLimit;
+            set => _settingsService.ParallelLimit = value.Clamp(1, 10);
+        }
+
         public SettingsViewModel(SettingsService settingsService)
         {
             _settingsService = settingsService;

DiscordChatExporter.Gui/ViewModels/RootViewModel.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
+using System.Threading;
 using System.Threading.Tasks;
 using DiscordChatExporter.Core.Models;
 using DiscordChatExporter.Core.Models.Exceptions;
@@ -260,10 +261,13 @@ namespace DiscordChatExporter.Gui.ViewModels
 
             // Export channels
             var successfulExportCount = 0;
-            for (var i = 0; i < dialog.Channels.Count; i++)
+            using var semaphore = new SemaphoreSlim(_settingsService.ParallelLimit.ClampMin(1));
+
+            await Task.WhenAll(dialog.Channels.Select(async (channel, i) =>
             {
                 var operation = operations[i];
-                var channel = dialog.Channels[i];
+
+                await semaphore.WaitAsync();
 
                 try
                 {
@@ -288,8 +292,9 @@ namespace DiscordChatExporter.Gui.ViewModels
                 finally
                 {
                     operation.Dispose();
+                    semaphore.Release();
                 }
-            }
+            }));
 
             // Notify of overall completion
             if (successfulExportCount > 0)

DiscordChatExporter.Gui/Views/Dialogs/SettingsView.xaml

@@ -56,6 +56,21 @@
                 IsChecked="{Binding IsTokenPersisted}" />
         </DockPanel>
 
+        <!--  Parallel limit  -->
+        <StackPanel Background="Transparent" ToolTip="How many channels can be exported at the same time">
+            <TextBlock Margin="16,8">
+                <Run Text="Parallel limit:" />
+                <Run Foreground="{DynamicResource PrimaryTextBrush}" Text="{Binding ParallelLimit, Mode=OneWay}" />
+            </TextBlock>
+            <Slider
+                Margin="16,8"
+                IsSnapToTickEnabled="True"
+                Maximum="10"
+                Minimum="1"
+                TickFrequency="1"
+                Value="{Binding ParallelLimit}" />
+        </StackPanel>
+
         <!--  Save button  -->
         <Button
             Margin="8"