From a0359b1e43c996554c068f73fe78e443df5b2114 Mon Sep 17 00:00:00 2001 From: Alexey Golub Date: Sat, 3 Nov 2018 22:53:20 +0200 Subject: [PATCH] Split output file into multiple partitions (#116) --- .../Verbs/ExportChatVerb.cs | 2 +- .../Verbs/Options/ExportChatOptions.cs | 3 ++ DiscordChatExporter.Core/Models/Extensions.cs | 42 +++++++++++++++++ .../Services/ExportService.TemplateModel.cs | 20 ++++---- .../Services/ExportService.cs | 47 ++++++++++++++++++- .../Services/IExportService.cs | 3 +- .../Services/ISettingsService.cs | 1 + .../Services/SettingsService.cs | 1 + .../Messages/StartExportMessage.cs | 5 +- .../ViewModels/ExportSetupViewModel.cs | 13 ++++- .../ViewModels/IExportSetupViewModel.cs | 1 + .../ViewModels/MainViewModel.cs | 7 +-- .../Views/ExportSetupDialog.xaml | 7 +++ 13 files changed, 133 insertions(+), 19 deletions(-) diff --git a/DiscordChatExporter.Cli/Verbs/ExportChatVerb.cs b/DiscordChatExporter.Cli/Verbs/ExportChatVerb.cs index 4bbf0cd2..653d5017 100644 --- a/DiscordChatExporter.Cli/Verbs/ExportChatVerb.cs +++ b/DiscordChatExporter.Cli/Verbs/ExportChatVerb.cs @@ -42,7 +42,7 @@ namespace DiscordChatExporter.Cli.Verbs } // Export - exportService.ExportChatLog(chatLog, filePath, Options.ExportFormat); + exportService.ExportChatLog(chatLog, filePath, Options.ExportFormat, Options.PartitionLimit); // Print result Console.WriteLine($"Exported chat to [{filePath}]"); diff --git a/DiscordChatExporter.Cli/Verbs/Options/ExportChatOptions.cs b/DiscordChatExporter.Cli/Verbs/Options/ExportChatOptions.cs index 870fd2ae..bb403663 100644 --- a/DiscordChatExporter.Cli/Verbs/Options/ExportChatOptions.cs +++ b/DiscordChatExporter.Cli/Verbs/Options/ExportChatOptions.cs @@ -22,6 +22,9 @@ namespace DiscordChatExporter.Cli.Verbs.Options [Option("before", Default = null, HelpText = "Limit to messages sent before this date.")] public DateTime? Before { get; set; } + [Option('p', "partition", Default = null, HelpText = "Split output into partitions limited to this number of messages.")] + public int? PartitionLimit { get; set; } + [Option("dateformat", Default = null, HelpText = "Date format used in output.")] public string DateFormat { get; set; } diff --git a/DiscordChatExporter.Core/Models/Extensions.cs b/DiscordChatExporter.Core/Models/Extensions.cs index 90817b9a..70efb2fa 100644 --- a/DiscordChatExporter.Core/Models/Extensions.cs +++ b/DiscordChatExporter.Core/Models/Extensions.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using System.Linq; namespace DiscordChatExporter.Core.Models { @@ -31,5 +33,45 @@ namespace DiscordChatExporter.Core.Models throw new ArgumentOutOfRangeException(nameof(format)); } + + public static IReadOnlyList SplitIntoPartitions(this ChatLog chatLog, int partitionLimit) + { + // If chat log has fewer messages than the limit - just return chat log in a list + if (chatLog.Messages.Count <= partitionLimit) + return new[] {chatLog}; + + var result = new List(); + + // Loop through messages + var buffer = new List(); + foreach (var message in chatLog.Messages) + { + // Add message to buffer + buffer.Add(message); + + // If reached the limit - split and reset buffer + if (buffer.Count >= partitionLimit) + { + // Add to result + var chatLogPartition = new ChatLog(chatLog.Guild, chatLog.Channel, chatLog.From, chatLog.To, buffer, + chatLog.Mentionables); + result.Add(chatLogPartition); + + // Reset the buffer instead of clearing to avoid mutations on existing references + buffer = new List(); + } + } + + // Add what's remaining in buffer + if (buffer.Any()) + { + // Add to result + var chatLogPartition = new ChatLog(chatLog.Guild, chatLog.Channel, chatLog.From, chatLog.To, buffer, + chatLog.Mentionables); + result.Add(chatLogPartition); + } + + return result; + } } } \ No newline at end of file diff --git a/DiscordChatExporter.Core/Services/ExportService.TemplateModel.cs b/DiscordChatExporter.Core/Services/ExportService.TemplateModel.cs index 4a21bc9f..0246ccd2 100644 --- a/DiscordChatExporter.Core/Services/ExportService.TemplateModel.cs +++ b/DiscordChatExporter.Core/Services/ExportService.TemplateModel.cs @@ -32,10 +32,10 @@ namespace DiscordChatExporter.Core.Services private IEnumerable GroupMessages(IEnumerable messages) { // Group adjacent messages by timestamp and author - var groupBuffer = new List(); + var buffer = new List(); foreach (var message in messages) { - var groupFirst = groupBuffer.FirstOrDefault(); + var groupFirst = buffer.FirstOrDefault(); // Group break condition var breakCondition = @@ -44,27 +44,29 @@ namespace DiscordChatExporter.Core.Services message.Author.Id != groupFirst.Author.Id || (message.Timestamp - groupFirst.Timestamp).TotalHours > 1 || message.Timestamp.Hour != groupFirst.Timestamp.Hour || - groupBuffer.Count >= _messageGroupLimit + buffer.Count >= _messageGroupLimit ); // If condition is true - flush buffer if (breakCondition) { - var group = new MessageGroup(groupFirst.Author, groupFirst.Timestamp, groupBuffer.ToArray()); - groupBuffer.Clear(); + var group = new MessageGroup(groupFirst.Author, groupFirst.Timestamp, buffer); + + // Reset the buffer instead of clearing to avoid mutations on existing references + buffer = new List(); yield return group; } // Add message to buffer - groupBuffer.Add(message); + buffer.Add(message); } // Add what's remaining in buffer - if (groupBuffer.Any()) + if (buffer.Any()) { - var groupFirst = groupBuffer.First(); - var group = new MessageGroup(groupFirst.Author, groupFirst.Timestamp, groupBuffer.ToArray()); + var groupFirst = buffer.First(); + var group = new MessageGroup(groupFirst.Author, groupFirst.Timestamp, buffer); yield return group; } diff --git a/DiscordChatExporter.Core/Services/ExportService.cs b/DiscordChatExporter.Core/Services/ExportService.cs index 69e92076..1e1a352e 100644 --- a/DiscordChatExporter.Core/Services/ExportService.cs +++ b/DiscordChatExporter.Core/Services/ExportService.cs @@ -1,4 +1,5 @@ -using System.IO; +using System.Collections.Generic; +using System.IO; using DiscordChatExporter.Core.Models; using Scriban; using Scriban.Runtime; @@ -15,7 +16,7 @@ namespace DiscordChatExporter.Core.Services _settingsService = settingsService; } - public void ExportChatLog(ChatLog chatLog, string filePath, ExportFormat format) + private void ExportChatLogSingle(ChatLog chatLog, string filePath, ExportFormat format) { // Create template loader var loader = new TemplateLoader(); @@ -55,5 +56,47 @@ namespace DiscordChatExporter.Core.Services template.Render(context); } } + + private void ExportChatLogPartitions(IReadOnlyList partitions, string filePath, ExportFormat format) + { + // Split file path into components + var dirPath = Path.GetDirectoryName(filePath); + var fileNameWithoutExt = Path.GetFileNameWithoutExtension(filePath); + var fileExt = Path.GetExtension(filePath); + + // Export each partition separately + var partitionNumber = 1; + foreach (var partition in partitions) + { + // Compose new file name + var partitionFilePath = $"{fileNameWithoutExt}-{partitionNumber}{fileExt}"; + + // Compose full file path + if (dirPath.IsNotBlank()) + partitionFilePath = Path.Combine(dirPath, partitionFilePath); + + // Export + ExportChatLogSingle(partition, partitionFilePath, format); + + // Increment partition number + partitionNumber++; + } + } + + public void ExportChatLog(ChatLog chatLog, string filePath, ExportFormat format, + int? partitionLimit = null) + { + // If partitioning is disabled or there are fewer messages in chat log than the limit - process it without partitioning + if (partitionLimit == null || chatLog.Messages.Count <= partitionLimit) + { + ExportChatLogSingle(chatLog, filePath, format); + } + // Otherwise split into partitions and export separately + else + { + var partitions = chatLog.SplitIntoPartitions(partitionLimit.Value); + ExportChatLogPartitions(partitions, filePath, format); + } + } } } \ No newline at end of file diff --git a/DiscordChatExporter.Core/Services/IExportService.cs b/DiscordChatExporter.Core/Services/IExportService.cs index 92105e88..f3d07382 100644 --- a/DiscordChatExporter.Core/Services/IExportService.cs +++ b/DiscordChatExporter.Core/Services/IExportService.cs @@ -4,6 +4,7 @@ namespace DiscordChatExporter.Core.Services { public interface IExportService { - void ExportChatLog(ChatLog chatLog, string filePath, ExportFormat format); + void ExportChatLog(ChatLog chatLog, string filePath, ExportFormat format, + int? partitionLimit = null); } } \ No newline at end of file diff --git a/DiscordChatExporter.Core/Services/ISettingsService.cs b/DiscordChatExporter.Core/Services/ISettingsService.cs index 02514662..ec50c89f 100644 --- a/DiscordChatExporter.Core/Services/ISettingsService.cs +++ b/DiscordChatExporter.Core/Services/ISettingsService.cs @@ -11,6 +11,7 @@ namespace DiscordChatExporter.Core.Services AuthToken LastToken { get; set; } ExportFormat LastExportFormat { get; set; } + int? LastPartitionLimit { get; set; } void Load(); void Save(); diff --git a/DiscordChatExporter.Core/Services/SettingsService.cs b/DiscordChatExporter.Core/Services/SettingsService.cs index 12f9e01b..11762796 100644 --- a/DiscordChatExporter.Core/Services/SettingsService.cs +++ b/DiscordChatExporter.Core/Services/SettingsService.cs @@ -12,6 +12,7 @@ namespace DiscordChatExporter.Core.Services public AuthToken LastToken { get; set; } public ExportFormat LastExportFormat { get; set; } = ExportFormat.HtmlDark; + public int? LastPartitionLimit { get; set; } public SettingsService() { diff --git a/DiscordChatExporter.Gui/Messages/StartExportMessage.cs b/DiscordChatExporter.Gui/Messages/StartExportMessage.cs index bfda1c64..2a115be9 100644 --- a/DiscordChatExporter.Gui/Messages/StartExportMessage.cs +++ b/DiscordChatExporter.Gui/Messages/StartExportMessage.cs @@ -15,14 +15,17 @@ namespace DiscordChatExporter.Gui.Messages public DateTime? To { get; } + public int? PartitionLimit { get; } + public StartExportMessage(Channel channel, string filePath, ExportFormat format, - DateTime? from, DateTime? to) + DateTime? from, DateTime? to, int? partitionLimit) { Channel = channel; FilePath = filePath; Format = format; From = from; To = to; + PartitionLimit = partitionLimit; } } } \ No newline at end of file diff --git a/DiscordChatExporter.Gui/ViewModels/ExportSetupViewModel.cs b/DiscordChatExporter.Gui/ViewModels/ExportSetupViewModel.cs index 84698218..df56550b 100644 --- a/DiscordChatExporter.Gui/ViewModels/ExportSetupViewModel.cs +++ b/DiscordChatExporter.Gui/ViewModels/ExportSetupViewModel.cs @@ -19,6 +19,7 @@ namespace DiscordChatExporter.Gui.ViewModels private ExportFormat _format; private DateTime? _from; private DateTime? _to; + private int? _partitionLimit; public Guild Guild { get; private set; } @@ -63,6 +64,12 @@ namespace DiscordChatExporter.Gui.ViewModels set => Set(ref _to, value); } + public int? PartitionLimit + { + get => _partitionLimit; + set => Set(ref _partitionLimit, value); + } + // Commands public RelayCommand ExportCommand { get; } @@ -83,13 +90,15 @@ namespace DiscordChatExporter.Gui.ViewModels .Replace(Path.GetInvalidFileNameChars(), '_'); From = null; To = null; + PartitionLimit = _settingsService.LastPartitionLimit; }); } private void Export() { - // Save format + // Persist preferences _settingsService.LastExportFormat = SelectedFormat; + _settingsService.LastPartitionLimit = PartitionLimit; // Clamp 'from' and 'to' values if (From > To) @@ -98,7 +107,7 @@ namespace DiscordChatExporter.Gui.ViewModels To = From; // Start export - MessengerInstance.Send(new StartExportMessage(Channel, FilePath, SelectedFormat, From, To)); + MessengerInstance.Send(new StartExportMessage(Channel, FilePath, SelectedFormat, From, To, PartitionLimit)); } } } \ No newline at end of file diff --git a/DiscordChatExporter.Gui/ViewModels/IExportSetupViewModel.cs b/DiscordChatExporter.Gui/ViewModels/IExportSetupViewModel.cs index a0d58c7a..7179da40 100644 --- a/DiscordChatExporter.Gui/ViewModels/IExportSetupViewModel.cs +++ b/DiscordChatExporter.Gui/ViewModels/IExportSetupViewModel.cs @@ -14,6 +14,7 @@ namespace DiscordChatExporter.Gui.ViewModels ExportFormat SelectedFormat { get; set; } DateTime? From { get; set; } DateTime? To { get; set; } + int? PartitionLimit { get; set; } RelayCommand ExportCommand { get; } } diff --git a/DiscordChatExporter.Gui/ViewModels/MainViewModel.cs b/DiscordChatExporter.Gui/ViewModels/MainViewModel.cs index 5b93a69a..c138ecde 100644 --- a/DiscordChatExporter.Gui/ViewModels/MainViewModel.cs +++ b/DiscordChatExporter.Gui/ViewModels/MainViewModel.cs @@ -129,7 +129,7 @@ namespace DiscordChatExporter.Gui.ViewModels // Messages MessengerInstance.Register(this, - m => Export(m.Channel, m.FilePath, m.Format, m.From, m.To)); + m => Export(m.Channel, m.FilePath, m.Format, m.From, m.To, m.PartitionLimit)); } private async void ViewLoaded() @@ -239,7 +239,8 @@ namespace DiscordChatExporter.Gui.ViewModels MessengerInstance.Send(new ShowExportSetupMessage(SelectedGuild, channel)); } - private async void Export(Channel channel, string filePath, ExportFormat format, DateTime? from, DateTime? to) + private async void Export(Channel channel, string filePath, ExportFormat format, + DateTime? from, DateTime? to, int? partitionLimit) { IsBusy = true; @@ -258,7 +259,7 @@ namespace DiscordChatExporter.Gui.ViewModels var chatLog = await _dataService.GetChatLogAsync(token, guild, channel, from, to, progressHandler); // Export - _exportService.ExportChatLog(chatLog, filePath, format); + _exportService.ExportChatLog(chatLog, filePath, format, partitionLimit); // Notify completion MessengerInstance.Send(new ShowNotificationMessage("Export complete")); diff --git a/DiscordChatExporter.Gui/Views/ExportSetupDialog.xaml b/DiscordChatExporter.Gui/Views/ExportSetupDialog.xaml index e24f2ca2..734114ff 100644 --- a/DiscordChatExporter.Gui/Views/ExportSetupDialog.xaml +++ b/DiscordChatExporter.Gui/Views/ExportSetupDialog.xaml @@ -55,6 +55,13 @@ SelectedDate="{Binding To}" /> + + +