Split output file into multiple partitions (#116)

DiscordChatExporter.Cli/Verbs/ExportChatVerb.cs

@@ -42,7 +42,7 @@ namespace DiscordChatExporter.Cli.Verbs
             }
 
             // Export
-            exportService.ExportChatLog(chatLog, filePath, Options.ExportFormat);
+            exportService.ExportChatLog(chatLog, filePath, Options.ExportFormat, Options.PartitionLimit);
 
             // Print result
             Console.WriteLine($"Exported chat to [{filePath}]");

DiscordChatExporter.Cli/Verbs/Options/ExportChatOptions.cs

@@ -22,6 +22,9 @@ namespace DiscordChatExporter.Cli.Verbs.Options
         [Option("before", Default = null, HelpText = "Limit to messages sent before this date.")]
         public DateTime? Before { get; set; }
 
+        [Option('p', "partition", Default = null, HelpText = "Split output into partitions limited to this number of messages.")]
+        public int? PartitionLimit { get; set; }
+
         [Option("dateformat", Default = null, HelpText = "Date format used in output.")]
         public string DateFormat { get; set; }
 

DiscordChatExporter.Core/Models/Extensions.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Collections.Generic;
+using System.Linq;
 
 namespace DiscordChatExporter.Core.Models
 {
@@ -31,5 +33,45 @@ namespace DiscordChatExporter.Core.Models
 
             throw new ArgumentOutOfRangeException(nameof(format));
         }
+
+        public static IReadOnlyList<ChatLog> SplitIntoPartitions(this ChatLog chatLog, int partitionLimit)
+        {
+            // If chat log has fewer messages than the limit - just return chat log in a list
+            if (chatLog.Messages.Count <= partitionLimit)
+                return new[] {chatLog};
+
+            var result = new List<ChatLog>();
+
+            // Loop through messages
+            var buffer = new List<Message>();
+            foreach (var message in chatLog.Messages)
+            {
+                // Add message to buffer
+                buffer.Add(message);
+
+                // If reached the limit - split and reset buffer
+                if (buffer.Count >= partitionLimit)
+                {
+                    // Add to result
+                    var chatLogPartition = new ChatLog(chatLog.Guild, chatLog.Channel, chatLog.From, chatLog.To, buffer,
+                        chatLog.Mentionables);
+                    result.Add(chatLogPartition);
+
+                    // Reset the buffer instead of clearing to avoid mutations on existing references
+                    buffer = new List<Message>();
+                }
+            }
+
+            // Add what's remaining in buffer
+            if (buffer.Any())
+            {
+                // Add to result
+                var chatLogPartition = new ChatLog(chatLog.Guild, chatLog.Channel, chatLog.From, chatLog.To, buffer,
+                    chatLog.Mentionables);
+                result.Add(chatLogPartition);
+            }
+
+            return result;
+        }
     }
 }

DiscordChatExporter.Core/Services/ExportService.TemplateModel.cs

@@ -32,10 +32,10 @@ namespace DiscordChatExporter.Core.Services
             private IEnumerable<MessageGroup> GroupMessages(IEnumerable<Message> messages)
             {
                 // Group adjacent messages by timestamp and author
-                var groupBuffer = new List<Message>();
+                var buffer = new List<Message>();
                 foreach (var message in messages)
                 {
-                    var groupFirst = groupBuffer.FirstOrDefault();
+                    var groupFirst = buffer.FirstOrDefault();
 
                     // Group break condition
                     var breakCondition =
@@ -44,27 +44,29 @@ namespace DiscordChatExporter.Core.Services
                             message.Author.Id != groupFirst.Author.Id ||
                             (message.Timestamp - groupFirst.Timestamp).TotalHours > 1 ||
                             message.Timestamp.Hour != groupFirst.Timestamp.Hour ||
-                            groupBuffer.Count >= _messageGroupLimit
+                            buffer.Count >= _messageGroupLimit
                         );
 
                     // If condition is true - flush buffer
                     if (breakCondition)
                     {
-                        var group = new MessageGroup(groupFirst.Author, groupFirst.Timestamp, groupBuffer.ToArray());
-                        groupBuffer.Clear();
+                        var group = new MessageGroup(groupFirst.Author, groupFirst.Timestamp, buffer);
+
+                        // Reset the buffer instead of clearing to avoid mutations on existing references
+                        buffer = new List<Message>();
 
                         yield return group;
                     }
 
                     // Add message to buffer
-                    groupBuffer.Add(message);
+                    buffer.Add(message);
                 }
 
                 // Add what's remaining in buffer
-                if (groupBuffer.Any())
+                if (buffer.Any())
                 {
-                    var groupFirst = groupBuffer.First();
-                    var group = new MessageGroup(groupFirst.Author, groupFirst.Timestamp, groupBuffer.ToArray());
+                    var groupFirst = buffer.First();
+                    var group = new MessageGroup(groupFirst.Author, groupFirst.Timestamp, buffer);
 
                     yield return group;
                 }

DiscordChatExporter.Core/Services/ExportService.cs

@@ -1,4 +1,5 @@
-using System.IO;
+using System.Collections.Generic;
+using System.IO;
 using DiscordChatExporter.Core.Models;
 using Scriban;
 using Scriban.Runtime;
@@ -15,7 +16,7 @@ namespace DiscordChatExporter.Core.Services
             _settingsService = settingsService;
         }
 
-        public void ExportChatLog(ChatLog chatLog, string filePath, ExportFormat format)
+        private void ExportChatLogSingle(ChatLog chatLog, string filePath, ExportFormat format)
         {
             // Create template loader
             var loader = new TemplateLoader();
@@ -55,5 +56,47 @@ namespace DiscordChatExporter.Core.Services
                 template.Render(context);
             }
         }
+
+        private void ExportChatLogPartitions(IReadOnlyList<ChatLog> partitions, string filePath, ExportFormat format)
+        {
+            // Split file path into components
+            var dirPath = Path.GetDirectoryName(filePath);
+            var fileNameWithoutExt = Path.GetFileNameWithoutExtension(filePath);
+            var fileExt = Path.GetExtension(filePath);
+
+            // Export each partition separately
+            var partitionNumber = 1;
+            foreach (var partition in partitions)
+            {
+                // Compose new file name
+                var partitionFilePath = $"{fileNameWithoutExt}-{partitionNumber}{fileExt}";
+
+                // Compose full file path
+                if (dirPath.IsNotBlank())
+                    partitionFilePath = Path.Combine(dirPath, partitionFilePath);
+
+                // Export
+                ExportChatLogSingle(partition, partitionFilePath, format);
+
+                // Increment partition number
+                partitionNumber++;
+            }
+        }
+
+        public void ExportChatLog(ChatLog chatLog, string filePath, ExportFormat format,
+            int? partitionLimit = null)
+        {
+            // If partitioning is disabled or there are fewer messages in chat log than the limit - process it without partitioning
+            if (partitionLimit == null || chatLog.Messages.Count <= partitionLimit)
+            {
+                ExportChatLogSingle(chatLog, filePath, format);
+            }
+            // Otherwise split into partitions and export separately
+            else
+            {
+                var partitions = chatLog.SplitIntoPartitions(partitionLimit.Value);
+                ExportChatLogPartitions(partitions, filePath, format);
+            }
+        }
     }
 }

DiscordChatExporter.Core/Services/IExportService.cs

@@ -4,6 +4,7 @@ namespace DiscordChatExporter.Core.Services
 {
     public interface IExportService
     {
-        void ExportChatLog(ChatLog chatLog, string filePath, ExportFormat format);
+        void ExportChatLog(ChatLog chatLog, string filePath, ExportFormat format,
+            int? partitionLimit = null);
     }
 }

DiscordChatExporter.Core/Services/ISettingsService.cs

@@ -11,6 +11,7 @@ namespace DiscordChatExporter.Core.Services
 
         AuthToken LastToken { get; set; }
         ExportFormat LastExportFormat { get; set; }
+        int? LastPartitionLimit { get; set; }
 
         void Load();
         void Save();

DiscordChatExporter.Core/Services/SettingsService.cs

@@ -12,6 +12,7 @@ namespace DiscordChatExporter.Core.Services
 
         public AuthToken LastToken { get; set; }
         public ExportFormat LastExportFormat { get; set; } = ExportFormat.HtmlDark;
+        public int? LastPartitionLimit { get; set; }
 
         public SettingsService()
         {

DiscordChatExporter.Gui/Messages/StartExportMessage.cs

@@ -15,14 +15,17 @@ namespace DiscordChatExporter.Gui.Messages
 
         public DateTime? To { get; }
 
+        public int? PartitionLimit { get; }
+
         public StartExportMessage(Channel channel, string filePath, ExportFormat format,
-            DateTime? from, DateTime? to)
+            DateTime? from, DateTime? to, int? partitionLimit)
         {
             Channel = channel;
             FilePath = filePath;
             Format = format;
             From = from;
             To = to;
+            PartitionLimit = partitionLimit;
         }
     }
 }

DiscordChatExporter.Gui/ViewModels/ExportSetupViewModel.cs

@@ -19,6 +19,7 @@ namespace DiscordChatExporter.Gui.ViewModels
         private ExportFormat _format;
         private DateTime? _from;
         private DateTime? _to;
+        private int? _partitionLimit;
 
         public Guild Guild { get; private set; }
 
@@ -63,6 +64,12 @@ namespace DiscordChatExporter.Gui.ViewModels
             set => Set(ref _to, value);
         }
 
+        public int? PartitionLimit
+        {
+            get => _partitionLimit;
+            set => Set(ref _partitionLimit, value);
+        }
+
         // Commands
         public RelayCommand ExportCommand { get; }
 
@@ -83,13 +90,15 @@ namespace DiscordChatExporter.Gui.ViewModels
                     .Replace(Path.GetInvalidFileNameChars(), '_');
                 From = null;
                 To = null;
+                PartitionLimit = _settingsService.LastPartitionLimit;
             });
         }
 
         private void Export()
         {
-            // Save format
+            // Persist preferences
             _settingsService.LastExportFormat = SelectedFormat;
+            _settingsService.LastPartitionLimit = PartitionLimit;
 
             // Clamp 'from' and 'to' values
             if (From > To)
@@ -98,7 +107,7 @@ namespace DiscordChatExporter.Gui.ViewModels
                 To = From;
 
             // Start export
-            MessengerInstance.Send(new StartExportMessage(Channel, FilePath, SelectedFormat, From, To));
+            MessengerInstance.Send(new StartExportMessage(Channel, FilePath, SelectedFormat, From, To, PartitionLimit));
         }
     }
 }

DiscordChatExporter.Gui/ViewModels/IExportSetupViewModel.cs

@@ -14,6 +14,7 @@ namespace DiscordChatExporter.Gui.ViewModels
         ExportFormat SelectedFormat { get; set; }
         DateTime? From { get; set; }
         DateTime? To { get; set; }
+        int? PartitionLimit { get; set; }
 
         RelayCommand ExportCommand { get; }
     }

DiscordChatExporter.Gui/ViewModels/MainViewModel.cs

@@ -129,7 +129,7 @@ namespace DiscordChatExporter.Gui.ViewModels
 
             // Messages
             MessengerInstance.Register<StartExportMessage>(this,
-                m => Export(m.Channel, m.FilePath, m.Format, m.From, m.To));
+                m => Export(m.Channel, m.FilePath, m.Format, m.From, m.To, m.PartitionLimit));
         }
 
         private async void ViewLoaded()
@@ -239,7 +239,8 @@ namespace DiscordChatExporter.Gui.ViewModels
             MessengerInstance.Send(new ShowExportSetupMessage(SelectedGuild, channel));
         }
 
-        private async void Export(Channel channel, string filePath, ExportFormat format, DateTime? from, DateTime? to)
+        private async void Export(Channel channel, string filePath, ExportFormat format, 
+            DateTime? from, DateTime? to, int? partitionLimit)
         {
             IsBusy = true;
 
@@ -258,7 +259,7 @@ namespace DiscordChatExporter.Gui.ViewModels
                 var chatLog = await _dataService.GetChatLogAsync(token, guild, channel, from, to, progressHandler);
 
                 // Export
-                _exportService.ExportChatLog(chatLog, filePath, format);
+                _exportService.ExportChatLog(chatLog, filePath, format, partitionLimit);
 
                 // Notify completion
                 MessengerInstance.Send(new ShowNotificationMessage("Export complete"));

DiscordChatExporter.Gui/Views/ExportSetupDialog.xaml

@@ -55,6 +55,13 @@
                 SelectedDate="{Binding To}" />
         </Grid>
 
+        <!--  Partitioning  -->
+        <TextBox
+            Margin="16,8,16,8"
+            materialDesign:HintAssist.Hint="Messages per partition (optional)"
+            materialDesign:HintAssist.IsFloating="True"
+            Text="{Binding PartitionLimit}" />
+
         <!--  Buttons  -->
         <StackPanel HorizontalAlignment="Right" Orientation="Horizontal">
             <Button