Stop redundantly downloading media when re-exporting (#395)

DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs

@@ -29,6 +29,9 @@ namespace DiscordChatExporter.Cli.Commands.Base
         [CommandOption("media", Description = "Download referenced media content.")]
         public bool ShouldDownloadMedia { get; set; }
 
+        [CommandOption("reuse-media", Description = "If the media folder already exists, reuse media inside it to skip downloads.")]
+        public bool ShouldReuseMedia { get; set; }
+
         [CommandOption("dateformat", Description = "Date format used in output.")]
         public string DateFormat { get; set; } = "dd-MMM-yy hh:mm tt";
 
@@ -48,6 +51,7 @@ namespace DiscordChatExporter.Cli.Commands.Base
                 Before,
                 PartitionLimit,
                 ShouldDownloadMedia,
+                ShouldReuseMedia,
                 DateFormat
             );
 

DiscordChatExporter.Cli/Commands/Base/ExportMultipleCommandBase.cs

@@ -48,6 +48,7 @@ namespace DiscordChatExporter.Cli.Commands.Base
                         Before,
                         PartitionLimit,
                         ShouldDownloadMedia,
+                        ShouldReuseMedia,
                         DateFormat
                     );
 

DiscordChatExporter.Domain/Exporting/ExportContext.cs

@@ -34,7 +34,7 @@ namespace DiscordChatExporter.Domain.Exporting
             Channels = channels;
             Roles = roles;
 
-            _mediaDownloader = new MediaDownloader(request.OutputMediaDirPath);
+            _mediaDownloader = new MediaDownloader(request.OutputMediaDirPath, request.ShouldReuseMedia);
         }
 
         public string FormatDate(DateTimeOffset date) => Request.DateFormat switch

DiscordChatExporter.Domain/Exporting/ExportRequest.cs

@@ -30,6 +30,8 @@ namespace DiscordChatExporter.Domain.Exporting
 
         public bool ShouldDownloadMedia { get; }
 
+        public bool ShouldReuseMedia { get; }
+
         public string DateFormat { get; }
 
         public ExportRequest(
@@ -41,6 +43,7 @@ namespace DiscordChatExporter.Domain.Exporting
             DateTimeOffset? before,
             int? partitionLimit,
             bool shouldDownloadMedia,
+            bool shouldReuseMedia,
             string dateFormat)
         {
             Guild = guild;
@@ -51,6 +54,7 @@ namespace DiscordChatExporter.Domain.Exporting
             Before = before;
             PartitionLimit = partitionLimit;
             ShouldDownloadMedia = shouldDownloadMedia;
+            ShouldReuseMedia = shouldReuseMedia;
             DateFormat = dateFormat;
 
             OutputBaseFilePath = GetOutputBaseFilePath(

DiscordChatExporter.Domain/Exporting/MediaDownloader.cs

@@ -3,6 +3,8 @@ using System.Collections.Generic;
 using System.IO;
 using System.Net;
 using System.Net.Http;
+using System.Security.Cryptography;
+using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using DiscordChatExporter.Domain.Internal;
@@ -16,13 +18,16 @@ namespace DiscordChatExporter.Domain.Exporting
     {
         private readonly HttpClient _httpClient = Singleton.HttpClient;
         private readonly string _workingDirPath;
+
+        private readonly bool _reuseMedia;
         private readonly AsyncRetryPolicy _httpRequestPolicy;
 
         private readonly Dictionary<string, string> _pathMap = new Dictionary<string, string>();
 
-        public MediaDownloader(string workingDirPath)
+        public MediaDownloader(string workingDirPath, bool reuseMedia)
         {
             _workingDirPath = workingDirPath;
+            _reuseMedia = reuseMedia;
 
             _httpRequestPolicy = Policy
                 .Handle<IOException>()
@@ -37,11 +42,18 @@ namespace DiscordChatExporter.Domain.Exporting
                     return cachedFilePath;
 
                 var fileName = GetFileNameFromUrl(url);
-                var filePath = PathEx.MakeUniqueFilePath(Path.Combine(_workingDirPath, fileName));
+                var filePath = Path.Combine(_workingDirPath, fileName);
 
+                if (!_reuseMedia)
+                {
+                    filePath = PathEx.MakeUniqueFilePath(filePath);
+                }
+
+                if (!_reuseMedia || !File.Exists(filePath))
+                {
                     Directory.CreateDirectory(_workingDirPath);
-
                     await _httpClient.DownloadAsync(url, filePath);
+                }
 
                 return _pathMap[url] = filePath;
             });
@@ -50,6 +62,23 @@ namespace DiscordChatExporter.Domain.Exporting
 
     internal partial class MediaDownloader
     {
+        private static int URL_HASH_LENGTH = 5;
+        private static string HashUrl(string url)
+        {
+            using (var md5 = MD5.Create())
+            {
+                var inputBytes = Encoding.UTF8.GetBytes(url);
+                var hashBytes = md5.ComputeHash(inputBytes);
+
+                var hashBuilder = new StringBuilder();
+                for (int i = 0; i < hashBytes.Length; i++)
+                {
+                    hashBuilder.Append(hashBytes[i].ToString("X2"));
+                }
+                return hashBuilder.ToString().Truncate(URL_HASH_LENGTH);
+            }
+        }
+
         private static string GetRandomFileName() => Guid.NewGuid().ToString().Replace("-", "").Substring(0, 16);
 
         private static string GetFileNameFromUrl(string url)
@@ -57,7 +86,7 @@ namespace DiscordChatExporter.Domain.Exporting
             var originalFileName = Regex.Match(url, @".+/([^?]*)").Groups[1].Value;
 
             var fileName = !string.IsNullOrWhiteSpace(originalFileName)
-                ? $"{Path.GetFileNameWithoutExtension(originalFileName).Truncate(50)}{Path.GetExtension(originalFileName)}"
+                ? $"{Path.GetFileNameWithoutExtension(originalFileName).Truncate(42)}-({HashUrl(url)}){Path.GetExtension(originalFileName)}"
                 : GetRandomFileName();
 
             return PathEx.EscapePath(fileName);

DiscordChatExporter.Gui/Services/SettingsService.cs

@@ -16,6 +16,8 @@ namespace DiscordChatExporter.Gui.Services
 
         public int ParallelLimit { get; set; } = 1;
 
+        public bool ShouldReuseMedia { get; set; } = false;
+
         public AuthToken? LastToken { get; set; }
 
         public ExportFormat LastExportFormat { get; set; } = ExportFormat.HtmlDark;

DiscordChatExporter.Gui/ViewModels/Dialogs/SettingsViewModel.cs

@@ -38,6 +38,12 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs
             set => _settingsService.ParallelLimit = value.Clamp(1, 10);
         }
         
+        public bool ShouldReuseMedia
+        {
+            get => _settingsService.ShouldReuseMedia;
+            set => _settingsService.ShouldReuseMedia = value;
+        }
+
         public SettingsViewModel(SettingsService settingsService)
         {
             _settingsService = settingsService;

DiscordChatExporter.Gui/ViewModels/RootViewModel.cs

@@ -204,6 +204,7 @@ namespace DiscordChatExporter.Gui.ViewModels
                         dialog.Before,
                         dialog.PartitionLimit,
                         dialog.ShouldDownloadMedia,
+                        _settingsService.ShouldReuseMedia,
                         _settingsService.DateFormat
                     );
 

DiscordChatExporter.Gui/Views/Dialogs/SettingsView.xaml

@@ -7,7 +7,7 @@
     xmlns:materialDesign="http://materialdesigninxaml.net/winfx/xaml/themes"
     xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
     xmlns:s="https://github.com/canton7/Stylet"
-    Width="300"
+    Width="310"
     d:DataContext="{d:DesignInstance Type=dialogs:SettingsViewModel}"
     Style="{DynamicResource MaterialDesignRoot}"
     mc:Ignorable="d">
@@ -65,6 +65,21 @@
                 IsChecked="{Binding IsTokenPersisted}" />
         </DockPanel>
 
+        <!--  Reuse Media  -->
+        <DockPanel
+            Background="Transparent"
+            LastChildFill="False"
+            ToolTip="If the media folder already exists, reuse media inside it to skip downloads">
+            <TextBlock
+                Margin="16,8"
+                DockPanel.Dock="Left"
+                Text="Reuse previously downloaded media" />
+            <ToggleButton
+                Margin="16,8"
+                DockPanel.Dock="Right"
+                IsChecked="{Binding ShouldReuseMedia}" />
+        </DockPanel>
+
         <!--  Date format  -->
         <TextBox
             Margin="16,8"