mirrors/DiscordChatExporter
1
0
Fork 0
mirror of https://github.com/Tyrrrz/DiscordChatExporter.git synced 2025-06-08 02:14:42 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Stop redundantly downloading media when re-exporting (#395)

Browse source
This commit is contained in:
Andrew Kolos 2020-10-23 09:38:15 -04:00 • committed by GitHub
parent 949c9d3f1e
commit 520e023aff
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 70 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

4
DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs
View file

@ -29,6 +29,9 @@ namespace DiscordChatExporter.Cli.Commands.Base
[CommandOption("media", Description = "Download referenced media content.")]
public bool ShouldDownloadMedia { get; set; }
[CommandOption("reuse-media", Description = "If the media folder already exists, reuse media inside it to skip downloads.")]
public bool ShouldReuseMedia { get; set; }
[CommandOption("dateformat", Description = "Date format used in output.")]
public string DateFormat { get; set; } = "dd-MMM-yy hh:mm tt";
@ -48,6 +51,7 @@ namespace DiscordChatExporter.Cli.Commands.Base
Before,
PartitionLimit,
ShouldDownloadMedia,
ShouldReuseMedia,
DateFormat
);

1
DiscordChatExporter.Cli/Commands/Base/ExportMultipleCommandBase.cs
View file

@ -48,6 +48,7 @@ namespace DiscordChatExporter.Cli.Commands.Base
Before,
PartitionLimit,
ShouldDownloadMedia,
ShouldReuseMedia,
DateFormat
);

2
DiscordChatExporter.Domain/Exporting/ExportContext.cs
View file

@ -34,7 +34,7 @@ namespace DiscordChatExporter.Domain.Exporting
Channels = channels;
Roles = roles;
_mediaDownloader = new MediaDownloader(request.OutputMediaDirPath);
_mediaDownloader = new MediaDownloader(request.OutputMediaDirPath, request.ShouldReuseMedia);
}
public string FormatDate(DateTimeOffset date) => Request.DateFormat switch

4
DiscordChatExporter.Domain/Exporting/ExportRequest.cs
View file

@ -30,6 +30,8 @@ namespace DiscordChatExporter.Domain.Exporting
public bool ShouldDownloadMedia { get; }
public bool ShouldReuseMedia { get; }
public string DateFormat { get; }
public ExportRequest(
@ -41,6 +43,7 @@ namespace DiscordChatExporter.Domain.Exporting
DateTimeOffset? before,
int? partitionLimit,
bool shouldDownloadMedia,
bool shouldReuseMedia,
string dateFormat)
{
Guild = guild;
@ -51,6 +54,7 @@ namespace DiscordChatExporter.Domain.Exporting
Before = before;
PartitionLimit = partitionLimit;
ShouldDownloadMedia = shouldDownloadMedia;
ShouldReuseMedia = shouldReuseMedia;
DateFormat = dateFormat;
OutputBaseFilePath = GetOutputBaseFilePath(

41
DiscordChatExporter.Domain/Exporting/MediaDownloader.cs
View file

@ -3,6 +3,8 @@ using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Internal;
@ -16,13 +18,16 @@ namespace DiscordChatExporter.Domain.Exporting
{
private readonly HttpClient _httpClient = Singleton.HttpClient;
private readonly string _workingDirPath;
private readonly bool _reuseMedia;
private readonly AsyncRetryPolicy _httpRequestPolicy;
private readonly Dictionary<string, string> _pathMap = new Dictionary<string, string>();
public MediaDownloader(string workingDirPath)
public MediaDownloader(string workingDirPath, bool reuseMedia)
{
_workingDirPath = workingDirPath;
_reuseMedia = reuseMedia;
_httpRequestPolicy = Policy
.Handle<IOException>()
@ -37,11 +42,18 @@ namespace DiscordChatExporter.Domain.Exporting
return cachedFilePath;
var fileName = GetFileNameFromUrl(url);
var filePath = PathEx.MakeUniqueFilePath(Path.Combine(_workingDirPath, fileName));
var filePath = Path.Combine(_workingDirPath, fileName);
Directory.CreateDirectory(_workingDirPath);
if (!_reuseMedia)
{
filePath = PathEx.MakeUniqueFilePath(filePath);
}
await _httpClient.DownloadAsync(url, filePath);
if (!_reuseMedia || !File.Exists(filePath))
{
Directory.CreateDirectory(_workingDirPath);
await _httpClient.DownloadAsync(url, filePath);
}
return _pathMap[url] = filePath;
});
@ -50,6 +62,23 @@ namespace DiscordChatExporter.Domain.Exporting
internal partial class MediaDownloader
{
private static int URL_HASH_LENGTH = 5;
private static string HashUrl(string url)
{
using (var md5 = MD5.Create())
{
var inputBytes = Encoding.UTF8.GetBytes(url);
var hashBytes = md5.ComputeHash(inputBytes);
var hashBuilder = new StringBuilder();
for (int i = 0; i < hashBytes.Length; i++)
{
hashBuilder.Append(hashBytes[i].ToString("X2"));
}
return hashBuilder.ToString().Truncate(URL_HASH_LENGTH);
}
}
private static string GetRandomFileName() => Guid.NewGuid().ToString().Replace("-", "").Substring(0, 16);
private static string GetFileNameFromUrl(string url)
@ -57,10 +86,10 @@ namespace DiscordChatExporter.Domain.Exporting
var originalFileName = Regex.Match(url, @".+/([^?]*)").Groups[1].Value;
var fileName = !string.IsNullOrWhiteSpace(originalFileName)
? $"{Path.GetFileNameWithoutExtension(originalFileName).Truncate(50)}{Path.GetExtension(originalFileName)}"
? $"{Path.GetFileNameWithoutExtension(originalFileName).Truncate(42)}-({HashUrl(url)}){Path.GetExtension(originalFileName)}"
: GetRandomFileName();
return PathEx.EscapePath(fileName);
}
}
}
}

2
DiscordChatExporter.Gui/Services/SettingsService.cs
View file

@ -16,6 +16,8 @@ namespace DiscordChatExporter.Gui.Services
public int ParallelLimit { get; set; } = 1;
public bool ShouldReuseMedia { get; set; } = false;
public AuthToken? LastToken { get; set; }
public ExportFormat LastExportFormat { get; set; } = ExportFormat.HtmlDark;

6
DiscordChatExporter.Gui/ViewModels/Dialogs/SettingsViewModel.cs
View file

@ -37,6 +37,12 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs
get => _settingsService.ParallelLimit;
set => _settingsService.ParallelLimit = value.Clamp(1, 10);
}
public bool ShouldReuseMedia
{
get => _settingsService.ShouldReuseMedia;
set => _settingsService.ShouldReuseMedia = value;
}
public SettingsViewModel(SettingsService settingsService)
{

1
DiscordChatExporter.Gui/ViewModels/RootViewModel.cs
View file

@ -204,6 +204,7 @@ namespace DiscordChatExporter.Gui.ViewModels
dialog.Before,
dialog.PartitionLimit,
dialog.ShouldDownloadMedia,
_settingsService.ShouldReuseMedia,
_settingsService.DateFormat
);

17
DiscordChatExporter.Gui/Views/Dialogs/SettingsView.xaml
View file

@ -7,7 +7,7 @@
xmlns:materialDesign="http://materialdesigninxaml.net/winfx/xaml/themes"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:s="https://github.com/canton7/Stylet"
Width="300"
Width="310"
d:DataContext="{d:DesignInstance Type=dialogs:SettingsViewModel}"
Style="{DynamicResource MaterialDesignRoot}"
mc:Ignorable="d">
@ -65,6 +65,21 @@
IsChecked="{Binding IsTokenPersisted}" />
</DockPanel>
<!-- Reuse Media -->
<DockPanel
Background="Transparent"
LastChildFill="False"
ToolTip="If the media folder already exists, reuse media inside it to skip downloads">
<TextBlock
Margin="16,8"
DockPanel.Dock="Left"
Text="Reuse previously downloaded media" />
<ToggleButton
Margin="16,8"
DockPanel.Dock="Right"
IsChecked="{Binding ShouldReuseMedia}" />
</DockPanel>
<!-- Date format -->
<TextBox
Margin="16,8"