mirrors/DiscordChatExporter
1
0
Fork 0
mirror of https://github.com/Tyrrrz/DiscordChatExporter.git synced 2025-05-22 10:55:15 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add managed cancellation support

Browse source 
Closes #716
This commit is contained in:
Tyrrrz 2021-10-07 17:06:05 +03:00
parent 2f3e165988
commit 21d89afa70
21 changed files with 274 additions and 147 deletions
Download patch file Download diff file Expand all files Collapse all files

12
DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs
View file

@ -15,7 +15,6 @@ using DiscordChatExporter.Core.Exporting;
using DiscordChatExporter.Core.Exporting.Filtering;
using DiscordChatExporter.Core.Exporting.Partitioning;
using DiscordChatExporter.Core.Utils.Extensions;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Cli.Commands.Base
{
@ -56,6 +55,8 @@ namespace DiscordChatExporter.Cli.Commands.Base
protected async ValueTask ExecuteAsync(IConsole console, IReadOnlyList<Channel> channels)
{
var cancellationToken = console.RegisterCancellationHandler();
if (ShouldReuseMedia && !ShouldDownloadMedia)
{
throw new CommandException("Option --reuse-media cannot be used without --media.");
@ -73,7 +74,7 @@ namespace DiscordChatExporter.Cli.Commands.Base
{
await progressContext.StartTaskAsync($"{channel.Category} / {channel.Name}", async progress =>
{
var guild = await Discord.GetGuildAsync(channel.GuildId);
var guild = await Discord.GetGuildAsync(channel.GuildId, cancellationToken);
var request = new ExportRequest(
guild,
@ -89,14 +90,14 @@ namespace DiscordChatExporter.Cli.Commands.Base
DateFormat
);
await Exporter.ExportChannelAsync(request, progress);
await Exporter.ExportChannelAsync(request, progress, cancellationToken);
});
}
catch (DiscordChatExporterException ex) when (!ex.IsFatal)
{
errors[channel] = ex.Message;
}
}, ParallelLimit.ClampMin(1));
}, Math.Max(ParallelLimit, 1), cancellationToken);
});
// Print result
@ -140,11 +141,12 @@ namespace DiscordChatExporter.Cli.Commands.Base
protected async ValueTask ExecuteAsync(IConsole console, IReadOnlyList<Snowflake> channelIds)
{
var cancellationToken = console.RegisterCancellationHandler();
var channels = new List<Channel>();
foreach (var channelId in channelIds)
{
var channel = await Discord.GetChannelAsync(channelId);
var channel = await Discord.GetChannelAsync(channelId, cancellationToken);
channels.Add(channel);
}

5
DiscordChatExporter.Cli/Commands/ExportAllCommand.cs
View file

@ -15,16 +15,17 @@ namespace DiscordChatExporter.Cli.Commands
public override async ValueTask ExecuteAsync(IConsole console)
{
var cancellationToken = console.RegisterCancellationHandler();
var channels = new List<Channel>();
await console.Output.WriteLineAsync("Fetching channels...");
await foreach (var guild in Discord.GetUserGuildsAsync())
await foreach (var guild in Discord.GetUserGuildsAsync(cancellationToken))
{
// Skip DMs if instructed to
if (!IncludeDirectMessages && guild.Id == Guild.DirectMessages.Id)
continue;
await foreach (var channel in Discord.GetGuildChannelsAsync(guild.Id))
await foreach (var channel in Discord.GetGuildChannelsAsync(guild.Id, cancellationToken))
{
// Skip non-text channels
if (!channel.IsTextChannel)

4
DiscordChatExporter.Cli/Commands/ExportDirectMessagesCommand.cs
View file

@ -13,8 +13,10 @@ namespace DiscordChatExporter.Cli.Commands
{
public override async ValueTask ExecuteAsync(IConsole console)
{
var cancellationToken = console.RegisterCancellationHandler();
await console.Output.WriteLineAsync("Fetching channels...");
var channels = await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id);
var channels = await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken);
var textChannels = channels.Where(c => c.IsTextChannel).ToArray();
await base.ExecuteAsync(console, textChannels);

4
DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs
View file

@ -16,8 +16,10 @@ namespace DiscordChatExporter.Cli.Commands
public override async ValueTask ExecuteAsync(IConsole console)
{
var cancellationToken = console.RegisterCancellationHandler();
await console.Output.WriteLineAsync("Fetching channels...");
var channels = await Discord.GetGuildChannelsAsync(GuildId);
var channels = await Discord.GetGuildChannelsAsync(GuildId, cancellationToken);
var textChannels = channels.Where(c => c.IsTextChannel).ToArray();
await base.ExecuteAsync(console, textChannels);

4
DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs
View file

@ -17,7 +17,9 @@ namespace DiscordChatExporter.Cli.Commands
public override async ValueTask ExecuteAsync(IConsole console)
{
var channels = await Discord.GetGuildChannelsAsync(GuildId);
var cancellationToken = console.RegisterCancellationHandler();
var channels = await Discord.GetGuildChannelsAsync(GuildId, cancellationToken);
var textChannels = channels
.Where(c => c.IsTextChannel)

4
DiscordChatExporter.Cli/Commands/GetDirectMessageChannelsCommand.cs
View file

@ -14,7 +14,9 @@ namespace DiscordChatExporter.Cli.Commands
{
public override async ValueTask ExecuteAsync(IConsole console)
{
var channels = await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id);
var cancellationToken = console.RegisterCancellationHandler();
var channels = await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken);
var textChannels = channels
.Where(c => c.IsTextChannel)

4
DiscordChatExporter.Cli/Commands/GetGuildsCommand.cs
View file

@ -13,7 +13,9 @@ namespace DiscordChatExporter.Cli.Commands
{
public override async ValueTask ExecuteAsync(IConsole console)
{
var guilds = await Discord.GetUserGuildsAsync();
var cancellationToken = console.RegisterCancellationHandler();
var guilds = await Discord.GetUserGuildsAsync(cancellationToken);
foreach (var guild in guilds.OrderBy(g => g.Name))
{

94
DiscordChatExporter.Core/Discord/DiscordClient.cs
View file

@ -3,7 +3,9 @@ using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exceptions;
@ -21,18 +23,28 @@ namespace DiscordChatExporter.Core.Discord
public DiscordClient(AuthToken token) => _token = token;
private async ValueTask<HttpResponseMessage> GetResponseAsync(string url) =>
await Http.ResponsePolicy.ExecuteAsync(async () =>
private async ValueTask<HttpResponseMessage> GetResponseAsync(
string url,
CancellationToken cancellationToken = default)
{
return await Http.ResponsePolicy.ExecuteAsync(async innerCancellationToken =>
{
using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_baseUri, url));
request.Headers.Authorization = _token.GetAuthenticationHeader();
return await Http.Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
});
return await Http.Client.SendAsync(
request,
HttpCompletionOption.ResponseHeadersRead,
innerCancellationToken
);
}, cancellationToken);
}
private async ValueTask<JsonElement> GetJsonResponseAsync(string url)
private async ValueTask<JsonElement> GetJsonResponseAsync(
string url,
CancellationToken cancellationToken = default)
{
using var response = await GetResponseAsync(url);
using var response = await GetResponseAsync(url, cancellationToken);
if (!response.IsSuccessStatusCode)
{
@ -45,19 +57,22 @@ namespace DiscordChatExporter.Core.Discord
};
}
return await response.Content.ReadAsJsonAsync();
return await response.Content.ReadAsJsonAsync(cancellationToken);
}
private async ValueTask<JsonElement?> TryGetJsonResponseAsync(string url)
private async ValueTask<JsonElement?> TryGetJsonResponseAsync(
string url,
CancellationToken cancellationToken = default)
{
using var response = await GetResponseAsync(url);
using var response = await GetResponseAsync(url, cancellationToken);
return response.IsSuccessStatusCode
? await response.Content.ReadAsJsonAsync()
? await response.Content.ReadAsJsonAsync(cancellationToken)
: null;
}
public async IAsyncEnumerable<Guild> GetUserGuildsAsync()
public async IAsyncEnumerable<Guild> GetUserGuildsAsync(
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
yield return Guild.DirectMessages;
@ -71,7 +86,7 @@ namespace DiscordChatExporter.Core.Discord
.SetQueryParameter("after", currentAfter.ToString())
.Build();
var response = await GetJsonResponseAsync(url);
var response = await GetJsonResponseAsync(url, cancellationToken);
var isEmpty = true;
foreach (var guild in response.EnumerateArray().Select(Guild.Parse))
@ -87,26 +102,30 @@ namespace DiscordChatExporter.Core.Discord
}
}
public async ValueTask<Guild> GetGuildAsync(Snowflake guildId)
public async ValueTask<Guild> GetGuildAsync(
Snowflake guildId,
CancellationToken cancellationToken = default)
{
if (guildId == Guild.DirectMessages.Id)
return Guild.DirectMessages;
var response = await GetJsonResponseAsync($"guilds/{guildId}");
var response = await GetJsonResponseAsync($"guilds/{guildId}", cancellationToken);
return Guild.Parse(response);
}
public async IAsyncEnumerable<Channel> GetGuildChannelsAsync(Snowflake guildId)
public async IAsyncEnumerable<Channel> GetGuildChannelsAsync(
Snowflake guildId,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
if (guildId == Guild.DirectMessages.Id)
{
var response = await GetJsonResponseAsync("users/@me/channels");
var response = await GetJsonResponseAsync("users/@me/channels", cancellationToken);
foreach (var channelJson in response.EnumerateArray())
yield return Channel.Parse(channelJson);
}
else
{
var response = await GetJsonResponseAsync($"guilds/{guildId}/channels");
var response = await GetJsonResponseAsync($"guilds/{guildId}/channels", cancellationToken);
var responseOrdered = response
.EnumerateArray()
@ -138,31 +157,38 @@ namespace DiscordChatExporter.Core.Discord
}
}
public async IAsyncEnumerable<Role> GetGuildRolesAsync(Snowflake guildId)
public async IAsyncEnumerable<Role> GetGuildRolesAsync(
Snowflake guildId,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
if (guildId == Guild.DirectMessages.Id)
yield break;
var response = await GetJsonResponseAsync($"guilds/{guildId}/roles");
var response = await GetJsonResponseAsync($"guilds/{guildId}/roles", cancellationToken);
foreach (var roleJson in response.EnumerateArray())
yield return Role.Parse(roleJson);
}
public async ValueTask<Member> GetGuildMemberAsync(Snowflake guildId, User user)
public async ValueTask<Member> GetGuildMemberAsync(
Snowflake guildId,
User user,
CancellationToken cancellationToken = default)
{
if (guildId == Guild.DirectMessages.Id)
return Member.CreateForUser(user);
var response = await TryGetJsonResponseAsync($"guilds/{guildId}/members/{user.Id}");
var response = await TryGetJsonResponseAsync($"guilds/{guildId}/members/{user.Id}", cancellationToken);
return response?.Pipe(Member.Parse) ?? Member.CreateForUser(user);
}
public async ValueTask<ChannelCategory> GetChannelCategoryAsync(Snowflake channelId)
public async ValueTask<ChannelCategory> GetChannelCategoryAsync(
Snowflake channelId,
CancellationToken cancellationToken = default)
{
try
{
var response = await GetJsonResponseAsync($"channels/{channelId}");
var response = await GetJsonResponseAsync($"channels/{channelId}", cancellationToken);
return ChannelCategory.Parse(response);
}
// In some cases, the Discord API returns an empty body when requesting channel category.
@ -173,20 +199,25 @@ namespace DiscordChatExporter.Core.Discord
}
}
public async ValueTask<Channel> GetChannelAsync(Snowflake channelId)
public async ValueTask<Channel> GetChannelAsync(
Snowflake channelId,
CancellationToken cancellationToken = default)
{
var response = await GetJsonResponseAsync($"channels/{channelId}");
var response = await GetJsonResponseAsync($"channels/{channelId}", cancellationToken);
var parentId = response.GetPropertyOrNull("parent_id")?.GetString().Pipe(Snowflake.Parse);
var category = parentId is not null
? await GetChannelCategoryAsync(parentId.Value)
? await GetChannelCategoryAsync(parentId.Value, cancellationToken)
: null;
return Channel.Parse(response, category);
}
private async ValueTask<Message?> TryGetLastMessageAsync(Snowflake channelId, Snowflake? before = null)
private async ValueTask<Message?> TryGetLastMessageAsync(
Snowflake channelId,
Snowflake? before = null,
CancellationToken cancellationToken = default)
{
var url = new UrlBuilder()
.SetPath($"channels/{channelId}/messages")
@ -194,7 +225,7 @@ namespace DiscordChatExporter.Core.Discord
.SetQueryParameter("before", before?.ToString())
.Build();
var response = await GetJsonResponseAsync(url);
var response = await GetJsonResponseAsync(url, cancellationToken);
return response.EnumerateArray().Select(Message.Parse).LastOrDefault();
}
@ -202,13 +233,14 @@ namespace DiscordChatExporter.Core.Discord
Snowflake channelId,
Snowflake? after = null,
Snowflake? before = null,
IProgress<double>? progress = null)
IProgress<double>? progress = null,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
// Get the last message in the specified range.
// This snapshots the boundaries, which means that messages posted after the export started
// will not appear in the output.
// Additionally, it provides the date of the last message, which is used to calculate progress.
var lastMessage = await TryGetLastMessageAsync(channelId, before);
var lastMessage = await TryGetLastMessageAsync(channelId, before, cancellationToken);
if (lastMessage is null || lastMessage.Timestamp < after?.ToDate())
yield break;
@ -224,7 +256,7 @@ namespace DiscordChatExporter.Core.Discord
.SetQueryParameter("after", currentAfter.ToString())
.Build();
var response = await GetJsonResponseAsync(url);
var response = await GetJsonResponseAsync(url, cancellationToken);
var messages = response
.EnumerateArray()

2
DiscordChatExporter.Core/DiscordChatExporter.Core.csproj
View file

@ -6,7 +6,7 @@
<ItemGroup>
<PackageReference Include="JsonExtensions" Version="1.1.0" />
<PackageReference Include="MiniRazor.CodeGen" Version="2.1.4" />
<PackageReference Include="MiniRazor.CodeGen" Version="2.2.0" />
<PackageReference Include="Polly" Version="7.2.2" />
<PackageReference Include="Superpower" Version="3.0.0" />
<PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" />

29
DiscordChatExporter.Core/Exporting/ChannelExporter.cs
View file

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
@ -18,12 +19,15 @@ namespace DiscordChatExporter.Core.Exporting
public ChannelExporter(AuthToken token) : this(new DiscordClient(token)) {}
public async ValueTask ExportChannelAsync(ExportRequest request, IProgress<double>? progress = null)
public async ValueTask ExportChannelAsync(
ExportRequest request,
IProgress<double>? progress = null,
CancellationToken cancellationToken = default)
{
// Build context
var contextMembers = new HashSet<Member>(IdBasedEqualityComparer.Instance);
var contextChannels = await _discord.GetGuildChannelsAsync(request.Guild.Id);
var contextRoles = await _discord.GetGuildRolesAsync(request.Guild.Id);
var contextChannels = await _discord.GetGuildChannelsAsync(request.Guild.Id, cancellationToken);
var contextRoles = await _discord.GetGuildRolesAsync(request.Guild.Id, cancellationToken);
var context = new ExportContext(
request,
@ -37,8 +41,16 @@ namespace DiscordChatExporter.Core.Exporting
var exportedAnything = false;
var encounteredUsers = new HashSet<User>(IdBasedEqualityComparer.Instance);
await foreach (var message in _discord.GetMessagesAsync(request.Channel.Id, request.After, request.Before, progress))
await foreach (var message in _discord.GetMessagesAsync(
request.Channel.Id,
request.After,
request.Before,
progress,
cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
// Skips any messages that fail to pass the supplied filter
if (!request.MessageFilter.IsMatch(message))
continue;
@ -49,12 +61,17 @@ namespace DiscordChatExporter.Core.Exporting
if (!encounteredUsers.Add(referencedUser))
continue;
var member = await _discord.GetGuildMemberAsync(request.Guild.Id, referencedUser);
var member = await _discord.GetGuildMemberAsync(
request.Guild.Id,
referencedUser,
cancellationToken
);
contextMembers.Add(member);
}
// Export message
await messageExporter.ExportMessageAsync(message);
await messageExporter.ExportMessageAsync(message, cancellationToken);
exportedAnything = true;
}

16
DiscordChatExporter.Core/Exporting/ExportContext.cs
View file

@ -4,11 +4,11 @@ using System.Drawing;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Utils.Extensions;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Exporting
{
@ -63,14 +63,14 @@ namespace DiscordChatExporter.Core.Exporting
.FirstOrDefault();
}
public async ValueTask<string> ResolveMediaUrlAsync(string url)
public async ValueTask<string> ResolveMediaUrlAsync(string url, CancellationToken cancellationToken = default)
{
if (!Request.ShouldDownloadMedia)
return url;
try
{
var filePath = await _mediaDownloader.DownloadAsync(url);
var filePath = await _mediaDownloader.DownloadAsync(url, cancellationToken);
// We want relative path so that the output files can be copied around without breaking.
// Base directory path may be null if the file is stored at the root or relative to working directory.
@ -82,10 +82,12 @@ namespace DiscordChatExporter.Core.Exporting
if (Request.Format is ExportFormat.HtmlDark or ExportFormat.HtmlLight)
{
// Need to escape each path segment while keeping the directory separators intact
return relativeFilePath
.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)
.Select(Uri.EscapeDataString)
.JoinToString(Path.AltDirectorySeparatorChar.ToString());
return string.Join(
Path.AltDirectorySeparatorChar,
relativeFilePath
.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)
.Select(Uri.EscapeDataString)
);
}
return relativeFilePath;

5
DiscordChatExporter.Core/Exporting/MediaDownloader.cs
View file

@ -5,6 +5,7 @@ using System.IO;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Utils;
using DiscordChatExporter.Core.Utils.Extensions;
@ -25,7 +26,7 @@ namespace DiscordChatExporter.Core.Exporting
_reuseMedia = reuseMedia;
}
public async ValueTask<string> DownloadAsync(string url)
public async ValueTask<string> DownloadAsync(string url, CancellationToken cancellationToken = default)
{
if (_pathCache.TryGetValue(url, out var cachedFilePath))
return cachedFilePath;
@ -43,7 +44,7 @@ namespace DiscordChatExporter.Core.Exporting
await Http.ExceptionPolicy.ExecuteAsync(async () =>
{
// Download the file
using var response = await Http.Client.GetAsync(url);
using var response = await Http.Client.GetAsync(url, cancellationToken);
await using (var output = File.Create(filePath))
{
await response.Content.CopyToAsync(output);

17
DiscordChatExporter.Core/Exporting/MessageExporter.cs
View file

@ -1,5 +1,6 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exporting.Writers;
@ -18,23 +19,23 @@ namespace DiscordChatExporter.Core.Exporting
_context = context;
}
private async ValueTask ResetWriterAsync()
private async ValueTask ResetWriterAsync(CancellationToken cancellationToken = default)
{
if (_writer is not null)
{
await _writer.WritePostambleAsync();
await _writer.WritePostambleAsync(cancellationToken);
await _writer.DisposeAsync();
_writer = null;
}
}
private async ValueTask<MessageWriter> GetWriterAsync()
private async ValueTask<MessageWriter> GetWriterAsync(CancellationToken cancellationToken = default)
{
// Ensure partition limit has not been reached
if (_writer is not null &&
_context.Request.PartitionLimit.IsReached(_writer.MessagesWritten, _writer.BytesWritten))
{
await ResetWriterAsync();
await ResetWriterAsync(cancellationToken);
_partitionIndex++;
}
@ -49,15 +50,15 @@ namespace DiscordChatExporter.Core.Exporting
Directory.CreateDirectory(dirPath);
var writer = CreateMessageWriter(filePath, _context.Request.Format, _context);
await writer.WritePreambleAsync();
await writer.WritePreambleAsync(cancellationToken);
return _writer = writer;
}
public async ValueTask ExportMessageAsync(Message message)
public async ValueTask ExportMessageAsync(Message message, CancellationToken cancellationToken = default)
{
var writer = await GetWriterAsync();
await writer.WriteMessageAsync(message);
var writer = await GetWriterAsync(cancellationToken);
await writer.WriteMessageAsync(message, cancellationToken);
}
public async ValueTask DisposeAsync() => await ResetWriterAsync();

27
DiscordChatExporter.Core/Exporting/Writers/CsvMessageWriter.cs
View file

@ -1,6 +1,7 @@
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors;
@ -21,29 +22,37 @@ namespace DiscordChatExporter.Core.Exporting.Writers
private string FormatMarkdown(string? markdown) =>
PlainTextMarkdownVisitor.Format(Context, markdown ?? "");
public override async ValueTask WritePreambleAsync() =>
public override async ValueTask WritePreambleAsync(CancellationToken cancellationToken = default) =>
await _writer.WriteLineAsync("AuthorID,Author,Date,Content,Attachments,Reactions");
private async ValueTask WriteAttachmentsAsync(IReadOnlyList<Attachment> attachments)
private async ValueTask WriteAttachmentsAsync(
IReadOnlyList<Attachment> attachments,
CancellationToken cancellationToken = default)
{
var buffer = new StringBuilder();
foreach (var attachment in attachments)
{
cancellationToken.ThrowIfCancellationRequested();
buffer
.AppendIfNotEmpty(',')
.Append(await Context.ResolveMediaUrlAsync(attachment.Url));
.Append(await Context.ResolveMediaUrlAsync(attachment.Url, cancellationToken));
}
await _writer.WriteAsync(CsvEncode(buffer.ToString()));
}
private async ValueTask WriteReactionsAsync(IReadOnlyList<Reaction> reactions)
private async ValueTask WriteReactionsAsync(
IReadOnlyList<Reaction> reactions,
CancellationToken cancellationToken = default)
{
var buffer = new StringBuilder();
foreach (var reaction in reactions)
{
cancellationToken.ThrowIfCancellationRequested();
buffer
.AppendIfNotEmpty(',')
.Append(reaction.Emoji.Name)
@ -56,9 +65,11 @@ namespace DiscordChatExporter.Core.Exporting.Writers
await _writer.WriteAsync(CsvEncode(buffer.ToString()));
}
public override async ValueTask WriteMessageAsync(Message message)
public override async ValueTask WriteMessageAsync(
Message message,
CancellationToken cancellationToken = default)
{
await base.WriteMessageAsync(message);
await base.WriteMessageAsync(message, cancellationToken);
// Author ID
await _writer.WriteAsync(CsvEncode(message.Author.Id.ToString()));
@ -77,11 +88,11 @@ namespace DiscordChatExporter.Core.Exporting.Writers
await _writer.WriteAsync(',');
// Attachments
await WriteAttachmentsAsync(message.Attachments);
await WriteAttachmentsAsync(message.Attachments, cancellationToken);
await _writer.WriteAsync(',');
// Reactions
await WriteReactionsAsync(message.Reactions);
await WriteReactionsAsync(message.Reactions, cancellationToken);
// Finish row
await _writer.WriteLineAsync();

2
DiscordChatExporter.Core/Exporting/Writers/Html/PreambleTemplate.cshtml
View file

@ -13,7 +13,7 @@
string FormatDate(DateTimeOffset date) => Model.ExportContext.FormatDate(date);
ValueTask<string> ResolveUrlAsync(string url) => Model.ExportContext.ResolveMediaUrlAsync(url);
ValueTask<string> ResolveUrlAsync(string url) => Model.ExportContext.ResolveMediaUrlAsync(url, CancellationToken);
}
<!DOCTYPE html>

30
DiscordChatExporter.Core/Exporting/Writers/HtmlMessageWriter.cs
View file

@ -1,6 +1,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exporting.Writers.Html;
@ -21,31 +22,35 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_themeName = themeName;
}
public override async ValueTask WritePreambleAsync()
public override async ValueTask WritePreambleAsync(CancellationToken cancellationToken = default)
{
var templateContext = new PreambleTemplateContext(Context, _themeName);
// We are not writing directly to output because Razor
// does not actually do asynchronous writes to stream.
await _writer.WriteLineAsync(
await PreambleTemplate.RenderAsync(templateContext)
await PreambleTemplate.RenderAsync(templateContext, cancellationToken)
);
}
private async ValueTask WriteMessageGroupAsync(MessageGroup messageGroup)
private async ValueTask WriteMessageGroupAsync(
MessageGroup messageGroup,
CancellationToken cancellationToken = default)
{
var templateContext = new MessageGroupTemplateContext(Context, messageGroup);
// We are not writing directly to output because Razor
// does not actually do asynchronous writes to stream.
await _writer.WriteLineAsync(
await MessageGroupTemplate.RenderAsync(templateContext)
await MessageGroupTemplate.RenderAsync(templateContext, cancellationToken)
);
}
public override async ValueTask WriteMessageAsync(Message message)
public override async ValueTask WriteMessageAsync(
Message message,
CancellationToken cancellationToken = default)
{
await base.WriteMessageAsync(message);
await base.WriteMessageAsync(message, cancellationToken);
// If message group is empty or the given message can be grouped, buffer the given message
if (!_messageGroupBuffer.Any() || MessageGroup.CanJoin(_messageGroupBuffer.Last(), message))
@ -55,25 +60,30 @@ namespace DiscordChatExporter.Core.Exporting.Writers
// Otherwise, flush the group and render messages
else
{
await WriteMessageGroupAsync(MessageGroup.Join(_messageGroupBuffer));
await WriteMessageGroupAsync(MessageGroup.Join(_messageGroupBuffer), cancellationToken);
_messageGroupBuffer.Clear();
_messageGroupBuffer.Add(message);
}
}
public override async ValueTask WritePostambleAsync()
public override async ValueTask WritePostambleAsync(CancellationToken cancellationToken = default)
{
// Flush current message group
if (_messageGroupBuffer.Any())
await WriteMessageGroupAsync(MessageGroup.Join(_messageGroupBuffer));
{
await WriteMessageGroupAsync(
MessageGroup.Join(_messageGroupBuffer),
cancellationToken
);
}
var templateContext = new PostambleTemplateContext(Context, MessagesWritten);
// We are not writing directly to output because Razor
// does not actually do asynchronous writes to stream.
await _writer.WriteLineAsync(
await PostambleTemplate.RenderAsync(templateContext)
await PostambleTemplate.RenderAsync(templateContext, cancellationToken)
);
}

105
DiscordChatExporter.Core/Exporting/Writers/JsonMessageWriter.cs
View file

@ -1,6 +1,7 @@
using System.IO;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Discord.Data.Embeds;
@ -30,20 +31,24 @@ namespace DiscordChatExporter.Core.Exporting.Writers
private string FormatMarkdown(string? markdown) =>
PlainTextMarkdownVisitor.Format(Context, markdown ?? "");
private async ValueTask WriteAttachmentAsync(Attachment attachment)
private async ValueTask WriteAttachmentAsync(
Attachment attachment,
CancellationToken cancellationToken = default)
{
_writer.WriteStartObject();
_writer.WriteString("id", attachment.Id.ToString());
_writer.WriteString("url", await Context.ResolveMediaUrlAsync(attachment.Url));
_writer.WriteString("url", await Context.ResolveMediaUrlAsync(attachment.Url, cancellationToken));
_writer.WriteString("fileName", attachment.FileName);
_writer.WriteNumber("fileSizeBytes", attachment.FileSize.TotalBytes);
_writer.WriteEndObject();
await _writer.FlushAsync();
await _writer.FlushAsync(cancellationToken);
}
private async ValueTask WriteEmbedAuthorAsync(EmbedAuthor embedAuthor)
private async ValueTask WriteEmbedAuthorAsync(
EmbedAuthor embedAuthor,
CancellationToken cancellationToken = default)
{
_writer.WriteStartObject("author");
@ -51,54 +56,62 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_writer.WriteString("url", embedAuthor.Url);
if (!string.IsNullOrWhiteSpace(embedAuthor.IconUrl))
_writer.WriteString("iconUrl", await Context.ResolveMediaUrlAsync(embedAuthor.IconProxyUrl ?? embedAuthor.IconUrl));
_writer.WriteString("iconUrl", await Context.ResolveMediaUrlAsync(embedAuthor.IconProxyUrl ?? embedAuthor.IconUrl, cancellationToken));
_writer.WriteEndObject();
await _writer.FlushAsync();
await _writer.FlushAsync(cancellationToken);
}
private async ValueTask WriteEmbedThumbnailAsync(EmbedImage embedThumbnail)
private async ValueTask WriteEmbedThumbnailAsync(
EmbedImage embedThumbnail,
CancellationToken cancellationToken = default)
{
_writer.WriteStartObject("thumbnail");
if (!string.IsNullOrWhiteSpace(embedThumbnail.Url))
_writer.WriteString("url", await Context.ResolveMediaUrlAsync(embedThumbnail.ProxyUrl ?? embedThumbnail.Url));
_writer.WriteString("url", await Context.ResolveMediaUrlAsync(embedThumbnail.ProxyUrl ?? embedThumbnail.Url, cancellationToken));
_writer.WriteNumber("width", embedThumbnail.Width);
_writer.WriteNumber("height", embedThumbnail.Height);
_writer.WriteEndObject();
await _writer.FlushAsync();
await _writer.FlushAsync(cancellationToken);
}
private async ValueTask WriteEmbedImageAsync(EmbedImage embedImage)
private async ValueTask WriteEmbedImageAsync(
EmbedImage embedImage,
CancellationToken cancellationToken = default)
{
_writer.WriteStartObject("image");
if (!string.IsNullOrWhiteSpace(embedImage.Url))
_writer.WriteString("url", await Context.ResolveMediaUrlAsync(embedImage.ProxyUrl ?? embedImage.Url));
_writer.WriteString("url", await Context.ResolveMediaUrlAsync(embedImage.ProxyUrl ?? embedImage.Url, cancellationToken));
_writer.WriteNumber("width", embedImage.Width);
_writer.WriteNumber("height", embedImage.Height);
_writer.WriteEndObject();
await _writer.FlushAsync();
await _writer.FlushAsync(cancellationToken);
}
private async ValueTask WriteEmbedFooterAsync(EmbedFooter embedFooter)
private async ValueTask WriteEmbedFooterAsync(
EmbedFooter embedFooter,
CancellationToken cancellationToken = default)
{
_writer.WriteStartObject("footer");
_writer.WriteString("text", embedFooter.Text);
if (!string.IsNullOrWhiteSpace(embedFooter.IconUrl))
_writer.WriteString("iconUrl", await Context.ResolveMediaUrlAsync(embedFooter.IconProxyUrl ?? embedFooter.IconUrl));
_writer.WriteString("iconUrl", await Context.ResolveMediaUrlAsync(embedFooter.IconProxyUrl ?? embedFooter.IconUrl, cancellationToken));
_writer.WriteEndObject();
await _writer.FlushAsync();
await _writer.FlushAsync(cancellationToken);
}
private async ValueTask WriteEmbedFieldAsync(EmbedField embedField)
private async ValueTask WriteEmbedFieldAsync(
EmbedField embedField,
CancellationToken cancellationToken = default)
{
_writer.WriteStartObject();
@ -107,10 +120,12 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_writer.WriteBoolean("isInline", embedField.IsInline);
_writer.WriteEndObject();
await _writer.FlushAsync();
await _writer.FlushAsync(cancellationToken);
}
private async ValueTask WriteEmbedAsync(Embed embed)
private async ValueTask WriteEmbedAsync(
Embed embed,
CancellationToken cancellationToken = default)
{
_writer.WriteStartObject();
@ -123,30 +138,32 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_writer.WriteString("color", embed.Color.Value.ToHex());
if (embed.Author is not null)
await WriteEmbedAuthorAsync(embed.Author);
await WriteEmbedAuthorAsync(embed.Author, cancellationToken);
if (embed.Thumbnail is not null)
await WriteEmbedThumbnailAsync(embed.Thumbnail);
await WriteEmbedThumbnailAsync(embed.Thumbnail, cancellationToken);
if (embed.Image is not null)
await WriteEmbedImageAsync(embed.Image);
await WriteEmbedImageAsync(embed.Image, cancellationToken);
if (embed.Footer is not null)
await WriteEmbedFooterAsync(embed.Footer);
await WriteEmbedFooterAsync(embed.Footer, cancellationToken);
// Fields
_writer.WriteStartArray("fields");
foreach (var field in embed.Fields)
await WriteEmbedFieldAsync(field);
await WriteEmbedFieldAsync(field, cancellationToken);
_writer.WriteEndArray();
_writer.WriteEndObject();
await _writer.FlushAsync();
await _writer.FlushAsync(cancellationToken);
}
private async ValueTask WriteReactionAsync(Reaction reaction)
private async ValueTask WriteReactionAsync(
Reaction reaction,
CancellationToken cancellationToken = default)
{
_writer.WriteStartObject();
@ -155,16 +172,18 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_writer.WriteString("id", reaction.Emoji.Id);
_writer.WriteString("name", reaction.Emoji.Name);
_writer.WriteBoolean("isAnimated", reaction.Emoji.IsAnimated);
_writer.WriteString("imageUrl", await Context.ResolveMediaUrlAsync(reaction.Emoji.ImageUrl));
_writer.WriteString("imageUrl", await Context.ResolveMediaUrlAsync(reaction.Emoji.ImageUrl, cancellationToken));
_writer.WriteEndObject();
_writer.WriteNumber("count", reaction.Count);
_writer.WriteEndObject();
await _writer.FlushAsync();
await _writer.FlushAsync(cancellationToken);
}
private async ValueTask WriteMentionAsync(User mentionedUser)
private async ValueTask WriteMentionAsync(
User mentionedUser,
CancellationToken cancellationToken = default)
{
_writer.WriteStartObject();
@ -175,10 +194,10 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_writer.WriteBoolean("isBot", mentionedUser.IsBot);
_writer.WriteEndObject();
await _writer.FlushAsync();
await _writer.FlushAsync(cancellationToken);
}
public override async ValueTask WritePreambleAsync()
public override async ValueTask WritePreambleAsync(CancellationToken cancellationToken = default)
{
// Root object (start)
_writer.WriteStartObject();
@ -187,7 +206,7 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_writer.WriteStartObject("guild");
_writer.WriteString("id", Context.Request.Guild.Id.ToString());
_writer.WriteString("name", Context.Request.Guild.Name);
_writer.WriteString("iconUrl", await Context.ResolveMediaUrlAsync(Context.Request.Guild.IconUrl));
_writer.WriteString("iconUrl", await Context.ResolveMediaUrlAsync(Context.Request.Guild.IconUrl, cancellationToken));
_writer.WriteEndObject();
// Channel
@ -208,12 +227,14 @@ namespace DiscordChatExporter.Core.Exporting.Writers
// Message array (start)
_writer.WriteStartArray("messages");
await _writer.FlushAsync();
await _writer.FlushAsync(cancellationToken);
}
public override async ValueTask WriteMessageAsync(Message message)
public override async ValueTask WriteMessageAsync(
Message message,
CancellationToken cancellationToken = default)
{
await base.WriteMessageAsync(message);
await base.WriteMessageAsync(message, cancellationToken);
_writer.WriteStartObject();
@ -236,14 +257,14 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_writer.WriteString("nickname", Context.TryGetMember(message.Author.Id)?.Nick ?? message.Author.Name);
_writer.WriteString("color", Context.TryGetUserColor(message.Author.Id)?.ToHex());
_writer.WriteBoolean("isBot", message.Author.IsBot);
_writer.WriteString("avatarUrl", await Context.ResolveMediaUrlAsync(message.Author.AvatarUrl));
_writer.WriteString("avatarUrl", await Context.ResolveMediaUrlAsync(message.Author.AvatarUrl, cancellationToken));
_writer.WriteEndObject();
// Attachments
_writer.WriteStartArray("attachments");
foreach (var attachment in message.Attachments)
await WriteAttachmentAsync(attachment);
await WriteAttachmentAsync(attachment, cancellationToken);
_writer.WriteEndArray();
@ -251,7 +272,7 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_writer.WriteStartArray("embeds");
foreach (var embed in message.Embeds)
await WriteEmbedAsync(embed);
await WriteEmbedAsync(embed, cancellationToken);
_writer.WriteEndArray();
@ -259,7 +280,7 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_writer.WriteStartArray("reactions");
foreach (var reaction in message.Reactions)
await WriteReactionAsync(reaction);
await WriteReactionAsync(reaction, cancellationToken);
_writer.WriteEndArray();
@ -267,7 +288,7 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_writer.WriteStartArray("mentions");
foreach (var mention in message.MentionedUsers)
await WriteMentionAsync(mention);
await WriteMentionAsync(mention, cancellationToken);
_writer.WriteEndArray();
@ -282,10 +303,10 @@ namespace DiscordChatExporter.Core.Exporting.Writers
}
_writer.WriteEndObject();
await _writer.FlushAsync();
await _writer.FlushAsync(cancellationToken);
}
public override async ValueTask WritePostambleAsync()
public override async ValueTask WritePostambleAsync(CancellationToken cancellationToken = default)
{
// Message array (end)
_writer.WriteEndArray();
@ -294,7 +315,7 @@ namespace DiscordChatExporter.Core.Exporting.Writers
// Root object (end)
_writer.WriteEndObject();
await _writer.FlushAsync();
await _writer.FlushAsync(cancellationToken);
}
public override async ValueTask DisposeAsync()

7
DiscordChatExporter.Core/Exporting/Writers/MessageWriter.cs
View file

@ -1,5 +1,6 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord.Data;
@ -21,15 +22,15 @@ namespace DiscordChatExporter.Core.Exporting.Writers
Context = context;
}
public virtual ValueTask WritePreambleAsync() => default;
public virtual ValueTask WritePreambleAsync(CancellationToken cancellationToken = default) => default;
public virtual ValueTask WriteMessageAsync(Message message)
public virtual ValueTask WriteMessageAsync(Message message, CancellationToken cancellationToken = default)
{
MessagesWritten++;
return default;
}
public virtual ValueTask WritePostambleAsync() => default;
public virtual ValueTask WritePostambleAsync(CancellationToken cancellationToken = default) => default;
public virtual async ValueTask DisposeAsync() => await Stream.DisposeAsync();
}

43
DiscordChatExporter.Core/Exporting/Writers/PlainTextMessageWriter.cs
View file

@ -1,6 +1,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Discord.Data.Embeds;
@ -35,7 +36,9 @@ namespace DiscordChatExporter.Core.Exporting.Writers
await _writer.WriteLineAsync();
}
private async ValueTask WriteAttachmentsAsync(IReadOnlyList<Attachment> attachments)
private async ValueTask WriteAttachmentsAsync(
IReadOnlyList<Attachment> attachments,
CancellationToken cancellationToken = default)
{
if (!attachments.Any())
return;
@ -43,15 +46,23 @@ namespace DiscordChatExporter.Core.Exporting.Writers
await _writer.WriteLineAsync("{Attachments}");
foreach (var attachment in attachments)
await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(attachment.Url));
{
cancellationToken.ThrowIfCancellationRequested();
await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(attachment.Url, cancellationToken));
}
await _writer.WriteLineAsync();
}
private async ValueTask WriteEmbedsAsync(IReadOnlyList<Embed> embeds)
private async ValueTask WriteEmbedsAsync(
IReadOnlyList<Embed> embeds,
CancellationToken cancellationToken = default)
{
foreach (var embed in embeds)
{
cancellationToken.ThrowIfCancellationRequested();
await _writer.WriteLineAsync("{Embed}");
if (!string.IsNullOrWhiteSpace(embed.Author?.Name))
@ -76,10 +87,10 @@ namespace DiscordChatExporter.Core.Exporting.Writers
}
if (!string.IsNullOrWhiteSpace(embed.Thumbnail?.Url))
await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(embed.Thumbnail.ProxyUrl ?? embed.Thumbnail.Url));
await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(embed.Thumbnail.ProxyUrl ?? embed.Thumbnail.Url, cancellationToken));
if (!string.IsNullOrWhiteSpace(embed.Image?.Url))
await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(embed.Image.ProxyUrl ?? embed.Image.Url));
await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(embed.Image.ProxyUrl ?? embed.Image.Url, cancellationToken));
if (!string.IsNullOrWhiteSpace(embed.Footer?.Text))
await _writer.WriteLineAsync(embed.Footer.Text);
@ -88,7 +99,9 @@ namespace DiscordChatExporter.Core.Exporting.Writers
}
}
private async ValueTask WriteReactionsAsync(IReadOnlyList<Reaction> reactions)
private async ValueTask WriteReactionsAsync(
IReadOnlyList<Reaction> reactions,
CancellationToken cancellationToken = default)
{
if (!reactions.Any())
return;
@ -97,6 +110,8 @@ namespace DiscordChatExporter.Core.Exporting.Writers
foreach (var reaction in reactions)
{
cancellationToken.ThrowIfCancellationRequested();
await _writer.WriteAsync(reaction.Emoji.Name);
if (reaction.Count > 1)
@ -108,7 +123,7 @@ namespace DiscordChatExporter.Core.Exporting.Writers
await _writer.WriteLineAsync();
}
public override async ValueTask WritePreambleAsync()
public override async ValueTask WritePreambleAsync(CancellationToken cancellationToken = default)
{
await _writer.WriteLineAsync('='.Repeat(62));
await _writer.WriteLineAsync($"Guild: {Context.Request.Guild.Name}");
@ -127,9 +142,11 @@ namespace DiscordChatExporter.Core.Exporting.Writers
await _writer.WriteLineAsync();
}
public override async ValueTask WriteMessageAsync(Message message)
public override async ValueTask WriteMessageAsync(
Message message,
CancellationToken cancellationToken = default)
{
await base.WriteMessageAsync(message);
await base.WriteMessageAsync(message, cancellationToken);
// Header
await WriteMessageHeaderAsync(message);
@ -141,14 +158,14 @@ namespace DiscordChatExporter.Core.Exporting.Writers
await _writer.WriteLineAsync();
// Attachments, embeds, reactions
await WriteAttachmentsAsync(message.Attachments);
await WriteEmbedsAsync(message.Embeds);
await WriteReactionsAsync(message.Reactions);
await WriteAttachmentsAsync(message.Attachments, cancellationToken);
await WriteEmbedsAsync(message.Embeds, cancellationToken);
await WriteReactionsAsync(message.Reactions, cancellationToken);
await _writer.WriteLineAsync();
}
public override async ValueTask WritePostambleAsync()
public override async ValueTask WritePostambleAsync(CancellationToken cancellationToken = default)
{
await _writer.WriteLineAsync('='.Repeat(62));
await _writer.WriteLineAsync($"Exported {MessagesWritten:N0} message(s)");

5
DiscordChatExporter.Core/Utils/Extensions/AsyncExtensions.cs
View file

@ -27,14 +27,15 @@ namespace DiscordChatExporter.Core.Utils.Extensions
public static async ValueTask ParallelForEachAsync<T>(
this IEnumerable<T> source,
Func<T, ValueTask> handleAsync,
int degreeOfParallelism)
int degreeOfParallelism,
CancellationToken cancellationToken = default)
{
using var semaphore = new SemaphoreSlim(degreeOfParallelism);
await Task.WhenAll(source.Select(async item =>
{
// ReSharper disable once AccessToDisposedClosure
await semaphore.WaitAsync();
await semaphore.WaitAsync(cancellationToken);
try
{

2
DiscordChatExporter.Gui/DiscordChatExporter.Gui.csproj
View file

@ -19,7 +19,7 @@
<PackageReference Include="Gress" Version="1.2.0" />
<PackageReference Include="MaterialDesignColors" Version="2.0.1" />
<PackageReference Include="MaterialDesignThemes" Version="4.0.0" />
<PackageReference Include="Microsoft.Xaml.Behaviors.Wpf" Version="1.1.31" />
<PackageReference Include="Microsoft.Xaml.Behaviors.Wpf" Version="1.1.37" />
<PackageReference Include="Ookii.Dialogs.Wpf" Version="4.0.0" />
<PackageReference Include="Onova" Version="2.6.2" />
<PackageReference Include="Stylet" Version="1.3.6" />