mirrors/DiscordChatExporter
1
0
Fork 0
mirror of https://github.com/Tyrrrz/DiscordChatExporter.git synced 2025-05-23 11:16:59 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Self-contained export (#321)

Browse source
This commit is contained in:
Alexey Golub 2020-07-18 15:45:09 +03:00 • committed by GitHub
parent 94a85cdb01
commit ac64d9943a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
56 changed files with 813 additions and 581 deletions
Download patch file Download diff file Expand all files Collapse all files

25
DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs
View file

@ -11,12 +11,12 @@ namespace DiscordChatExporter.Cli.Commands.Base
{
public abstract class ExportCommandBase : TokenCommandBase
{
[CommandOption("format", 'f', Description = "Output file format.")]
public ExportFormat ExportFormat { get; set; } = ExportFormat.HtmlDark;
[CommandOption("output", 'o', Description = "Output file or directory path.")]
public string OutputPath { get; set; } = Directory.GetCurrentDirectory();
[CommandOption("format", 'f', Description = "Output file format.")]
public ExportFormat ExportFormat { get; set; } = ExportFormat.HtmlDark;
[CommandOption("after", Description = "Limit to messages sent after this date.")]
public DateTimeOffset? After { get; set; }
@ -26,6 +26,9 @@ namespace DiscordChatExporter.Cli.Commands.Base
[CommandOption("partition", 'p', Description = "Split output into partitions limited to this number of messages.")]
public int? PartitionLimit { get; set; }
[CommandOption("media", Description = "Download referenced media content.")]
public bool ShouldDownloadMedia { get; set; }
[CommandOption("dateformat", Description = "Date format used in output.")]
public string DateFormat { get; set; } = "dd-MMM-yy hh:mm tt";
@ -36,9 +39,19 @@ namespace DiscordChatExporter.Cli.Commands.Base
console.Output.Write($"Exporting channel '{channel.Category} / {channel.Name}'... ");
var progress = console.CreateProgressTicker();
await GetChannelExporter().ExportAsync(guild, channel,
OutputPath, ExportFormat, DateFormat, PartitionLimit,
After, Before, progress);
var request = new ExportRequest(
guild,
channel,
OutputPath,
ExportFormat,
After,
Before,
PartitionLimit,
ShouldDownloadMedia,
DateFormat
);
await GetChannelExporter().ExportChannelAsync(request, progress);
console.Output.WriteLine();
console.Output.WriteLine("Done.");

22
DiscordChatExporter.Cli/Commands/Base/ExportMultipleCommandBase.cs
View file

@ -7,6 +7,7 @@ using CliFx.Attributes;
using CliFx.Utilities;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exceptions;
using DiscordChatExporter.Domain.Exporting;
using DiscordChatExporter.Domain.Utilities;
using Gress;
using Tyrrrz.Extensions;
@ -15,13 +16,12 @@ namespace DiscordChatExporter.Cli.Commands.Base
{
public abstract class ExportMultipleCommandBase : ExportCommandBase
{
[CommandOption("parallel", Description = "Export this number of separate channels in parallel.")]
[CommandOption("parallel", Description = "Export this number of channels in parallel.")]
public int ParallelLimit { get; set; } = 1;
protected async ValueTask ExportMultipleAsync(IConsole console, IReadOnlyList<Channel> channels)
{
// This uses a separate route from ExportCommandBase because the progress ticker is not thread-safe
// Ugly code ahead. Will need to refactor.
// HACK: this uses a separate route from ExportCommandBase because the progress ticker is not thread-safe
console.Output.Write($"Exporting {channels.Count} channels... ");
var progress = console.CreateProgressTicker();
@ -39,9 +39,19 @@ namespace DiscordChatExporter.Cli.Commands.Base
{
var guild = await GetDiscordClient().GetGuildAsync(channel.GuildId);
await GetChannelExporter().ExportAsync(guild, channel,
OutputPath, ExportFormat, DateFormat, PartitionLimit,
After, Before, operation);
var request = new ExportRequest(
guild,
channel,
OutputPath,
ExportFormat,
After,
Before,
PartitionLimit,
ShouldDownloadMedia,
DateFormat
);
await GetChannelExporter().ExportChannelAsync(request, operation);
Interlocked.Increment(ref successfulExportCount);
}

8
DiscordChatExporter.Domain/Discord/AuthToken.cs
View file

@ -16,9 +16,11 @@ namespace DiscordChatExporter.Domain.Discord
Value = value;
}
public AuthenticationHeaderValue GetAuthorizationHeader() => Type == AuthTokenType.User
? new AuthenticationHeaderValue(Value)
: new AuthenticationHeaderValue("Bot", Value);
public AuthenticationHeaderValue GetAuthorizationHeader() => Type switch
{
AuthTokenType.Bot => new AuthenticationHeaderValue("Bot", Value),
_ => new AuthenticationHeaderValue(Value)
};
public override string ToString() => Value;
}

52
DiscordChatExporter.Domain/Discord/DiscordClient.cs
View file

@ -8,25 +8,26 @@ using System.Threading.Tasks;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exceptions;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Internal.Extensions;
using Polly;
namespace DiscordChatExporter.Domain.Discord
{
public partial class DiscordClient
public class DiscordClient
{
private readonly AuthToken _token;
private readonly HttpClient _httpClient;
private readonly HttpClient _httpClient = Singleton.HttpClient;
private readonly IAsyncPolicy<HttpResponseMessage> _httpRequestPolicy;
private readonly Uri _baseUri = new Uri("https://discordapp.com/api/v6/", UriKind.Absolute);
public DiscordClient(AuthToken token, HttpClient httpClient)
public DiscordClient(AuthToken token)
{
_token = token;
_httpClient = httpClient;
// Discord seems to always respond with 429 on the first request with unreasonable wait time (10+ minutes).
// For that reason the policy will start respecting their retry-after header only after Nth failed response.
// For that reason the policy will ignore such errors at first, then wait a constant amount of time, and
// finally wait the specified amount of time, based on how many requests have failed in a row.
_httpRequestPolicy = Policy
.HandleResult<HttpResponseMessage>(m => m.StatusCode == HttpStatusCode.TooManyRequests)
.OrResult(m => m.StatusCode >= HttpStatusCode.InternalServerError)
@ -41,24 +42,17 @@ namespace DiscordChatExporter.Domain.Discord
return result.Result.Headers.RetryAfter.Delta ?? TimeSpan.FromSeconds(10 * i);
},
(response, timespan, retryCount, context) => Task.CompletedTask);
(response, timespan, retryCount, context) => Task.CompletedTask
);
}
public DiscordClient(AuthToken token)
: this(token, LazyHttpClient.Value)
{
}
private async Task<HttpResponseMessage> GetResponseAsync(string url)
{
return await _httpRequestPolicy.ExecuteAsync(async () =>
private async Task<HttpResponseMessage> GetResponseAsync(string url) => await _httpRequestPolicy.ExecuteAsync(async () =>
{
using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_baseUri, url));
request.Headers.Authorization = _token.GetAuthorizationHeader();
return await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
});
}
private async Task<JsonElement> GetJsonResponseAsync(string url)
{
@ -97,15 +91,14 @@ namespace DiscordChatExporter.Domain.Discord
var url = new UrlBuilder()
.SetPath("users/@me/guilds")
.SetQueryParameter("limit", "100")
.SetQueryParameterIfNotNullOrWhiteSpace("after", afterId)
.SetQueryParameter("after", afterId)
.Build();
var response = await GetJsonResponseAsync(url);
var isEmpty = true;
foreach (var guildJson in response.EnumerateArray())
foreach (var guild in response.EnumerateArray().Select(Guild.Parse))
{
var guild = Guild.Parse(guildJson);
yield return guild;
afterId = guild.Id;
@ -206,7 +199,7 @@ namespace DiscordChatExporter.Domain.Discord
var url = new UrlBuilder()
.SetPath($"channels/{channelId}/messages")
.SetQueryParameter("limit", "1")
.SetQueryParameterIfNotNullOrWhiteSpace("before", before?.ToSnowflake())
.SetQueryParameter("before", before?.ToSnowflake())
.Build();
var response = await GetJsonResponseAsync(url);
@ -219,11 +212,15 @@ namespace DiscordChatExporter.Domain.Discord
DateTimeOffset? before = null,
IProgress<double>? progress = null)
{
// Get the last message in the specified range
// Get the last message in the specified range.
// This snapshots the boundaries, which means that messages posted after the exported started
// will not appear in the output.
// Additionally, it provides the date of the last message, which is used to calculate progress.
var lastMessage = await TryGetLastMessageAsync(channelId, before);
if (lastMessage == null || lastMessage.Timestamp < after)
yield break;
// Keep track of first message in range in order to calculate progress
var firstMessage = default(Message);
var afterId = after?.ToSnowflake() ?? "0";
@ -267,19 +264,4 @@ namespace DiscordChatExporter.Domain.Discord
}
}
}
public partial class DiscordClient
{
private static readonly Lazy<HttpClient> LazyHttpClient = new Lazy<HttpClient>(() =>
{
var handler = new HttpClientHandler();
if (handler.SupportsAutomaticDecompression)
handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
handler.UseCookies = false;
return new HttpClient(handler, true);
});
}
}

2
DiscordChatExporter.Domain/Discord/Models/Attachment.cs
View file

@ -3,7 +3,7 @@ using System.IO;
using System.Linq;
using System.Text.Json;
using DiscordChatExporter.Domain.Discord.Models.Common;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Internal.Extensions;
namespace DiscordChatExporter.Domain.Discord.Models
{

2
DiscordChatExporter.Domain/Discord/Models/Channel.cs
View file

@ -1,7 +1,7 @@
using System.Linq;
using System.Text.Json;
using DiscordChatExporter.Domain.Discord.Models.Common;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Internal.Extensions;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Domain.Discord.Models

2
DiscordChatExporter.Domain/Discord/Models/Embed.cs
View file

@ -3,7 +3,7 @@ using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Text.Json;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Internal.Extensions;
namespace DiscordChatExporter.Domain.Discord.Models
{

2
DiscordChatExporter.Domain/Discord/Models/EmbedAuthor.cs
View file

@ -1,5 +1,5 @@
using System.Text.Json;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Internal.Extensions;
namespace DiscordChatExporter.Domain.Discord.Models
{

2
DiscordChatExporter.Domain/Discord/Models/EmbedField.cs
View file

@ -1,5 +1,5 @@
using System.Text.Json;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Internal.Extensions;
namespace DiscordChatExporter.Domain.Discord.Models
{

2
DiscordChatExporter.Domain/Discord/Models/EmbedFooter.cs
View file

@ -1,5 +1,5 @@
using System.Text.Json;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Internal.Extensions;
namespace DiscordChatExporter.Domain.Discord.Models
{

2
DiscordChatExporter.Domain/Discord/Models/EmbedImage.cs
View file

@ -1,5 +1,5 @@
using System.Text.Json;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Internal.Extensions;
namespace DiscordChatExporter.Domain.Discord.Models
{

2
DiscordChatExporter.Domain/Discord/Models/Emoji.cs
View file

@ -2,7 +2,7 @@
using System.Linq;
using System.Text;
using System.Text.Json;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Internal.Extensions;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Domain.Discord.Models

22
DiscordChatExporter.Domain/Discord/Models/Guild.cs
View file

@ -12,12 +12,11 @@ namespace DiscordChatExporter.Domain.Discord.Models
public string IconUrl { get; }
public Guild(string id, string name, string? iconHash)
public Guild(string id, string name, string iconUrl)
{
Id = id;
Name = name;
IconUrl = GetIconUrl(id, iconHash);
IconUrl = iconUrl;
}
public override string ToString() => Name;
@ -26,12 +25,13 @@ namespace DiscordChatExporter.Domain.Discord.Models
public partial class Guild
{
public static Guild DirectMessages { get; } =
new Guild("@me", "Direct Messages", null);
new Guild("@me", "Direct Messages", GetDefaultIconUrl());
private static string GetIconUrl(string id, string? iconHash) =>
!string.IsNullOrWhiteSpace(iconHash)
? $"https://cdn.discordapp.com/icons/{id}/{iconHash}.png"
: "https://cdn.discordapp.com/embed/avatars/0.png";
private static string GetDefaultIconUrl() =>
"https://cdn.discordapp.com/embed/avatars/0.png";
private static string GetIconUrl(string id, string iconHash) =>
$"https://cdn.discordapp.com/icons/{id}/{iconHash}.png";
public static Guild Parse(JsonElement json)
{
@ -39,7 +39,11 @@ namespace DiscordChatExporter.Domain.Discord.Models
var name = json.GetProperty("name").GetString();
var iconHash = json.GetProperty("icon").GetString();
return new Guild(id, name, iconHash);
var iconUrl = !string.IsNullOrWhiteSpace(iconHash)
? GetIconUrl(id, iconHash)
: GetDefaultIconUrl();
return new Guild(id, name, iconUrl);
}
}
}

2
DiscordChatExporter.Domain/Discord/Models/Member.cs
View file

@ -3,7 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using DiscordChatExporter.Domain.Discord.Models.Common;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Internal.Extensions;
namespace DiscordChatExporter.Domain.Discord.Models
{

7
DiscordChatExporter.Domain/Discord/Models/Message.cs
View file

@ -3,7 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using DiscordChatExporter.Domain.Discord.Models.Common;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Internal.Extensions;
namespace DiscordChatExporter.Domain.Discord.Models
{
@ -71,10 +71,7 @@ namespace DiscordChatExporter.Domain.Discord.Models
MentionedUsers = mentionedUsers;
}
public override string ToString() =>
Content ?? (Embeds.Any()
? "<embed>"
: "<no content>");
public override string ToString() => Content;
}
public partial class Message

2
DiscordChatExporter.Domain/Discord/Models/Reaction.cs
View file

@ -1,5 +1,5 @@
using System.Text.Json;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Internal.Extensions;
namespace DiscordChatExporter.Domain.Discord.Models
{

2
DiscordChatExporter.Domain/Discord/Models/Role.cs
View file

@ -1,6 +1,6 @@
using System.Drawing;
using System.Text.Json;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Internal.Extensions;
namespace DiscordChatExporter.Domain.Discord.Models
{

25
DiscordChatExporter.Domain/Discord/Models/User.cs
View file

@ -1,7 +1,7 @@
using System;
using System.Text.Json;
using DiscordChatExporter.Domain.Discord.Models.Common;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Internal.Extensions;
namespace DiscordChatExporter.Domain.Discord.Models
{
@ -20,14 +20,13 @@ namespace DiscordChatExporter.Domain.Discord.Models
public string AvatarUrl { get; }
public User(string id, bool isBot, int discriminator, string name, string? avatarHash)
public User(string id, bool isBot, int discriminator, string name, string avatarUrl)
{
Id = id;
IsBot = isBot;
Discriminator = discriminator;
Name = name;
AvatarUrl = GetAvatarUrl(id, discriminator, avatarHash);
AvatarUrl = avatarUrl;
}
public override string ToString() => FullName;
@ -35,10 +34,10 @@ namespace DiscordChatExporter.Domain.Discord.Models
public partial class User
{
private static string GetAvatarUrl(string id, int discriminator, string? avatarHash)
{
// Custom avatar
if (!string.IsNullOrWhiteSpace(avatarHash))
private static string GetDefaultAvatarUrl(int discriminator) =>
$"https://cdn.discordapp.com/embed/avatars/{discriminator % 5}.png";
private static string GetAvatarUrl(string id, string avatarHash)
{
// Animated
if (avatarHash.StartsWith("a_", StringComparison.Ordinal))
@ -48,10 +47,6 @@ namespace DiscordChatExporter.Domain.Discord.Models
return $"https://cdn.discordapp.com/avatars/{id}/{avatarHash}.png";
}
// Default avatar
return $"https://cdn.discordapp.com/embed/avatars/{discriminator % 5}.png";
}
public static User Parse(JsonElement json)
{
var id = json.GetProperty("id").GetString();
@ -60,7 +55,11 @@ namespace DiscordChatExporter.Domain.Discord.Models
var avatarHash = json.GetProperty("avatar").GetString();
var isBot = json.GetPropertyOrNull("bot")?.GetBoolean() ?? false;
return new User(id, isBot, discriminator, name, avatarHash);
var avatarUrl = !string.IsNullOrWhiteSpace(avatarHash)
? GetAvatarUrl(id, avatarHash)
: GetDefaultAvatarUrl(discriminator);
return new User(id, isBot, discriminator, name, avatarUrl);
}
}
}

10
DiscordChatExporter.Domain/DiscordChatExporter.Domain.csproj
View file

@ -8,11 +8,11 @@
</ItemGroup>
<ItemGroup>
<EmbeddedResource Include="Exporting\Resources\HtmlCore.css" />
<EmbeddedResource Include="Exporting\Resources\HtmlDark.css" />
<EmbeddedResource Include="Exporting\Resources\HtmlLayoutTemplate.html" />
<EmbeddedResource Include="Exporting\Resources\HtmlLight.css" />
<EmbeddedResource Include="Exporting\Resources\HtmlMessageGroupTemplate.html" />
<EmbeddedResource Include="Exporting\Writers\Html\Core.css" />
<EmbeddedResource Include="Exporting\Writers\Html\Dark.css" />
<EmbeddedResource Include="Exporting\Writers\Html\LayoutTemplate.html" />
<EmbeddedResource Include="Exporting\Writers\Html\Light.css" />
<EmbeddedResource Include="Exporting\Writers\Html\MessageGroupTemplate.html" />
</ItemGroup>
</Project>

8
DiscordChatExporter.Domain/Exceptions/DiscordChatExporterException.cs
View file

@ -1,6 +1,5 @@
using System;
using System.Net.Http;
using DiscordChatExporter.Domain.Discord.Models;
namespace DiscordChatExporter.Domain.Exceptions
{
@ -43,17 +42,14 @@ Failed to perform an HTTP request.
internal static DiscordChatExporterException NotFound()
{
const string message = "Not found.";
const string message = "Requested resource does not exist.";
return new DiscordChatExporterException(message);
}
internal static DiscordChatExporterException ChannelEmpty(string channel)
internal static DiscordChatExporterException ChannelIsEmpty(string channel)
{
var message = $"Channel '{channel}' contains no messages for the specified period.";
return new DiscordChatExporterException(message);
}
internal static DiscordChatExporterException ChannelEmpty(Channel channel) =>
ChannelEmpty(channel.Name);
}
}

115
DiscordChatExporter.Domain/Exporting/ChannelExporter.cs
View file

@ -1,8 +1,6 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Discord;
using DiscordChatExporter.Domain.Discord.Models;
@ -12,7 +10,7 @@ using DiscordChatExporter.Domain.Utilities;
namespace DiscordChatExporter.Domain.Exporting
{
public partial class ChannelExporter
public class ChannelExporter
{
private readonly DiscordClient _discord;
@ -20,50 +18,39 @@ namespace DiscordChatExporter.Domain.Exporting
public ChannelExporter(AuthToken token) : this(new DiscordClient(token)) {}
public async Task ExportAsync(
Guild guild,
Channel channel,
string outputPath,
ExportFormat format,
string dateFormat,
int? partitionLimit,
DateTimeOffset? after = null,
DateTimeOffset? before = null,
IProgress<double>? progress = null)
public async Task ExportChannelAsync(ExportRequest request, IProgress<double>? progress = null)
{
var baseFilePath = GetFilePathFromOutputPath(guild, channel, outputPath, format, after, before);
// Options
var options = new ExportOptions(baseFilePath, format, partitionLimit);
// Context
// Build context
var contextMembers = new HashSet<Member>(IdBasedEqualityComparer.Instance);
var contextChannels = await _discord.GetGuildChannelsAsync(guild.Id);
var contextRoles = await _discord.GetGuildRolesAsync(guild.Id);
var contextChannels = await _discord.GetGuildChannelsAsync(request.Guild.Id);
var contextRoles = await _discord.GetGuildRolesAsync(request.Guild.Id);
var context = new ExportContext(
guild, channel, after, before, dateFormat,
contextMembers, contextChannels, contextRoles
request,
contextMembers,
contextChannels,
contextRoles
);
await using var messageExporter = new MessageExporter(options, context);
// Export messages
await using var messageExporter = new MessageExporter(context);
var exportedAnything = false;
var encounteredUsers = new HashSet<User>(IdBasedEqualityComparer.Instance);
await foreach (var message in _discord.GetMessagesAsync(channel.Id, after, before, progress))
await foreach (var message in _discord.GetMessagesAsync(request.Channel.Id, request.After, request.Before, progress))
{
// Resolve members for referenced users
foreach (var referencedUser in message.MentionedUsers.Prepend(message.Author))
{
if (encounteredUsers.Add(referencedUser))
{
if (!encounteredUsers.Add(referencedUser))
continue;
var member =
await _discord.TryGetGuildMemberAsync(guild.Id, referencedUser) ??
await _discord.TryGetGuildMemberAsync(request.Guild.Id, referencedUser) ??
Member.CreateForUser(referencedUser);
contextMembers.Add(member);
}
}
// Export message
await messageExporter.ExportMessageAsync(message);
@ -72,75 +59,7 @@ namespace DiscordChatExporter.Domain.Exporting
// Throw if no messages were exported
if (!exportedAnything)
throw DiscordChatExporterException.ChannelEmpty(channel);
}
}
public partial class ChannelExporter
{
public static string GetDefaultExportFileName(
Guild guild,
Channel channel,
ExportFormat format,
DateTimeOffset? after = null,
DateTimeOffset? before = null)
{
var buffer = new StringBuilder();
// Guild and channel names
buffer.Append($"{guild.Name} - {channel.Category} - {channel.Name} [{channel.Id}]");
// Date range
if (after != null || before != null)
{
buffer.Append(" (");
// Both 'after' and 'before' are set
if (after != null && before != null)
{
buffer.Append($"{after:yyyy-MM-dd} to {before:yyyy-MM-dd}");
}
// Only 'after' is set
else if (after != null)
{
buffer.Append($"after {after:yyyy-MM-dd}");
}
// Only 'before' is set
else
{
buffer.Append($"before {before:yyyy-MM-dd}");
}
buffer.Append(")");
}
// File extension
buffer.Append($".{format.GetFileExtension()}");
// Replace invalid chars
foreach (var invalidChar in Path.GetInvalidFileNameChars())
buffer.Replace(invalidChar, '_');
return buffer.ToString();
}
private static string GetFilePathFromOutputPath(
Guild guild,
Channel channel,
string outputPath,
ExportFormat format,
DateTimeOffset? after = null,
DateTimeOffset? before = null)
{
// Output is a directory
if (Directory.Exists(outputPath) || string.IsNullOrWhiteSpace(Path.GetExtension(outputPath)))
{
var fileName = GetDefaultExportFileName(guild, channel, format, after, before);
return Path.Combine(outputPath, fileName);
}
// Output is a file
return outputPath;
throw DiscordChatExporterException.ChannelIsEmpty(request.Channel.Name);
}
}
}

57
DiscordChatExporter.Domain/Exporting/ExportContext.cs
View file

@ -1,22 +1,17 @@
using System;
using System.Collections.Generic;
using System.Collections.Generic;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Discord.Models;
namespace DiscordChatExporter.Domain.Exporting
{
public class ExportContext
internal class ExportContext
{
public Guild Guild { get; }
private readonly MediaDownloader _mediaDownloader;
public Channel Channel { get; }
public DateTimeOffset? After { get; }
public DateTimeOffset? Before { get; }
public string DateFormat { get; }
public ExportRequest Request { get; }
public IReadOnlyCollection<Member> Members { get; }
@ -25,46 +20,50 @@ namespace DiscordChatExporter.Domain.Exporting
public IReadOnlyCollection<Role> Roles { get; }
public ExportContext(
Guild guild,
Channel channel,
DateTimeOffset? after,
DateTimeOffset? before,
string dateFormat,
ExportRequest request,
IReadOnlyCollection<Member> members,
IReadOnlyCollection<Channel> channels,
IReadOnlyCollection<Role> roles)
{
Guild = guild;
Channel = channel;
After = after;
Before = before;
DateFormat = dateFormat;
Request = request;
Members = members;
Channels = channels;
Roles = roles;
_mediaDownloader = new MediaDownloader(request.OutputMediaDirPath);
}
public Member? TryGetMentionedMember(string id) =>
public Member? TryGetMember(string id) =>
Members.FirstOrDefault(m => m.Id == id);
public Channel? TryGetMentionedChannel(string id) =>
public Channel? TryGetChannel(string id) =>
Channels.FirstOrDefault(c => c.Id == id);
public Role? TryGetMentionedRole(string id) =>
public Role? TryGetRole(string id) =>
Roles.FirstOrDefault(r => r.Id == id);
public Member? TryGetUserMember(User user) => Members
.FirstOrDefault(m => m.Id == user.Id);
public Color? TryGetUserColor(User user)
{
var member = TryGetUserMember(user);
var member = TryGetMember(user.Id);
var roles = member?.RoleIds.Join(Roles, i => i, r => r.Id, (_, role) => role);
return roles?
.Where(r => r.Color != null)
.OrderByDescending(r => r.Position)
.Select(r => r.Color)
.FirstOrDefault(c => c != null);
.FirstOrDefault();
}
// HACK: ConfigureAwait() is crucial here to enable sync-over-async in HtmlMessageWriter
public async Task<string> ResolveMediaUrlAsync(string url)
{
if (!Request.ShouldDownloadMedia)
return url;
var filePath = await _mediaDownloader.DownloadAsync(url).ConfigureAwait(false);
// Return relative path so that the output files can be copied around without breaking
return Path.GetRelativePath(Request.OutputBaseDirPath, filePath);
}
}
}

18
DiscordChatExporter.Domain/Exporting/ExportOptions.cs
View file

@ -1,18 +0,0 @@
namespace DiscordChatExporter.Domain.Exporting
{
public class ExportOptions
{
public string BaseFilePath { get; }
public ExportFormat Format { get; }
public int? PartitionLimit { get; }
public ExportOptions(string baseFilePath, ExportFormat format, int? partitionLimit)
{
BaseFilePath = baseFilePath;
Format = format;
PartitionLimit = partitionLimit;
}
}
}

136
DiscordChatExporter.Domain/Exporting/ExportRequest.cs Normal file
View file

@ -0,0 +1,136 @@
using System;
using System.IO;
using System.Text;
using DiscordChatExporter.Domain.Discord.Models;
namespace DiscordChatExporter.Domain.Exporting
{
public partial class ExportRequest
{
public Guild Guild { get; }
public Channel Channel { get; }
public string OutputPath { get; }
public string OutputBaseFilePath { get; }
public string OutputBaseDirPath { get; }
public string OutputMediaDirPath { get; }
public ExportFormat Format { get; }
public DateTimeOffset? After { get; }
public DateTimeOffset? Before { get; }
public int? PartitionLimit { get; }
public bool ShouldDownloadMedia { get; }
public string DateFormat { get; }
public ExportRequest(
Guild guild,
Channel channel,
string outputPath,
ExportFormat format,
DateTimeOffset? after,
DateTimeOffset? before,
int? partitionLimit,
bool shouldDownloadMedia,
string dateFormat)
{
Guild = guild;
Channel = channel;
OutputPath = outputPath;
Format = format;
After = after;
Before = before;
PartitionLimit = partitionLimit;
ShouldDownloadMedia = shouldDownloadMedia;
DateFormat = dateFormat;
OutputBaseFilePath = GetOutputBaseFilePath(
guild,
channel,
outputPath,
format,
after,
before
);
OutputBaseDirPath = Path.GetDirectoryName(OutputBaseFilePath) ?? OutputPath;
OutputMediaDirPath = $"{OutputBaseFilePath}_Files{Path.DirectorySeparatorChar}";
}
}
public partial class ExportRequest
{
private static string GetOutputBaseFilePath(
Guild guild,
Channel channel,
string outputPath,
ExportFormat format,
DateTimeOffset? after = null,
DateTimeOffset? before = null)
{
// Output is a directory
if (Directory.Exists(outputPath) || string.IsNullOrWhiteSpace(Path.GetExtension(outputPath)))
{
var fileName = GetDefaultOutputFileName(guild, channel, format, after, before);
return Path.Combine(outputPath, fileName);
}
// Output is a file
return outputPath;
}
public static string GetDefaultOutputFileName(
Guild guild,
Channel channel,
ExportFormat format,
DateTimeOffset? after = null,
DateTimeOffset? before = null)
{
var buffer = new StringBuilder();
// Guild and channel names
buffer.Append($"{guild.Name} - {channel.Category} - {channel.Name} [{channel.Id}]");
// Date range
if (after != null || before != null)
{
buffer.Append(" (");
// Both 'after' and 'before' are set
if (after != null && before != null)
{
buffer.Append($"{after:yyyy-MM-dd} to {before:yyyy-MM-dd}");
}
// Only 'after' is set
else if (after != null)
{
buffer.Append($"after {after:yyyy-MM-dd}");
}
// Only 'before' is set
else
{
buffer.Append($"before {before:yyyy-MM-dd}");
}
buffer.Append(")");
}
// File extension
buffer.Append($".{format.GetFileExtension()}");
// Replace invalid chars
foreach (var invalidChar in Path.GetInvalidFileNameChars())
buffer.Replace(invalidChar, '_');
return buffer.ToString();
}
}
}

47
DiscordChatExporter.Domain/Exporting/MediaDownloader.cs Normal file
View file

@ -0,0 +1,47 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Internal.Extensions;
namespace DiscordChatExporter.Domain.Exporting
{
internal partial class MediaDownloader
{
private readonly HttpClient _httpClient = Singleton.HttpClient;
private readonly string _workingDirPath;
private readonly Dictionary<string, string> _mediaPathMap = new Dictionary<string, string>();
public MediaDownloader(string workingDirPath)
{
_workingDirPath = workingDirPath;
}
// HACK: ConfigureAwait() is crucial here to enable sync-over-async in HtmlMessageWriter
public async Task<string> DownloadAsync(string url)
{
if (_mediaPathMap.TryGetValue(url, out var cachedFilePath))
return cachedFilePath;
Directory.CreateDirectory(_workingDirPath);
var extension = Path.GetExtension(GetFileNameFromUrl(url));
var fileName = $"{Guid.NewGuid()}{extension}";
var filePath = Path.Combine(_workingDirPath, fileName);
await _httpClient.DownloadAsync(url, filePath).ConfigureAwait(false);
return _mediaPathMap[url] = filePath;
}
}
internal partial class MediaDownloader
{
private static string GetFileNameFromUrl(string url) =>
Regex.Match(url, @".+/([^?]*)").Groups[1].Value;
}
}

22
DiscordChatExporter.Domain/Exporting/MessageExporter.cs
View file

@ -8,24 +8,22 @@ namespace DiscordChatExporter.Domain.Exporting
{
internal partial class MessageExporter : IAsyncDisposable
{
private readonly ExportOptions _options;
private readonly ExportContext _context;
private long _renderedMessageCount;
private long _messageCount;
private int _partitionIndex;
private MessageWriter? _writer;
public MessageExporter(ExportOptions options, ExportContext context)
public MessageExporter(ExportContext context)
{
_options = options;
_context = context;
}
private bool IsPartitionLimitReached() =>
_renderedMessageCount > 0 &&
_options.PartitionLimit != null &&
_options.PartitionLimit != 0 &&
_renderedMessageCount % _options.PartitionLimit == 0;
_messageCount > 0 &&
_context.Request.PartitionLimit != null &&
_context.Request.PartitionLimit != 0 &&
_messageCount % _context.Request.PartitionLimit == 0;
private async Task ResetWriterAsync()
{
@ -50,13 +48,13 @@ namespace DiscordChatExporter.Domain.Exporting
if (_writer != null)
return _writer;
var filePath = GetPartitionFilePath(_options.BaseFilePath, _partitionIndex);
var filePath = GetPartitionFilePath(_context.Request.OutputBaseFilePath, _partitionIndex);
var dirPath = Path.GetDirectoryName(_options.BaseFilePath);
var dirPath = Path.GetDirectoryName(_context.Request.OutputBaseFilePath);
if (!string.IsNullOrWhiteSpace(dirPath))
Directory.CreateDirectory(dirPath);
var writer = CreateMessageWriter(filePath, _options.Format, _context);
var writer = CreateMessageWriter(filePath, _context.Request.Format, _context);
await writer.WritePreambleAsync();
return _writer = writer;
@ -66,7 +64,7 @@ namespace DiscordChatExporter.Domain.Exporting
{
var writer = await GetWriterAsync();
await writer.WriteMessageAsync(message);
_renderedMessageCount++;
_messageCount++;
}
public async ValueTask DisposeAsync() => await ResetWriterAsync();

69
DiscordChatExporter.Domain/Exporting/Writers/CsvMessageWriter.cs
View file

@ -1,11 +1,10 @@
using System.IO;
using System.Linq;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors;
using DiscordChatExporter.Domain.Internal;
using Tyrrrz.Extensions;
using DiscordChatExporter.Domain.Internal.Extensions;
namespace DiscordChatExporter.Domain.Exporting.Writers
{
@ -25,19 +24,65 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
public override async Task WritePreambleAsync() =>
await _writer.WriteLineAsync("AuthorID,Author,Date,Content,Attachments,Reactions");
public override async Task WriteMessageAsync(Message message)
private async Task WriteAttachmentsAsync(IReadOnlyList<Attachment> attachments)
{
var buffer = new StringBuilder();
foreach (var attachment in attachments)
{
buffer
.Append(CsvEncode(message.Author.Id)).Append(',')
.Append(CsvEncode(message.Author.FullName)).Append(',')
.Append(CsvEncode(message.Timestamp.ToLocalString(Context.DateFormat))).Append(',')
.Append(CsvEncode(FormatMarkdown(message.Content))).Append(',')
.Append(CsvEncode(message.Attachments.Select(a => a.Url).JoinToString(","))).Append(',')
.Append(CsvEncode(message.Reactions.Select(r => $"{r.Emoji.Name} ({r.Count})").JoinToString(",")));
.AppendIfNotEmpty(',')
.Append(await Context.ResolveMediaUrlAsync(attachment.Url));
}
await _writer.WriteLineAsync(buffer.ToString());
await _writer.WriteAsync(CsvEncode(buffer.ToString()));
}
private async Task WriteReactionsAsync(IReadOnlyList<Reaction> reactions)
{
var buffer = new StringBuilder();
foreach (var reaction in reactions)
{
buffer
.AppendIfNotEmpty(',')
.Append(reaction.Emoji.Name)
.Append(' ')
.Append('(')
.Append(reaction.Count)
.Append(')');
}
await _writer.WriteAsync(CsvEncode(buffer.ToString()));
}
public override async Task WriteMessageAsync(Message message)
{
// Author ID
await _writer.WriteAsync(CsvEncode(message.Author.Id));
await _writer.WriteAsync(',');
// Author name
await _writer.WriteAsync(CsvEncode(message.Author.FullName));
await _writer.WriteAsync(',');
// Message timestamp
await _writer.WriteAsync(CsvEncode(message.Timestamp.ToLocalString(Context.Request.DateFormat)));
await _writer.WriteAsync(',');
// Message content
await _writer.WriteAsync(CsvEncode(FormatMarkdown(message.Content)));
await _writer.WriteAsync(',');
// Attachments
await WriteAttachmentsAsync(message.Attachments);
await _writer.WriteAsync(',');
// Reactions
await WriteReactionsAsync(message.Reactions);
// Finish row
await _writer.WriteLineAsync();
}
public override async ValueTask DisposeAsync()

0
DiscordChatExporter.Domain/Exporting/Resources/HtmlCore.css → DiscordChatExporter.Domain/Exporting/Writers/Html/Core.css
View file

0
DiscordChatExporter.Domain/Exporting/Resources/HtmlDark.css → DiscordChatExporter.Domain/Exporting/Writers/Html/Dark.css
View file

26
DiscordChatExporter.Domain/Exporting/Resources/HtmlLayoutTemplate.html → DiscordChatExporter.Domain/Exporting/Writers/Html/LayoutTemplate.html
View file

@ -3,7 +3,7 @@
<head>
{{~ # Metadata ~}}
<title>{{ Context.Guild.Name | html.escape }} - {{ Context.Channel.Name | html.escape }}</title>
<title>{{ Context.Request.Guild.Name | html.escape }} - {{ Context.Request.Channel.Name | html.escape }}</title>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width">
@ -58,24 +58,24 @@
{{~ # Preamble ~}}
<div class="preamble">
<div class="preamble__guild-icon-container">
<img class="preamble__guild-icon" src="{{ Context.Guild.IconUrl }}" alt="Guild icon">
<img class="preamble__guild-icon" src="{{ Context.Request.Guild.IconUrl | ResolveUrl }}" alt="Guild icon">
</div>
<div class="preamble__entries-container">
<div class="preamble__entry">{{ Context.Guild.Name | html.escape }}</div>
<div class="preamble__entry">{{ Context.Channel.Category | html.escape }} / {{ Context.Channel.Name | html.escape }}</div>
<div class="preamble__entry">{{ Context.Request.Guild.Name | html.escape }}</div>
<div class="preamble__entry">{{ Context.Request.Channel.Category | html.escape }} / {{ Context.Request.Channel.Name | html.escape }}</div>
{{~ if Context.Channel.Topic ~}}
<div class="preamble__entry preamble__entry--small">{{ Context.Channel.Topic | html.escape }}</div>
{{~ if Context.Request.Channel.Topic ~}}
<div class="preamble__entry preamble__entry--small">{{ Context.Request.Channel.Topic | html.escape }}</div>
{{~ end ~}}
{{~ if Context.After || Context.Before ~}}
{{~ if Context.Request.After || Context.Request.Before ~}}
<div class="preamble__entry preamble__entry--small">
{{~ if Context.After && Context.Before ~}}
Between {{ Context.After | FormatDate | html.escape }} and {{ Context.Before | FormatDate | html.escape }}
{{~ else if Context.After ~}}
After {{ Context.After | FormatDate | html.escape }}
{{~ else if Context.Before ~}}
Before {{ Context.Before | FormatDate | html.escape }}
{{~ if Context.Request.After && Context.Request.Before ~}}
Between {{ Context.Request.After | FormatDate | html.escape }} and {{ Context.Request.Before | FormatDate | html.escape }}
{{~ else if Context.Request.After ~}}
After {{ Context.Request.After | FormatDate | html.escape }}
{{~ else if Context.Request.Before ~}}
Before {{ Context.Request.Before | FormatDate | html.escape }}
{{~ end ~}}
</div>
{{~ end ~}}

0
DiscordChatExporter.Domain/Exporting/Resources/HtmlLight.css → DiscordChatExporter.Domain/Exporting/Writers/Html/Light.css
View file

16
DiscordChatExporter.Domain/Exporting/MessageGroup.cs → DiscordChatExporter.Domain/Exporting/Writers/Html/MessageGroup.cs
View file

@ -1,8 +1,9 @@
using System;
using System.Collections.Generic;
using System.Linq;
using DiscordChatExporter.Domain.Discord.Models;
namespace DiscordChatExporter.Domain.Exporting
namespace DiscordChatExporter.Domain.Exporting.Writers.Html
{
// Used for grouping contiguous messages in HTML export
internal partial class MessageGroup
@ -23,9 +24,20 @@ namespace DiscordChatExporter.Domain.Exporting
internal partial class MessageGroup
{
public static bool CanGroup(Message message1, Message message2) =>
public static bool CanJoin(Message message1, Message message2) =>
string.Equals(message1.Author.Id, message2.Author.Id, StringComparison.Ordinal) &&
string.Equals(message1.Author.FullName, message2.Author.FullName, StringComparison.Ordinal) &&
(message2.Timestamp - message1.Timestamp).Duration().TotalMinutes <= 7;
public static MessageGroup Join(IReadOnlyList<Message> messages)
{
var first = messages.First();
return new MessageGroup(
first.Author,
first.Timestamp,
messages
);
}
}
}

24
DiscordChatExporter.Domain/Exporting/Resources/HtmlMessageGroupTemplate.html → DiscordChatExporter.Domain/Exporting/Writers/Html/MessageGroupTemplate.html
View file

@ -1,7 +1,7 @@
<div class="chatlog__message-group">
{{~ # Avatar ~}}
<div class="chatlog__author-avatar-container">
<img class="chatlog__author-avatar" src="{{ MessageGroup.Author.AvatarUrl }}" alt="Avatar">
<img class="chatlog__author-avatar" src="{{ MessageGroup.Author.AvatarUrl | ResolveUrl }}" alt="Avatar">
</div>
<div class="chatlog__messages">
{{~ # Author name and timestamp ~}}
@ -39,16 +39,16 @@
{{~ if attachment.IsSpoiler ~}}
<div class="spoiler spoiler--hidden" onclick="showSpoiler(event, this)">
<div class="spoiler-image">
<a href="{{ attachment.Url }}">
<img class="chatlog__attachment-thumbnail" src="{{ attachment.Url }}" alt="Attachment">
<a href="{{ attachment.Url | ResolveUrl }}">
<img class="chatlog__attachment-thumbnail" src="{{ attachment.Url | ResolveUrl }}" alt="Attachment">
</a>
</div>
</div>
{{~ else ~}}
<a href="{{ attachment.Url }}">
<a href="{{ attachment.Url | ResolveUrl }}">
{{ # Non-spoiler image }}
{{~ if attachment.IsImage ~}}
<img class="chatlog__attachment-thumbnail" src="{{ attachment.Url }}" alt="Attachment">
<img class="chatlog__attachment-thumbnail" src="{{ attachment.Url | ResolveUrl }}" alt="Attachment">
{{~ # Non-image ~}}
{{~ else ~}}
Attachment: {{ attachment.FileName }} ({{ attachment.FileSize }})
@ -73,7 +73,7 @@
{{~ if embed.Author ~}}
<div class="chatlog__embed-author">
{{~ if embed.Author.IconUrl ~}}
<img class="chatlog__embed-author-icon" src="{{ embed.Author.IconUrl }}" alt="Author icon">
<img class="chatlog__embed-author-icon" src="{{ embed.Author.IconUrl | ResolveUrl }}" alt="Author icon">
{{~ end ~}}
{{~ if embed.Author.Name ~}}
@ -124,8 +124,8 @@
{{~ # Thumbnail ~}}
{{~ if embed.Thumbnail ~}}
<div class="chatlog__embed-thumbnail-container">
<a class="chatlog__embed-thumbnail-link" href="{{ embed.Thumbnail.Url }}">
<img class="chatlog__embed-thumbnail" src="{{ embed.Thumbnail.Url }}" alt="Thumbnail">
<a class="chatlog__embed-thumbnail-link" href="{{ embed.Thumbnail.Url | ResolveUrl }}">
<img class="chatlog__embed-thumbnail" src="{{ embed.Thumbnail.Url | ResolveUrl }}" alt="Thumbnail">
</a>
</div>
{{~ end ~}}
@ -134,8 +134,8 @@
{{~ # Image ~}}
{{~ if embed.Image ~}}
<div class="chatlog__embed-image-container">
<a class="chatlog__embed-image-link" href="{{ embed.Image.Url }}">
<img class="chatlog__embed-image" src="{{ embed.Image.Url }}" alt="Image">
<a class="chatlog__embed-image-link" href="{{ embed.Image.Url | ResolveUrl }}">
<img class="chatlog__embed-image" src="{{ embed.Image.Url | ResolveUrl }}" alt="Image">
</a>
</div>
{{~ end ~}}
@ -145,7 +145,7 @@
<div class="chatlog__embed-footer">
{{~ if embed.Footer ~}}
{{~ if embed.Footer.Text && embed.Footer.IconUrl ~}}
<img class="chatlog__embed-footer-icon" src="{{ embed.Footer.IconUrl }}" alt="Footer icon">
<img class="chatlog__embed-footer-icon" src="{{ embed.Footer.IconUrl | ResolveUrl }}" alt="Footer icon">
{{~ end ~}}
{{~ end ~}}
@ -172,7 +172,7 @@
<div class="chatlog__reactions">
{{~ for reaction in message.Reactions ~}}
<div class="chatlog__reaction">
<img class="emoji emoji--small" alt="{{ reaction.Emoji.Name }}" title="{{ reaction.Emoji.Name }}" src="{{ reaction.Emoji.ImageUrl }}">
<img class="emoji emoji--small" alt="{{ reaction.Emoji.Name }}" title="{{ reaction.Emoji.Name }}" src="{{ reaction.Emoji.ImageUrl | ResolveUrl }}">
<span class="chatlog__reaction-count">{{ reaction.Count }}</span>
</div>
{{~ end ~}}

41
DiscordChatExporter.Domain/Exporting/Writers/HtmlMessageWriter.cs
View file

@ -6,8 +6,9 @@ using System.Linq;
using System.Reflection;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exporting.Writers.Html;
using DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Internal.Extensions;
using Scriban;
using Scriban.Runtime;
using Tyrrrz.Extensions;
@ -18,6 +19,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
{
private readonly TextWriter _writer;
private readonly string _themeName;
private readonly List<Message> _messageGroupBuffer = new List<Message>();
private readonly Template _preambleTemplate;
@ -37,12 +39,6 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_postambleTemplate = Template.Parse(GetPostambleTemplateCode());
}
private MessageGroup GetCurrentMessageGroup()
{
var firstMessage = _messageGroupBuffer.First();
return new MessageGroup(firstMessage.Author, firstMessage.Timestamp, _messageGroupBuffer);
}
private TemplateContext CreateTemplateContext(IReadOnlyDictionary<string, object>? constants = null)
{
// Template context
@ -72,7 +68,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
// Functions
scriptObject.Import("FormatDate",
new Func<DateTimeOffset, string>(d => d.ToLocalString(Context.DateFormat)));
new Func<DateTimeOffset, string>(d => d.ToLocalString(Context.Request.DateFormat)));
scriptObject.Import("FormatColorRgb",
new Func<Color?, string?>(c => c != null ? $"rgb({c?.R}, {c?.G}, {c?.B})" : null));
@ -81,7 +77,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
new Func<User, Color?>(Context.TryGetUserColor));
scriptObject.Import("TryGetUserNick",
new Func<User, string?>(u => Context.TryGetUserMember(u)?.Nick));
new Func<User, string?>(u => Context.TryGetMember(u.Id)?.Nick));
scriptObject.Import("FormatMarkdown",
new Func<string?, string>(m => FormatMarkdown(m)));
@ -89,6 +85,11 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
scriptObject.Import("FormatEmbedMarkdown",
new Func<string?, string>(m => FormatMarkdown(m, false)));
// HACK: Scriban doesn't support async, so we have to resort to this and be careful about deadlocks.
// TODO: move to Razor.
scriptObject.Import("ResolveUrl",
new Func<string, string>(u => Context.ResolveMediaUrlAsync(u).GetAwaiter().GetResult()));
// Push model
templateContext.PushGlobal(scriptObject);
@ -101,11 +102,11 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
private string FormatMarkdown(string? markdown, bool isJumboAllowed = true) =>
HtmlMarkdownVisitor.Format(Context, markdown ?? "", isJumboAllowed);
private async Task RenderCurrentMessageGroupAsync()
private async Task WriteCurrentMessageGroupAsync()
{
var templateContext = CreateTemplateContext(new Dictionary<string, object>
{
["MessageGroup"] = GetCurrentMessageGroup()
["MessageGroup"] = MessageGroup.Join(_messageGroupBuffer)
});
await templateContext.EvaluateAsync(_messageGroupTemplate.Page);
@ -120,14 +121,14 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
public override async Task WriteMessageAsync(Message message)
{
// If message group is empty or the given message can be grouped, buffer the given message
if (!_messageGroupBuffer.Any() || MessageGroup.CanGroup(_messageGroupBuffer.Last(), message))
if (!_messageGroupBuffer.Any() || MessageGroup.CanJoin(_messageGroupBuffer.Last(), message))
{
_messageGroupBuffer.Add(message);
}
// Otherwise, flush the group and render messages
else
{
await RenderCurrentMessageGroupAsync();
await WriteCurrentMessageGroupAsync();
_messageGroupBuffer.Clear();
_messageGroupBuffer.Add(message);
@ -141,7 +142,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
{
// Flush current message group
if (_messageGroupBuffer.Any())
await RenderCurrentMessageGroupAsync();
await WriteCurrentMessageGroupAsync();
var templateContext = CreateTemplateContext(new Dictionary<string, object>
{
@ -161,28 +162,28 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
internal partial class HtmlMessageWriter
{
private static readonly Assembly ResourcesAssembly = typeof(HtmlMessageWriter).Assembly;
private static readonly string ResourcesNamespace = $"{ResourcesAssembly.GetName().Name}.Exporting.Resources";
private static readonly string ResourcesNamespace = $"{ResourcesAssembly.GetName().Name}.Exporting.Writers.Html";
private static string GetCoreStyleSheetCode() =>
ResourcesAssembly
.GetManifestResourceString($"{ResourcesNamespace}.HtmlCore.css");
.GetManifestResourceString($"{ResourcesNamespace}.Core.css");
private static string GetThemeStyleSheetCode(string themeName) =>
ResourcesAssembly
.GetManifestResourceString($"{ResourcesNamespace}.Html{themeName}.css");
.GetManifestResourceString($"{ResourcesNamespace}.{themeName}.css");
private static string GetPreambleTemplateCode() =>
ResourcesAssembly
.GetManifestResourceString($"{ResourcesNamespace}.HtmlLayoutTemplate.html")
.GetManifestResourceString($"{ResourcesNamespace}.LayoutTemplate.html")
.SubstringUntil("{{~ %SPLIT% ~}}");
private static string GetMessageGroupTemplateCode() =>
ResourcesAssembly
.GetManifestResourceString($"{ResourcesNamespace}.HtmlMessageGroupTemplate.html");
.GetManifestResourceString($"{ResourcesNamespace}.MessageGroupTemplate.html");
private static string GetPostambleTemplateCode() =>
ResourcesAssembly
.GetManifestResourceString($"{ResourcesNamespace}.HtmlLayoutTemplate.html")
.GetManifestResourceString($"{ResourcesNamespace}.LayoutTemplate.html")
.SubstringAfter("{{~ %SPLIT% ~}}");
}
}

91
DiscordChatExporter.Domain/Exporting/Writers/JsonMessageWriter.cs
View file

@ -3,7 +3,7 @@ using System.Text.Json;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Internal.Extensions;
namespace DiscordChatExporter.Domain.Exporting.Writers
{
@ -25,62 +25,75 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
private string FormatMarkdown(string? markdown) =>
PlainTextMarkdownVisitor.Format(Context, markdown ?? "");
private void WriteAttachment(Attachment attachment)
private async Task WriteAttachmentAsync(Attachment attachment)
{
_writer.WriteStartObject();
_writer.WriteString("id", attachment.Id);
_writer.WriteString("url", attachment.Url);
_writer.WriteString("url", await Context.ResolveMediaUrlAsync(attachment.Url));
_writer.WriteString("fileName", attachment.FileName);
_writer.WriteNumber("fileSizeBytes", attachment.FileSize.TotalBytes);
_writer.WriteEndObject();
await _writer.FlushAsync();
}
private void WriteEmbedAuthor(EmbedAuthor embedAuthor)
private async Task WriteEmbedAuthorAsync(EmbedAuthor embedAuthor)
{
_writer.WriteStartObject("author");
_writer.WriteString("name", embedAuthor.Name);
_writer.WriteString("url", embedAuthor.Url);
_writer.WriteString("iconUrl", embedAuthor.IconUrl);
if (!string.IsNullOrWhiteSpace(embedAuthor.IconUrl))
_writer.WriteString("iconUrl", await Context.ResolveMediaUrlAsync(embedAuthor.IconUrl));
_writer.WriteEndObject();
await _writer.FlushAsync();
}
private void WriteEmbedThumbnail(EmbedImage embedThumbnail)
private async Task WriteEmbedThumbnailAsync(EmbedImage embedThumbnail)
{
_writer.WriteStartObject("thumbnail");
_writer.WriteString("url", embedThumbnail.Url);
if (!string.IsNullOrWhiteSpace(embedThumbnail.Url))
_writer.WriteString("url", await Context.ResolveMediaUrlAsync(embedThumbnail.Url));
_writer.WriteNumber("width", embedThumbnail.Width);
_writer.WriteNumber("height", embedThumbnail.Height);
_writer.WriteEndObject();
await _writer.FlushAsync();
}
private void WriteEmbedImage(EmbedImage embedImage)
private async Task WriteEmbedImageAsync(EmbedImage embedImage)
{
_writer.WriteStartObject("image");
_writer.WriteString("url", embedImage.Url);
if (!string.IsNullOrWhiteSpace(embedImage.Url))
_writer.WriteString("url", await Context.ResolveMediaUrlAsync(embedImage.Url));
_writer.WriteNumber("width", embedImage.Width);
_writer.WriteNumber("height", embedImage.Height);
_writer.WriteEndObject();
await _writer.FlushAsync();
}
private void WriteEmbedFooter(EmbedFooter embedFooter)
private async Task WriteEmbedFooterAsync(EmbedFooter embedFooter)
{
_writer.WriteStartObject("footer");
_writer.WriteString("text", embedFooter.Text);
_writer.WriteString("iconUrl", embedFooter.IconUrl);
if (!string.IsNullOrWhiteSpace(embedFooter.IconUrl))
_writer.WriteString("iconUrl", await Context.ResolveMediaUrlAsync(embedFooter.IconUrl));
_writer.WriteEndObject();
await _writer.FlushAsync();
}
private void WriteEmbedField(EmbedField embedField)
private async Task WriteEmbedFieldAsync(EmbedField embedField)
{
_writer.WriteStartObject();
@ -89,9 +102,10 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer.WriteBoolean("isInline", embedField.IsInline);
_writer.WriteEndObject();
await _writer.FlushAsync();
}
private void WriteEmbed(Embed embed)
private async Task WriteEmbedAsync(Embed embed)
{
_writer.WriteStartObject();
@ -101,29 +115,30 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer.WriteString("description", FormatMarkdown(embed.Description));
if (embed.Author != null)
WriteEmbedAuthor(embed.Author);
await WriteEmbedAuthorAsync(embed.Author);
if (embed.Thumbnail != null)
WriteEmbedThumbnail(embed.Thumbnail);
await WriteEmbedThumbnailAsync(embed.Thumbnail);
if (embed.Image != null)
WriteEmbedImage(embed.Image);
await WriteEmbedImageAsync(embed.Image);
if (embed.Footer != null)
WriteEmbedFooter(embed.Footer);
await WriteEmbedFooterAsync(embed.Footer);
// Fields
_writer.WriteStartArray("fields");
foreach (var field in embed.Fields)
WriteEmbedField(field);
await WriteEmbedFieldAsync(field);
_writer.WriteEndArray();
_writer.WriteEndObject();
await _writer.FlushAsync();
}
private void WriteReaction(Reaction reaction)
private async Task WriteReactionAsync(Reaction reaction)
{
_writer.WriteStartObject();
@ -132,12 +147,13 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer.WriteString("id", reaction.Emoji.Id);
_writer.WriteString("name", reaction.Emoji.Name);
_writer.WriteBoolean("isAnimated", reaction.Emoji.IsAnimated);
_writer.WriteString("imageUrl", reaction.Emoji.ImageUrl);
_writer.WriteString("imageUrl", await Context.ResolveMediaUrlAsync(reaction.Emoji.ImageUrl));
_writer.WriteEndObject();
_writer.WriteNumber("count", reaction.Count);
_writer.WriteEndObject();
await _writer.FlushAsync();
}
public override async Task WritePreambleAsync()
@ -147,29 +163,28 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
// Guild
_writer.WriteStartObject("guild");
_writer.WriteString("id", Context.Guild.Id);
_writer.WriteString("name", Context.Guild.Name);
_writer.WriteString("iconUrl", Context.Guild.IconUrl);
_writer.WriteString("id", Context.Request.Guild.Id);
_writer.WriteString("name", Context.Request.Guild.Name);
_writer.WriteString("iconUrl", await Context.ResolveMediaUrlAsync(Context.Request.Guild.IconUrl));
_writer.WriteEndObject();
// Channel
_writer.WriteStartObject("channel");
_writer.WriteString("id", Context.Channel.Id);
_writer.WriteString("type", Context.Channel.Type.ToString());
_writer.WriteString("category", Context.Channel.Category);
_writer.WriteString("name", Context.Channel.Name);
_writer.WriteString("topic", Context.Channel.Topic);
_writer.WriteString("id", Context.Request.Channel.Id);
_writer.WriteString("type", Context.Request.Channel.Type.ToString());
_writer.WriteString("category", Context.Request.Channel.Category);
_writer.WriteString("name", Context.Request.Channel.Name);
_writer.WriteString("topic", Context.Request.Channel.Topic);
_writer.WriteEndObject();
// Date range
_writer.WriteStartObject("dateRange");
_writer.WriteString("after", Context.After);
_writer.WriteString("before", Context.Before);
_writer.WriteString("after", Context.Request.After);
_writer.WriteString("before", Context.Request.Before);
_writer.WriteEndObject();
// Message array (start)
_writer.WriteStartArray("messages");
await _writer.FlushAsync();
}
@ -193,14 +208,14 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer.WriteString("name", message.Author.Name);
_writer.WriteString("discriminator", $"{message.Author.Discriminator:0000}");
_writer.WriteBoolean("isBot", message.Author.IsBot);
_writer.WriteString("avatarUrl", message.Author.AvatarUrl);
_writer.WriteString("avatarUrl", await Context.ResolveMediaUrlAsync(message.Author.AvatarUrl));
_writer.WriteEndObject();
// Attachments
_writer.WriteStartArray("attachments");
foreach (var attachment in message.Attachments)
WriteAttachment(attachment);
await WriteAttachmentAsync(attachment);
_writer.WriteEndArray();
@ -208,7 +223,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer.WriteStartArray("embeds");
foreach (var embed in message.Embeds)
WriteEmbed(embed);
await WriteEmbedAsync(embed);
_writer.WriteEndArray();
@ -216,15 +231,14 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer.WriteStartArray("reactions");
foreach (var reaction in message.Reactions)
WriteReaction(reaction);
await WriteReactionAsync(reaction);
_writer.WriteEndArray();
_writer.WriteEndObject();
// Flush every 100 messages
if (_messageCount++ % 100 == 0)
await _writer.FlushAsync();
_messageCount++;
}
public override async Task WritePostambleAsync()
@ -236,7 +250,6 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
// Root object (end)
_writer.WriteEndObject();
await _writer.FlushAsync();
}

21
DiscordChatExporter.Domain/Exporting/Writers/MarkdownVisitors/HtmlMarkdownVisitor.cs
View file

@ -4,7 +4,6 @@ using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Markdown;
using DiscordChatExporter.Domain.Markdown.Ast;
@ -23,13 +22,13 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
_isJumbo = isJumbo;
}
public override MarkdownNode VisitText(TextNode text)
protected override MarkdownNode VisitText(TextNode text)
{
_buffer.Append(HtmlEncode(text.Text));
return base.VisitText(text);
}
public override MarkdownNode VisitFormatted(FormattedNode formatted)
protected override MarkdownNode VisitFormatted(FormattedNode formatted)
{
var (tagOpen, tagClose) = formatted.Formatting switch
{
@ -50,7 +49,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
return result;
}
public override MarkdownNode VisitInlineCodeBlock(InlineCodeBlockNode inlineCodeBlock)
protected override MarkdownNode VisitInlineCodeBlock(InlineCodeBlockNode inlineCodeBlock)
{
_buffer
.Append("<span class=\"pre pre--inline\">")
@ -60,7 +59,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
return base.VisitInlineCodeBlock(inlineCodeBlock);
}
public override MarkdownNode VisitMultiLineCodeBlock(MultiLineCodeBlockNode multiLineCodeBlock)
protected override MarkdownNode VisitMultiLineCodeBlock(MultiLineCodeBlockNode multiLineCodeBlock)
{
var highlightCssClass = !string.IsNullOrWhiteSpace(multiLineCodeBlock.Language)
? $"language-{multiLineCodeBlock.Language}"
@ -74,7 +73,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
return base.VisitMultiLineCodeBlock(multiLineCodeBlock);
}
public override MarkdownNode VisitMention(MentionNode mention)
protected override MarkdownNode VisitMention(MentionNode mention)
{
if (mention.Type == MentionType.Meta)
{
@ -85,7 +84,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
}
else if (mention.Type == MentionType.User)
{
var member = _context.TryGetMentionedMember(mention.Id);
var member = _context.TryGetMember(mention.Id);
var fullName = member?.User.FullName ?? "Unknown";
var nick = member?.Nick ?? "Unknown";
@ -96,7 +95,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
}
else if (mention.Type == MentionType.Channel)
{
var channel = _context.TryGetMentionedChannel(mention.Id);
var channel = _context.TryGetChannel(mention.Id);
var name = channel?.Name ?? "deleted-channel";
_buffer
@ -106,7 +105,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
}
else if (mention.Type == MentionType.Role)
{
var role = _context.TryGetMentionedRole(mention.Id);
var role = _context.TryGetRole(mention.Id);
var name = role?.Name ?? "deleted-role";
var color = role?.Color;
@ -123,7 +122,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
return base.VisitMention(mention);
}
public override MarkdownNode VisitEmoji(EmojiNode emoji)
protected override MarkdownNode VisitEmoji(EmojiNode emoji)
{
var emojiImageUrl = Emoji.GetImageUrl(emoji.Id, emoji.Name, emoji.IsAnimated);
var jumboClass = _isJumbo ? "emoji--large" : "";
@ -134,7 +133,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
return base.VisitEmoji(emoji);
}
public override MarkdownNode VisitLink(LinkNode link)
protected override MarkdownNode VisitLink(LinkNode link)
{
// Extract message ID if the link points to a Discord message
var linkedMessageId = Regex.Match(link.Url, "^https?://discordapp.com/channels/.*?/(\\d+)/?$").Groups[1].Value;

12
DiscordChatExporter.Domain/Exporting/Writers/MarkdownVisitors/PlainTextMarkdownVisitor.cs
View file

@ -15,13 +15,13 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
_buffer = buffer;
}
public override MarkdownNode VisitText(TextNode text)
protected override MarkdownNode VisitText(TextNode text)
{
_buffer.Append(text.Text);
return base.VisitText(text);
}
public override MarkdownNode VisitMention(MentionNode mention)
protected override MarkdownNode VisitMention(MentionNode mention)
{
if (mention.Type == MentionType.Meta)
{
@ -29,21 +29,21 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
}
else if (mention.Type == MentionType.User)
{
var member = _context.TryGetMentionedMember(mention.Id);
var member = _context.TryGetMember(mention.Id);
var name = member?.User.Name ?? "Unknown";
_buffer.Append($"@{name}");
}
else if (mention.Type == MentionType.Channel)
{
var channel = _context.TryGetMentionedChannel(mention.Id);
var channel = _context.TryGetChannel(mention.Id);
var name = channel?.Name ?? "deleted-channel";
_buffer.Append($"#{name}");
}
else if (mention.Type == MentionType.Role)
{
var role = _context.TryGetMentionedRole(mention.Id);
var role = _context.TryGetRole(mention.Id);
var name = role?.Name ?? "deleted-role";
_buffer.Append($"@{name}");
@ -52,7 +52,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
return base.VisitMention(mention);
}
public override MarkdownNode VisitEmoji(EmojiNode emoji)
protected override MarkdownNode VisitEmoji(EmojiNode emoji)
{
_buffer.Append(
emoji.IsCustomEmoji

170
DiscordChatExporter.Domain/Exporting/Writers/PlainTextMessageWriter.cs
View file

@ -1,12 +1,11 @@
using System;
using System.Collections.Generic;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Internal.Extensions;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Domain.Exporting.Writers
{
@ -25,135 +24,124 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
private string FormatMarkdown(string? markdown) =>
PlainTextMarkdownVisitor.Format(Context, markdown ?? "");
private string FormatMessageHeader(Message message)
private async Task WriteMessageHeaderAsync(Message message)
{
var buffer = new StringBuilder();
// Timestamp & author
buffer
.Append($"[{message.Timestamp.ToLocalString(Context.DateFormat)}]")
.Append(' ')
.Append($"{message.Author.FullName}");
await _writer.WriteAsync($"[{message.Timestamp.ToLocalString(Context.Request.DateFormat)}]");
await _writer.WriteAsync($" {message.Author.FullName}");
// Whether the message is pinned
if (message.IsPinned)
buffer.Append(' ').Append("(pinned)");
await _writer.WriteAsync(" (pinned)");
return buffer.ToString();
await _writer.WriteLineAsync();
}
private string FormatAttachments(IReadOnlyList<Attachment> attachments)
private async Task WriteAttachmentsAsync(IReadOnlyList<Attachment> attachments)
{
if (!attachments.Any())
return "";
return;
var buffer = new StringBuilder();
await _writer.WriteLineAsync("{Attachments}");
buffer
.AppendLine("{Attachments}")
.AppendJoin(Environment.NewLine, attachments.Select(a => a.Url))
.AppendLine();
foreach (var attachment in attachments)
await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(attachment.Url));
return buffer.ToString();
await _writer.WriteLineAsync();
}
private string FormatEmbeds(IReadOnlyList<Embed> embeds)
private async Task WriteEmbedsAsync(IReadOnlyList<Embed> embeds)
{
if (!embeds.Any())
return "";
var buffer = new StringBuilder();
foreach (var embed in embeds)
{
buffer
.AppendLine("{Embed}")
.AppendLineIfNotNullOrWhiteSpace(embed.Author?.Name)
.AppendLineIfNotNullOrWhiteSpace(embed.Url)
.AppendLineIfNotNullOrWhiteSpace(FormatMarkdown(embed.Title))
.AppendLineIfNotNullOrWhiteSpace(FormatMarkdown(embed.Description));
await _writer.WriteLineAsync("{Embed}");
if (!string.IsNullOrWhiteSpace(embed.Author?.Name))
await _writer.WriteLineAsync(embed.Author.Name);
if (!string.IsNullOrWhiteSpace(embed.Url))
await _writer.WriteLineAsync(embed.Url);
if (!string.IsNullOrWhiteSpace(embed.Title))
await _writer.WriteLineAsync(FormatMarkdown(embed.Title));
if (!string.IsNullOrWhiteSpace(embed.Description))
await _writer.WriteLineAsync(FormatMarkdown(embed.Description));
foreach (var field in embed.Fields)
{
buffer
.AppendLineIfNotNullOrWhiteSpace(FormatMarkdown(field.Name))
.AppendLineIfNotNullOrWhiteSpace(FormatMarkdown(field.Value));
if (!string.IsNullOrWhiteSpace(field.Name))
await _writer.WriteLineAsync(FormatMarkdown(field.Name));
if (!string.IsNullOrWhiteSpace(field.Value))
await _writer.WriteLineAsync(FormatMarkdown(field.Value));
}
buffer
.AppendLineIfNotNullOrWhiteSpace(embed.Thumbnail?.Url)
.AppendLineIfNotNullOrWhiteSpace(embed.Image?.Url)
.AppendLineIfNotNullOrWhiteSpace(embed.Footer?.Text)
.AppendLine();
if (!string.IsNullOrWhiteSpace(embed.Thumbnail?.Url))
await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(embed.Thumbnail.Url));
if (!string.IsNullOrWhiteSpace(embed.Image?.Url))
await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(embed.Image.Url));
if (!string.IsNullOrWhiteSpace(embed.Footer?.Text))
await _writer.WriteLineAsync(embed.Footer.Text);
await _writer.WriteLineAsync();
}
}
return buffer.ToString();
}
private string FormatReactions(IReadOnlyList<Reaction> reactions)
private async Task WriteReactionsAsync(IReadOnlyList<Reaction> reactions)
{
if (!reactions.Any())
return "";
return;
var buffer = new StringBuilder();
buffer.AppendLine("{Reactions}");
await _writer.WriteLineAsync("{Reactions}");
foreach (var reaction in reactions)
{
buffer.Append(reaction.Emoji.Name);
await _writer.WriteAsync(reaction.Emoji.Name);
if (reaction.Count > 1)
buffer.Append($" ({reaction.Count})");
await _writer.WriteAsync($" ({reaction.Count})");
buffer.Append(" ");
await _writer.WriteAsync(' ');
}
buffer.AppendLine();
return buffer.ToString();
}
private string FormatMessage(Message message)
{
var buffer = new StringBuilder();
buffer
.AppendLine(FormatMessageHeader(message))
.AppendLineIfNotNullOrWhiteSpace(FormatMarkdown(message.Content))
.AppendLine()
.AppendLineIfNotNullOrWhiteSpace(FormatAttachments(message.Attachments))
.AppendLineIfNotNullOrWhiteSpace(FormatEmbeds(message.Embeds))
.AppendLineIfNotNullOrWhiteSpace(FormatReactions(message.Reactions));
return buffer.Trim().ToString();
await _writer.WriteLineAsync();
}
public override async Task WritePreambleAsync()
{
var buffer = new StringBuilder();
await _writer.WriteLineAsync('='.Repeat(62));
await _writer.WriteLineAsync($"Guild: {Context.Request.Guild.Name}");
await _writer.WriteLineAsync($"Channel: {Context.Request.Channel.Category} / {Context.Request.Channel.Name}");
buffer.Append('=', 62).AppendLine();
buffer.AppendLine($"Guild: {Context.Guild.Name}");
buffer.AppendLine($"Channel: {Context.Channel.Category} / {Context.Channel.Name}");
if (!string.IsNullOrWhiteSpace(Context.Request.Channel.Topic))
await _writer.WriteLineAsync($"Topic: {Context.Request.Channel.Topic}");
if (!string.IsNullOrWhiteSpace(Context.Channel.Topic))
buffer.AppendLine($"Topic: {Context.Channel.Topic}");
if (Context.Request.After != null)
await _writer.WriteLineAsync($"After: {Context.Request.After.Value.ToLocalString(Context.Request.DateFormat)}");
if (Context.After != null)
buffer.AppendLine($"After: {Context.After.Value.ToLocalString(Context.DateFormat)}");
if (Context.Request.Before != null)
await _writer.WriteLineAsync($"Before: {Context.Request.Before.Value.ToLocalString(Context.Request.DateFormat)}");
if (Context.Before != null)
buffer.AppendLine($"Before: {Context.Before.Value.ToLocalString(Context.DateFormat)}");
buffer.Append('=', 62).AppendLine();
await _writer.WriteLineAsync(buffer.ToString());
await _writer.WriteLineAsync('='.Repeat(62));
await _writer.WriteLineAsync();
}
public override async Task WriteMessageAsync(Message message)
{
await _writer.WriteLineAsync(FormatMessage(message));
await WriteMessageHeaderAsync(message);
if (!string.IsNullOrWhiteSpace(message.Content))
await _writer.WriteLineAsync(FormatMarkdown(message.Content));
await _writer.WriteLineAsync();
await WriteAttachmentsAsync(message.Attachments);
await WriteEmbedsAsync(message.Embeds);
await WriteReactionsAsync(message.Reactions);
await _writer.WriteLineAsync();
_messageCount++;
@ -161,15 +149,9 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
public override async Task WritePostambleAsync()
{
var buffer = new StringBuilder();
buffer
.Append('=', 62).AppendLine()
.AppendLine($"Exported {_messageCount:N0} message(s)")
.Append('=', 62).AppendLine()
.AppendLine();
await _writer.WriteLineAsync(buffer.ToString());
await _writer.WriteLineAsync('='.Repeat(62));
await _writer.WriteLineAsync($"Exported {_messageCount:N0} message(s)");
await _writer.WriteLineAsync('='.Repeat(62));
}
public override async ValueTask DisposeAsync()

2
DiscordChatExporter.Domain/Internal/ColorExtensions.cs → DiscordChatExporter.Domain/Internal/Extensions/ColorExtensions.cs
View file

@ -1,6 +1,6 @@
using System.Drawing;
namespace DiscordChatExporter.Domain.Internal
namespace DiscordChatExporter.Domain.Internal.Extensions
{
internal static class ColorExtensions
{

2
DiscordChatExporter.Domain/Internal/DateExtensions.cs → DiscordChatExporter.Domain/Internal/Extensions/DateExtensions.cs
View file

@ -1,7 +1,7 @@
using System;
using System.Globalization;
namespace DiscordChatExporter.Domain.Internal
namespace DiscordChatExporter.Domain.Internal.Extensions
{
internal static class DateExtensions
{

2
DiscordChatExporter.Domain/Internal/GenericExtensions.cs → DiscordChatExporter.Domain/Internal/Extensions/GenericExtensions.cs
View file

@ -1,6 +1,6 @@
using System;
namespace DiscordChatExporter.Domain.Internal
namespace DiscordChatExporter.Domain.Internal.Extensions
{
internal static class GenericExtensions
{

28
DiscordChatExporter.Domain/Internal/Extensions/HttpClientExtensions.cs Normal file
View file

@ -0,0 +1,28 @@
using System.IO;
using System.Net.Http;
using System.Text.Json;
using System.Threading.Tasks;
namespace DiscordChatExporter.Domain.Internal.Extensions
{
internal static class HttpClientExtensions
{
// HACK: ConfigureAwait() is crucial here to enable sync-over-async in HtmlMessageWriter
public static async Task DownloadAsync(this HttpClient httpClient, string uri, string outputFilePath)
{
await using var input = await httpClient.GetStreamAsync(uri).ConfigureAwait(false);
var output = File.Create(outputFilePath);
await input.CopyToAsync(output).ConfigureAwait(false);
await output.DisposeAsync().ConfigureAwait(false);
}
public static async Task<JsonElement> ReadAsJsonAsync(this HttpContent content)
{
await using var stream = await content.ReadAsStreamAsync();
using var doc = await JsonDocument.ParseAsync(stream);
return doc.RootElement.Clone();
}
}
}

2
DiscordChatExporter.Domain/Internal/JsonElementExtensions.cs → DiscordChatExporter.Domain/Internal/Extensions/JsonElementExtensions.cs
View file

@ -1,6 +1,6 @@
using System.Text.Json;
namespace DiscordChatExporter.Domain.Internal
namespace DiscordChatExporter.Domain.Internal.Extensions
{
internal static class JsonElementExtensions
{

12
DiscordChatExporter.Domain/Internal/Extensions/StringExtensions.cs Normal file
View file

@ -0,0 +1,12 @@
using System.Text;
namespace DiscordChatExporter.Domain.Internal.Extensions
{
internal static class StringExtensions
{
public static StringBuilder AppendIfNotEmpty(this StringBuilder builder, char value) =>
builder.Length > 0
? builder.Append(value)
: builder;
}
}

2
DiscordChatExporter.Domain/Internal/Utf8JsonWriterExtensions.cs → DiscordChatExporter.Domain/Internal/Extensions/Utf8JsonWriterExtensions.cs
View file

@ -1,7 +1,7 @@
using System;
using System.Text.Json;
namespace DiscordChatExporter.Domain.Internal
namespace DiscordChatExporter.Domain.Internal.Extensions
{
internal static class Utf8JsonWriterExtensions
{

17
DiscordChatExporter.Domain/Internal/HttpClientExtensions.cs
View file

@ -1,17 +0,0 @@
using System.Net.Http;
using System.Text.Json;
using System.Threading.Tasks;
namespace DiscordChatExporter.Domain.Internal
{
internal static class HttpClientExtensions
{
public static async Task<JsonElement> ReadAsJsonAsync(this HttpContent content)
{
await using var stream = await content.ReadAsStreamAsync();
using var doc = await JsonDocument.ParseAsync(stream);
return doc.RootElement.Clone();
}
}
}

23
DiscordChatExporter.Domain/Internal/Singleton.cs Normal file
View file

@ -0,0 +1,23 @@
using System;
using System.Net;
using System.Net.Http;
namespace DiscordChatExporter.Domain.Internal
{
internal static class Singleton
{
private static readonly Lazy<HttpClient> LazyHttpClient = new Lazy<HttpClient>(() =>
{
var handler = new HttpClientHandler();
if (handler.SupportsAutomaticDecompression)
handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
handler.UseCookies = false;
return new HttpClient(handler, true);
});
public static HttpClient HttpClient { get; } = LazyHttpClient.Value;
}
}

21
DiscordChatExporter.Domain/Internal/StringExtensions.cs
View file

@ -1,21 +0,0 @@
using System.Text;
namespace DiscordChatExporter.Domain.Internal
{
internal static class StringExtensions
{
public static StringBuilder AppendLineIfNotNullOrWhiteSpace(this StringBuilder builder, string? value) =>
!string.IsNullOrWhiteSpace(value) ? builder.AppendLine(value) : builder;
public static StringBuilder Trim(this StringBuilder builder)
{
while (builder.Length > 0 && char.IsWhiteSpace(builder[0]))
builder.Remove(0, 1);
while (builder.Length > 0 && char.IsWhiteSpace(builder[^1]))
builder.Remove(builder.Length - 1, 1);
return builder;
}
}
}

12
DiscordChatExporter.Domain/Discord/UrlBuilder.cs → DiscordChatExporter.Domain/Internal/UrlBuilder.cs
View file

@ -4,7 +4,7 @@ using System.Linq;
using System.Net;
using System.Text;
namespace DiscordChatExporter.Domain.Discord
namespace DiscordChatExporter.Domain.Internal
{
internal class UrlBuilder
{
@ -19,8 +19,11 @@ namespace DiscordChatExporter.Domain.Discord
return this;
}
public UrlBuilder SetQueryParameter(string key, string? value)
public UrlBuilder SetQueryParameter(string key, string? value, bool ignoreUnsetValue = true)
{
if (ignoreUnsetValue && string.IsNullOrWhiteSpace(value))
return this;
var keyEncoded = WebUtility.UrlEncode(key);
var valueEncoded = WebUtility.UrlEncode(value);
_queryParameters[keyEncoded] = valueEncoded;
@ -28,11 +31,6 @@ namespace DiscordChatExporter.Domain.Discord
return this;
}
public UrlBuilder SetQueryParameterIfNotNullOrWhiteSpace(string key, string? value) =>
!string.IsNullOrWhiteSpace(value)
? SetQueryParameter(key, value)
: this;
public string Build()
{
var buffer = new StringBuilder();

14
DiscordChatExporter.Domain/Markdown/MarkdownVisitor.cs
View file

@ -6,23 +6,23 @@ namespace DiscordChatExporter.Domain.Markdown
{
internal abstract class MarkdownVisitor
{
public virtual MarkdownNode VisitText(TextNode text) => text;
protected virtual MarkdownNode VisitText(TextNode text) => text;
public virtual MarkdownNode VisitFormatted(FormattedNode formatted)
protected virtual MarkdownNode VisitFormatted(FormattedNode formatted)
{
Visit(formatted.Children);
return formatted;
}
public virtual MarkdownNode VisitInlineCodeBlock(InlineCodeBlockNode inlineCodeBlock) => inlineCodeBlock;
protected virtual MarkdownNode VisitInlineCodeBlock(InlineCodeBlockNode inlineCodeBlock) => inlineCodeBlock;
public virtual MarkdownNode VisitMultiLineCodeBlock(MultiLineCodeBlockNode multiLineCodeBlock) => multiLineCodeBlock;
protected virtual MarkdownNode VisitMultiLineCodeBlock(MultiLineCodeBlockNode multiLineCodeBlock) => multiLineCodeBlock;
public virtual MarkdownNode VisitLink(LinkNode link) => link;
protected virtual MarkdownNode VisitLink(LinkNode link) => link;
public virtual MarkdownNode VisitEmoji(EmojiNode emoji) => emoji;
protected virtual MarkdownNode VisitEmoji(EmojiNode emoji) => emoji;
public virtual MarkdownNode VisitMention(MentionNode mention) => mention;
protected virtual MarkdownNode VisitMention(MentionNode mention) => mention;
public MarkdownNode Visit(MarkdownNode node) => node switch
{

2
DiscordChatExporter.Gui/DiscordChatExporter.Gui.csproj
View file

@ -19,7 +19,7 @@
<PackageReference Include="Microsoft.Xaml.Behaviors.Wpf" Version="1.1.19" />
<PackageReference Include="Ookii.Dialogs.Wpf" Version="1.1.0" />
<PackageReference Include="Onova" Version="2.6.0" />
<PackageReference Include="Stylet" Version="1.3.2" />
<PackageReference Include="Stylet" Version="1.3.3" />
<PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" />
<PackageReference Include="Tyrrrz.Settings" Version="1.3.4" />
<PackageReference Include="PropertyChanged.Fody" Version="3.2.8" PrivateAssets="all" />

2
DiscordChatExporter.Gui/Services/SettingsService.cs
View file

@ -20,6 +20,8 @@ namespace DiscordChatExporter.Gui.Services
public int? LastPartitionLimit { get; set; }
public bool LastShouldDownloadMedia { get; set; }
public SettingsService()
{
Configuration.StorageSpace = StorageSpace.Instance;

22
DiscordChatExporter.Gui/ViewModels/Dialogs/ExportSetupViewModel.cs
View file

@ -32,6 +32,16 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs
public int? PartitionLimit { get; set; }
public bool ShouldDownloadMedia { get; set; }
// Whether to show the "advanced options" by default when the dialog opens.
// This is active if any of the advanced options are set to non-default values.
public bool IsAdvancedSectionDisplayedByDefault =>
After != default ||
Before != default ||
PartitionLimit != default ||
ShouldDownloadMedia != default;
public ExportSetupViewModel(DialogManager dialogManager, SettingsService settingsService)
{
_dialogManager = dialogManager;
@ -40,6 +50,7 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs
// Persist preferences
SelectedFormat = _settingsService.LastExportFormat;
PartitionLimit = _settingsService.LastPartitionLimit;
ShouldDownloadMedia = _settingsService.LastShouldDownloadMedia;
}
public void Confirm()
@ -47,6 +58,7 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs
// Persist preferences
_settingsService.LastExportFormat = SelectedFormat;
_settingsService.LastPartitionLimit = PartitionLimit;
_settingsService.LastShouldDownloadMedia = ShouldDownloadMedia;
// Clamp 'after' and 'before' values
if (After > Before)
@ -57,23 +69,18 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs
// If single channel - prompt file path
if (IsSingleChannel)
{
// Get single channel
var channel = Channels.Single();
var defaultFileName = ExportRequest.GetDefaultOutputFileName(Guild!, channel, SelectedFormat, After, Before);
// Generate default file name
var defaultFileName = ChannelExporter.GetDefaultExportFileName(Guild!, channel, SelectedFormat, After, Before);
// Generate filter
// Filter
var ext = SelectedFormat.GetFileExtension();
var filter = $"{ext.ToUpperInvariant()} files|*.{ext}";
// Prompt user
OutputPath = _dialogManager.PromptSaveFilePath(filter, defaultFileName);
}
// If multiple channels - prompt dir path
else
{
// Prompt user
OutputPath = _dialogManager.PromptDirectoryPath();
}
@ -81,7 +88,6 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs
if (string.IsNullOrWhiteSpace(OutputPath))
return;
// Close dialog
Close(true);
}
}

16
DiscordChatExporter.Gui/ViewModels/RootViewModel.cs
View file

@ -190,9 +190,19 @@ namespace DiscordChatExporter.Gui.ViewModels
try
{
await exporter.ExportAsync(dialog.Guild!, channel!,
dialog.OutputPath!, dialog.SelectedFormat, _settingsService.DateFormat,
dialog.PartitionLimit, dialog.After, dialog.Before, operation);
var request = new ExportRequest(
dialog.Guild!,
channel!,
dialog.OutputPath!,
dialog.SelectedFormat,
dialog.After,
dialog.Before,
dialog.PartitionLimit,
dialog.ShouldDownloadMedia,
_settingsService.DateFormat
);
await exporter.ExportChannelAsync(request, operation);
Interlocked.Increment(ref successfulExportCount);
}

50
DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml
View file

@ -74,6 +74,8 @@
</ComboBox.ItemTemplate>
</ComboBox>
<!-- Advanced section -->
<StackPanel Visibility="{Binding IsChecked, ElementName=AdvancedSectionToggleButton, Converter={x:Static s:BoolToVisibilityConverter.Instance}}">
<!-- Date limits -->
<Grid>
<Grid.ColumnDefinitions>
@ -108,20 +110,64 @@
Text="{Binding PartitionLimit, TargetNullValue=''}"
ToolTip="If this is set, the exported file will be split into multiple partitions, each containing no more than specified number of messages" />
<!-- Download media -->
<Grid Margin="16,16" ToolTip="If this is set, the export will include additional files such as user avatars, attached files, embedded images, etc">
<Grid.ColumnDefinitions>
<ColumnDefinition Width="*" />
<ColumnDefinition Width="Auto" />
</Grid.ColumnDefinitions>
<TextBlock
Grid.Column="0"
VerticalAlignment="Center"
Text="Download referenced media content" />
<ToggleButton
Grid.Column="1"
HorizontalAlignment="Right"
VerticalAlignment="Center"
IsChecked="{Binding ShouldDownloadMedia}" />
</Grid>
</StackPanel>
<!-- Buttons -->
<StackPanel HorizontalAlignment="Right" Orientation="Horizontal">
<Grid>
<Grid.ColumnDefinitions>
<ColumnDefinition Width="Auto" />
<ColumnDefinition Width="*" />
<ColumnDefinition Width="Auto" />
<ColumnDefinition Width="Auto" />
</Grid.ColumnDefinitions>
<ToggleButton
x:Name="AdvancedSectionToggleButton"
Grid.Column="0"
Margin="8"
Cursor="Hand"
Loaded="AdvancedSectionToggleButton_OnLoaded"
Style="{DynamicResource MaterialDesignFlatToggleButton}"
ToolTip="Show advanced options">
<ToggleButton.Content>
<materialDesign:PackIcon
Width="24"
Height="24"
Kind="Menu" />
</ToggleButton.Content>
</ToggleButton>
<Button
Grid.Column="2"
Margin="8"
Command="{s:Action Confirm}"
Content="EXPORT"
IsDefault="True"
Style="{DynamicResource MaterialDesignFlatButton}" />
<Button
Grid.Column="3"
Margin="8"
Command="{s:Action Close}"
Content="CANCEL"
IsCancel="True"
Style="{DynamicResource MaterialDesignFlatButton}" />
</StackPanel>
</Grid>
</StackPanel>
</UserControl>

11
DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml.cs
View file

@ -1,4 +1,7 @@
namespace DiscordChatExporter.Gui.Views.Dialogs
using System.Windows;
using DiscordChatExporter.Gui.ViewModels.Dialogs;
namespace DiscordChatExporter.Gui.Views.Dialogs
{
public partial class ExportSetupView
{
@ -6,5 +9,11 @@
{
InitializeComponent();
}
private void AdvancedSectionToggleButton_OnLoaded(object sender, RoutedEventArgs e)
{
if (DataContext is ExportSetupViewModel vm)
AdvancedSectionToggleButton.IsChecked = vm.IsAdvancedSectionDisplayedByDefault;
}
}
}