mirrors/DiscordChatExporter
1
0
Fork 0
mirror of https://github.com/Tyrrrz/DiscordChatExporter.git synced 2025-06-05 01:03:44 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Rework architecture

Browse source
This commit is contained in:
Alexey Golub 2020-04-21 21:30:42 +03:00
parent 130c0b6fe2
commit 8685a3d7e3
119 changed files with 1520 additions and 1560 deletions
Download patch file Download diff file Expand all files Collapse all files

32
DiscordChatExporter.Cli/Commands/ExportCommandBase.cs → DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs
View file

@ -4,17 +4,13 @@ using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using CliFx.Utilities;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exporting;
namespace DiscordChatExporter.Cli.Commands
namespace DiscordChatExporter.Cli.Commands.Base
{
public abstract class ExportCommandBase : TokenCommandBase
{
protected SettingsService SettingsService { get; }
protected ExportService ExportService { get; }
[CommandOption("format", 'f', Description = "Output file format.")]
public ExportFormat ExportFormat { get; set; } = ExportFormat.HtmlDark;
@ -31,25 +27,17 @@ namespace DiscordChatExporter.Cli.Commands
public int? PartitionLimit { get; set; }
[CommandOption("dateformat", Description = "Date format used in output.")]
public string? DateFormat { get; set; }
public string DateFormat { get; set; } = "dd-MMM-yy hh:mm tt";
protected ExportCommandBase(SettingsService settingsService, DataService dataService, ExportService exportService)
: base(dataService)
{
SettingsService = settingsService;
ExportService = exportService;
}
protected Exporter GetExporter() => new Exporter(GetDiscordClient());
protected async ValueTask ExportAsync(IConsole console, Guild guild, Channel channel)
{
if (!string.IsNullOrWhiteSpace(DateFormat))
SettingsService.DateFormat = DateFormat;
console.Output.Write($"Exporting channel [{channel.Name}]... ");
console.Output.Write($"Exporting channel '{channel.Name}'... ");
var progress = console.CreateProgressTicker();
await ExportService.ExportChatLogAsync(Token, guild, channel,
OutputPath, ExportFormat, PartitionLimit,
await GetExporter().ExportChatLogAsync(guild, channel,
OutputPath, ExportFormat, DateFormat, PartitionLimit,
After, Before, progress);
console.Output.WriteLine();
@ -58,13 +46,13 @@ namespace DiscordChatExporter.Cli.Commands
protected async ValueTask ExportAsync(IConsole console, Channel channel)
{
var guild = await DataService.GetGuildAsync(Token, channel.GuildId);
var guild = await GetDiscordClient().GetGuildAsync(channel.GuildId);
await ExportAsync(console, guild, channel);
}
protected async ValueTask ExportAsync(IConsole console, string channelId)
{
var channel = await DataService.GetChannelAsync(Token, channelId);
var channel = await GetDiscordClient().GetChannelAsync(channelId);
await ExportAsync(console, channel);
}
}

50
DiscordChatExporter.Cli/Commands/ExportMultipleCommandBase.cs → DiscordChatExporter.Cli/Commands/Base/ExportMultipleCommandBase.cs
View file

@ -1,38 +1,28 @@
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using CliFx.Utilities;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Models.Exceptions;
using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Core.Services.Exceptions;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exceptions;
using DiscordChatExporter.Domain.Utilities;
using Gress;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Cli.Commands
namespace DiscordChatExporter.Cli.Commands.Base
{
public abstract class ExportMultipleCommandBase : ExportCommandBase
{
[CommandOption("parallel", Description = "Export this number of separate channels in parallel.")]
public int ParallelLimit { get; set; } = 1;
protected ExportMultipleCommandBase(SettingsService settingsService, DataService dataService, ExportService exportService)
: base(settingsService, dataService, exportService)
{
}
protected async ValueTask ExportMultipleAsync(IConsole console, IReadOnlyList<Channel> channels)
{
// This uses a separate route from ExportCommandBase because the progress ticker is not thread-safe
// Ugly code ahead. Will need to refactor.
if (!string.IsNullOrWhiteSpace(DateFormat))
SettingsService.DateFormat = DateFormat;
// Progress
console.Output.Write($"Exporting {channels.Count} channels... ");
var ticker = console.CreateProgressTicker();
@ -44,41 +34,33 @@ namespace DiscordChatExporter.Cli.Commands
var operations = progressManager.CreateOperations(channels.Count);
// Export channels
using var semaphore = new SemaphoreSlim(ParallelLimit.ClampMin(1));
var errors = new List<string>();
await Task.WhenAll(channels.Select(async (channel, i) =>
var successfulExportCount = 0;
await channels.Zip(operations).ParallelForEachAsync(async tuple =>
{
var operation = operations[i];
await semaphore.WaitAsync();
var guild = await DataService.GetGuildAsync(Token, channel.GuildId);
var (channel, operation) = tuple;
try
{
await ExportService.ExportChatLogAsync(Token, guild, channel,
OutputPath, ExportFormat, PartitionLimit,
var guild = await GetDiscordClient().GetGuildAsync(channel.GuildId);
await GetExporter().ExportChatLogAsync(guild, channel,
OutputPath, ExportFormat, DateFormat, PartitionLimit,
After, Before, operation);
Interlocked.Increment(ref successfulExportCount);
}
catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.Forbidden)
{
errors.Add("You don't have access to this channel.");
}
catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
{
errors.Add("This channel doesn't exist.");
}
catch (DomainException ex)
catch (DiscordChatExporterException ex) when (!ex.IsCritical)
{
errors.Add(ex.Message);
}
finally
{
semaphore.Release();
operation.Dispose();
}
}));
}, ParallelLimit.ClampMin(1));
ticker.Report(1);
console.Output.WriteLine();
@ -86,7 +68,7 @@ namespace DiscordChatExporter.Cli.Commands
foreach (var error in errors)
console.Error.WriteLine(error);
console.Output.WriteLine("Done.");
console.Output.WriteLine($"Successfully exported {successfulExportCount} channel(s).");
}
}
}

14
DiscordChatExporter.Cli/Commands/TokenCommandBase.cs → DiscordChatExporter.Cli/Commands/Base/TokenCommandBase.cs
View file

@ -1,15 +1,12 @@
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Domain.Discord;
namespace DiscordChatExporter.Cli.Commands
namespace DiscordChatExporter.Cli.Commands.Base
{
public abstract class TokenCommandBase : ICommand
{
protected DataService DataService { get; }
[CommandOption("token", 't', IsRequired = true, EnvironmentVariableName = "DISCORD_TOKEN",
Description = "Authorization token.")]
public string TokenValue { get; set; } = "";
@ -18,12 +15,9 @@ namespace DiscordChatExporter.Cli.Commands
Description = "Whether this authorization token belongs to a bot.")]
public bool IsBotToken { get; set; }
protected AuthToken Token => new AuthToken(IsBotToken ? AuthTokenType.Bot : AuthTokenType.User, TokenValue);
protected AuthToken GetAuthToken() => new AuthToken(IsBotToken ? AuthTokenType.Bot : AuthTokenType.User, TokenValue);
protected TokenCommandBase(DataService dataService)
{
DataService = dataService;
}
protected DiscordClient GetDiscordClient() => new DiscordClient(GetAuthToken());
public abstract ValueTask ExecuteAsync(IConsole console);
}

7
DiscordChatExporter.Cli/Commands/ExportChannelCommand.cs
View file

@ -1,7 +1,7 @@
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Cli.Commands.Base;
namespace DiscordChatExporter.Cli.Commands
{
@ -11,11 +11,6 @@ namespace DiscordChatExporter.Cli.Commands
[CommandOption("channel", 'c', IsRequired = true, Description = "Channel ID.")]
public string ChannelId { get; set; } = "";
public ExportChannelCommand(SettingsService settingsService, DataService dataService, ExportService exportService)
: base(settingsService, dataService, exportService)
{
}
public override async ValueTask ExecuteAsync(IConsole console) => await ExportAsync(console, ChannelId);
}
}

9
DiscordChatExporter.Cli/Commands/ExportDirectMessagesCommand.cs
View file

@ -2,21 +2,16 @@
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Cli.Commands.Base;
namespace DiscordChatExporter.Cli.Commands
{
[Command("exportdm", Description = "Export all direct message channels.")]
public class ExportDirectMessagesCommand : ExportMultipleCommandBase
{
public ExportDirectMessagesCommand(SettingsService settingsService, DataService dataService, ExportService exportService)
: base(settingsService, dataService, exportService)
{
}
public override async ValueTask ExecuteAsync(IConsole console)
{
var directMessageChannels = await DataService.GetDirectMessageChannelsAsync(Token);
var directMessageChannels = await GetDiscordClient().GetDirectMessageChannelsAsync();
var channels = directMessageChannels.OrderBy(c => c.Name).ToArray();
await ExportMultipleAsync(console, channels);

12
DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs
View file

@ -2,8 +2,7 @@
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Cli.Commands.Base;
namespace DiscordChatExporter.Cli.Commands
{
@ -13,17 +12,12 @@ namespace DiscordChatExporter.Cli.Commands
[CommandOption("guild", 'g', IsRequired = true, Description = "Guild ID.")]
public string GuildId { get; set; } = "";
public ExportGuildCommand(SettingsService settingsService, DataService dataService, ExportService exportService)
: base(settingsService, dataService, exportService)
{
}
public override async ValueTask ExecuteAsync(IConsole console)
{
var guildChannels = await DataService.GetGuildChannelsAsync(Token, GuildId);
var guildChannels = await GetDiscordClient().GetGuildChannelsAsync(GuildId);
var channels = guildChannels
.Where(c => c.Type.IsExportable())
.Where(c => c.IsTextChannel)
.OrderBy(c => c.Name)
.ToArray();

12
DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs
View file

@ -2,8 +2,7 @@
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Cli.Commands.Base;
namespace DiscordChatExporter.Cli.Commands
{
@ -13,17 +12,12 @@ namespace DiscordChatExporter.Cli.Commands
[CommandOption("guild", 'g', IsRequired = true, Description = "Guild ID.")]
public string GuildId { get; set; } = "";
public GetChannelsCommand(DataService dataService)
: base(dataService)
{
}
public override async ValueTask ExecuteAsync(IConsole console)
{
var guildChannels = await DataService.GetGuildChannelsAsync(Token, GuildId);
var guildChannels = await GetDiscordClient().GetGuildChannelsAsync(GuildId);
var channels = guildChannels
.Where(c => c.Type.IsExportable())
.Where(c => c.IsTextChannel)
.OrderBy(c => c.Name)
.ToArray();

9
DiscordChatExporter.Cli/Commands/GetDirectMessageChannelsCommand.cs
View file

@ -2,21 +2,16 @@
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Cli.Commands.Base;
namespace DiscordChatExporter.Cli.Commands
{
[Command("dm", Description = "Get the list of direct message channels.")]
public class GetDirectMessageChannelsCommand : TokenCommandBase
{
public GetDirectMessageChannelsCommand(DataService dataService)
: base(dataService)
{
}
public override async ValueTask ExecuteAsync(IConsole console)
{
var directMessageChannels = await DataService.GetDirectMessageChannelsAsync(Token);
var directMessageChannels = await GetDiscordClient().GetDirectMessageChannelsAsync();
var channels = directMessageChannels.OrderBy(c => c.Name).ToArray();
foreach (var channel in channels)

10
DiscordChatExporter.Cli/Commands/GetGuildsCommand.cs
View file

@ -2,21 +2,17 @@
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Cli.Commands.Base;
using DiscordChatExporter.Domain.Discord;
namespace DiscordChatExporter.Cli.Commands
{
[Command("guilds", Description = "Get the list of accessible guilds.")]
public class GetGuildsCommand : TokenCommandBase
{
public GetGuildsCommand(DataService dataService)
: base(dataService)
{
}
public override async ValueTask ExecuteAsync(IConsole console)
{
var guilds = await DataService.GetUserGuildsAsync(Token);
var guilds = await GetDiscordClient().GetUserGuildsAsync();
foreach (var guild in guilds.OrderBy(g => g.Name))
console.Output.WriteLine($"{guild.Id} | {guild.Name}");

4
DiscordChatExporter.Cli/DiscordChatExporter.Cli.csproj
View file

@ -9,13 +9,11 @@
<ItemGroup>
<PackageReference Include="CliFx" Version="1.0.0" />
<PackageReference Include="Gress" Version="1.1.1" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="3.1.1" />
<PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\DiscordChatExporter.Core.Models\DiscordChatExporter.Core.Models.csproj" />
<ProjectReference Include="..\DiscordChatExporter.Core.Services\DiscordChatExporter.Core.Services.csproj" />
<ProjectReference Include="..\DiscordChatExporter.Domain\DiscordChatExporter.Domain.csproj" />
</ItemGroup>
</Project>

36
DiscordChatExporter.Cli/Program.cs
View file

@ -1,44 +1,14 @@
using System;
using System.Threading.Tasks;
using System.Threading.Tasks;
using CliFx;
using DiscordChatExporter.Cli.Commands;
using DiscordChatExporter.Core.Services;
using Microsoft.Extensions.DependencyInjection;
namespace DiscordChatExporter.Cli
{
public static class Program
{
private static IServiceProvider ConfigureServices()
{
var services = new ServiceCollection();
// Register services
services.AddSingleton<DataService>();
services.AddSingleton<ExportService>();
services.AddSingleton<SettingsService>();
// Register commands
services.AddTransient<ExportChannelCommand>();
services.AddTransient<ExportDirectMessagesCommand>();
services.AddTransient<ExportGuildCommand>();
services.AddTransient<GetChannelsCommand>();
services.AddTransient<GetDirectMessageChannelsCommand>();
services.AddTransient<GetGuildsCommand>();
services.AddTransient<GuideCommand>();
return services.BuildServiceProvider();
}
public static async Task<int> Main(string[] args)
{
var serviceProvider = ConfigureServices();
return await new CliApplicationBuilder()
public static async Task<int> Main(string[] args) =>
await new CliApplicationBuilder()
.AddCommandsFromThisAssembly()
.UseTypeActivator(serviceProvider.GetService)
.Build()
.RunAsync(args);
}
}
}

19
DiscordChatExporter.Core.Markdown/Ast/FormattedNode.cs
View file

@ -1,19 +0,0 @@
using System.Collections.Generic;
namespace DiscordChatExporter.Core.Markdown.Ast
{
public class FormattedNode : Node
{
public TextFormatting Formatting { get; }
public IReadOnlyList<Node> Children { get; }
public FormattedNode(TextFormatting formatting, IReadOnlyList<Node> children)
{
Formatting = formatting;
Children = children;
}
public override string ToString() => $"<{Formatting}> ({Children.Count} direct children)";
}
}

10
DiscordChatExporter.Core.Markdown/Ast/MentionType.cs
View file

@ -1,10 +0,0 @@
namespace DiscordChatExporter.Core.Markdown.Ast
{
public enum MentionType
{
Meta,
User,
Channel,
Role
}
}

6
DiscordChatExporter.Core.Markdown/Ast/Node.cs
View file

@ -1,6 +0,0 @@
namespace DiscordChatExporter.Core.Markdown.Ast
{
public abstract class Node
{
}
}

12
DiscordChatExporter.Core.Markdown/Ast/TextFormatting.cs
View file

@ -1,12 +0,0 @@
namespace DiscordChatExporter.Core.Markdown.Ast
{
public enum TextFormatting
{
Bold,
Italic,
Underline,
Strikethrough,
Spoiler,
Quote
}
}

8
DiscordChatExporter.Core.Markdown/DiscordChatExporter.Core.Markdown.csproj
View file

@ -1,8 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<Import Project="../DiscordChatExporter.props" />
<ItemGroup>
<PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" />
</ItemGroup>
</Project>

7
DiscordChatExporter.Core.Markdown/Internal/IMatcher.cs
View file

@ -1,7 +0,0 @@
namespace DiscordChatExporter.Core.Markdown.Internal
{
internal interface IMatcher<T>
{
ParsedMatch<T>? Match(StringPart stringPart);
}
}

15
DiscordChatExporter.Core.Models/AuthToken.cs
View file

@ -1,15 +0,0 @@
namespace DiscordChatExporter.Core.Models
{
public class AuthToken
{
public AuthTokenType Type { get; }
public string Value { get; }
public AuthToken(AuthTokenType type, string value)
{
Type = type;
Value = value;
}
}
}

8
DiscordChatExporter.Core.Models/AuthTokenType.cs
View file

@ -1,8 +0,0 @@
namespace DiscordChatExporter.Core.Models
{
public enum AuthTokenType
{
User,
Bot
}
}

37
DiscordChatExporter.Core.Models/Channel.cs
View file

@ -1,37 +0,0 @@
namespace DiscordChatExporter.Core.Models
{
// https://discordapp.com/developers/docs/resources/channel#channel-object
public partial class Channel : IHasId
{
public string Id { get; }
public string? ParentId { get; }
public string GuildId { get; }
public string Name { get; }
public string? Topic { get; }
public ChannelType Type { get; }
public Channel(string id, string? parentId, string guildId, string name, string? topic, ChannelType type)
{
Id = id;
ParentId = parentId;
GuildId = guildId;
Name = name;
Topic = topic;
Type = type;
}
public override string ToString() => Name;
}
public partial class Channel
{
public static Channel CreateDeletedChannel(string id) =>
new Channel(id, null, "unknown-guild", "deleted-channel", null, ChannelType.GuildTextChat);
}
}

16
DiscordChatExporter.Core.Models/ChannelType.cs
View file

@ -1,16 +0,0 @@
namespace DiscordChatExporter.Core.Models
{
// https://discordapp.com/developers/docs/resources/channel#channel-object-channel-types
// Order of enum fields needs to match the order in the docs.
public enum ChannelType
{
GuildTextChat,
DirectTextChat,
GuildVoiceChat,
DirectGroupTextChat,
GuildCategory,
GuildNews,
GuildStore
}
}

8
DiscordChatExporter.Core.Models/DiscordChatExporter.Core.Models.csproj
View file

@ -1,8 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<Import Project="../DiscordChatExporter.props" />
<ItemGroup>
<PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" />
</ItemGroup>
</Project>

12
DiscordChatExporter.Core.Models/Exceptions/DomainException.cs
View file

@ -1,12 +0,0 @@
using System;
namespace DiscordChatExporter.Core.Models.Exceptions
{
public class DomainException : Exception
{
public DomainException(string message)
: base(message)
{
}
}
}

11
DiscordChatExporter.Core.Models/ExportFormat.cs
View file

@ -1,11 +0,0 @@
namespace DiscordChatExporter.Core.Models
{
public enum ExportFormat
{
PlainText,
HtmlDark,
HtmlLight,
Csv,
Json
}
}

36
DiscordChatExporter.Core.Models/Extensions.cs
View file

@ -1,36 +0,0 @@
using System;
namespace DiscordChatExporter.Core.Models
{
public static class Extensions
{
public static bool IsExportable(this ChannelType channelType) =>
channelType == ChannelType.GuildTextChat ||
channelType == ChannelType.DirectTextChat ||
channelType == ChannelType.DirectGroupTextChat ||
channelType == ChannelType.GuildNews ||
channelType == ChannelType.GuildStore;
public static string GetFileExtension(this ExportFormat format) =>
format switch
{
ExportFormat.PlainText => "txt",
ExportFormat.HtmlDark => "html",
ExportFormat.HtmlLight => "html",
ExportFormat.Csv => "csv",
ExportFormat.Json => "json",
_ => throw new ArgumentOutOfRangeException(nameof(format))
};
public static string GetDisplayName(this ExportFormat format) =>
format switch
{
ExportFormat.PlainText => "TXT",
ExportFormat.HtmlDark => "HTML (Dark)",
ExportFormat.HtmlLight => "HTML (Light)",
ExportFormat.Csv => "CSV",
ExportFormat.Json => "JSON",
_ => throw new ArgumentOutOfRangeException(nameof(format))
};
}
}

23
DiscordChatExporter.Core.Models/MessageGroup.cs
View file

@ -1,23 +0,0 @@
using System;
using System.Collections.Generic;
namespace DiscordChatExporter.Core.Models
{
// Used for grouping contiguous messages in HTML export
public class MessageGroup
{
public User Author { get; }
public DateTimeOffset Timestamp { get; }
public IReadOnlyList<Message> Messages { get; }
public MessageGroup(User author, DateTimeOffset timestamp, IReadOnlyList<Message> messages)
{
Author = author;
Timestamp = timestamp;
Messages = messages;
}
}
}

16
DiscordChatExporter.Core.Models/MessageType.cs
View file

@ -1,16 +0,0 @@
namespace DiscordChatExporter.Core.Models
{
// https://discordapp.com/developers/docs/resources/channel#message-object-message-types
public enum MessageType
{
Default,
RecipientAdd,
RecipientRemove,
Call,
ChannelNameChange,
ChannelIconChange,
ChannelPinnedMessage,
GuildMemberJoin
}
}

21
DiscordChatExporter.Core.Rendering/DiscordChatExporter.Core.Rendering.csproj
View file

@ -1,21 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<Import Project="../DiscordChatExporter.props" />
<ItemGroup>
<EmbeddedResource Include="Resources\HtmlCore.css" />
<EmbeddedResource Include="Resources\HtmlDark.css" />
<EmbeddedResource Include="Resources\HtmlLight.css" />
<EmbeddedResource Include="Resources\HtmlLayoutTemplate.html" />
<EmbeddedResource Include="Resources\HtmlMessageGroupTemplate.html" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Scriban" Version="2.1.1" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\DiscordChatExporter.Core.Markdown\DiscordChatExporter.Core.Markdown.csproj" />
<ProjectReference Include="..\DiscordChatExporter.Core.Models\DiscordChatExporter.Core.Models.csproj" />
</ItemGroup>
</Project>

34
DiscordChatExporter.Core.Rendering/Formatters/CsvMessageWriter.cs
View file

@ -1,34 +0,0 @@
using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering.Logic;
namespace DiscordChatExporter.Core.Rendering.Formatters
{
public class CsvMessageWriter : MessageWriterBase
{
private readonly TextWriter _writer;
public CsvMessageWriter(Stream stream, RenderContext context)
: base(stream, context)
{
_writer = new StreamWriter(stream);
}
public override async Task WritePreambleAsync()
{
await _writer.WriteLineAsync(CsvRenderingLogic.FormatHeader(Context));
}
public override async Task WriteMessageAsync(Message message)
{
await _writer.WriteLineAsync(CsvRenderingLogic.FormatMessage(Context, message));
}
public override async ValueTask DisposeAsync()
{
await _writer.DisposeAsync();
await base.DisposeAsync();
}
}
}

45
DiscordChatExporter.Core.Rendering/Formatters/PlainTextMessageWriter.cs
View file

@ -1,45 +0,0 @@
using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering.Logic;
namespace DiscordChatExporter.Core.Rendering.Formatters
{
public class PlainTextMessageWriter : MessageWriterBase
{
private readonly TextWriter _writer;
private long _messageCount;
public PlainTextMessageWriter(Stream stream, RenderContext context)
: base(stream, context)
{
_writer = new StreamWriter(stream);
}
public override async Task WritePreambleAsync()
{
await _writer.WriteLineAsync(PlainTextRenderingLogic.FormatPreamble(Context));
}
public override async Task WriteMessageAsync(Message message)
{
await _writer.WriteLineAsync(PlainTextRenderingLogic.FormatMessage(Context, message));
await _writer.WriteLineAsync();
_messageCount++;
}
public override async Task WritePostambleAsync()
{
await _writer.WriteLineAsync();
await _writer.WriteLineAsync(PlainTextRenderingLogic.FormatPostamble(_messageCount));
}
public override async ValueTask DisposeAsync()
{
await _writer.DisposeAsync();
await base.DisposeAsync();
}
}
}

39
DiscordChatExporter.Core.Rendering/Logic/CsvRenderingLogic.cs
View file

@ -1,39 +0,0 @@
using System.Linq;
using System.Text;
using DiscordChatExporter.Core.Models;
using Tyrrrz.Extensions;
using static DiscordChatExporter.Core.Rendering.Logic.SharedRenderingLogic;
namespace DiscordChatExporter.Core.Rendering.Logic
{
public static class CsvRenderingLogic
{
// Header is always the same
public static string FormatHeader(RenderContext context) => "AuthorID,Author,Date,Content,Attachments,Reactions";
private static string EncodeValue(string value)
{
value = value.Replace("\"", "\"\"");
return $"\"{value}\"";
}
public static string FormatMarkdown(RenderContext context, string markdown) =>
PlainTextRenderingLogic.FormatMarkdown(context, markdown);
public static string FormatMessage(RenderContext context, Message message)
{
var buffer = new StringBuilder();
buffer
.Append(EncodeValue(message.Author.Id)).Append(',')
.Append(EncodeValue(message.Author.FullName)).Append(',')
.Append(EncodeValue(FormatDate(message.Timestamp, context.DateFormat))).Append(',')
.Append(EncodeValue(FormatMarkdown(context, message.Content ?? ""))).Append(',')
.Append(EncodeValue(message.Attachments.Select(a => a.Url).JoinToString(","))).Append(',')
.Append(EncodeValue(message.Reactions.Select(r => $"{r.Emoji.Name} ({r.Count})").JoinToString(",")));
return buffer.ToString();
}
}
}

172
DiscordChatExporter.Core.Rendering/Logic/HtmlRenderingLogic.cs
View file

@ -1,172 +0,0 @@
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using DiscordChatExporter.Core.Markdown;
using DiscordChatExporter.Core.Markdown.Ast;
using DiscordChatExporter.Core.Models;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Rendering.Logic
{
internal static class HtmlRenderingLogic
{
public static bool CanBeGrouped(Message message1, Message message2)
{
if (!string.Equals(message1.Author.Id, message2.Author.Id, StringComparison.Ordinal))
return false;
// Bots can post message under different usernames, so need to check this too
if (!string.Equals(message1.Author.FullName, message2.Author.FullName, StringComparison.Ordinal))
return false;
if ((message2.Timestamp - message1.Timestamp).Duration().TotalMinutes > 7)
return false;
return true;
}
private static string HtmlEncode(string s) => WebUtility.HtmlEncode(s);
private static string FormatMarkdownNode(RenderContext context, Node node, bool isJumbo)
{
// Text node
if (node is TextNode textNode)
{
// Return HTML-encoded text
return HtmlEncode(textNode.Text);
}
// Formatted node
if (node is FormattedNode formattedNode)
{
// Recursively get inner html
var innerHtml = FormatMarkdownNodes(context, formattedNode.Children, false);
// Bold
if (formattedNode.Formatting == TextFormatting.Bold)
return $"<strong>{innerHtml}</strong>";
// Italic
if (formattedNode.Formatting == TextFormatting.Italic)
return $"<em>{innerHtml}</em>";
// Underline
if (formattedNode.Formatting == TextFormatting.Underline)
return $"<u>{innerHtml}</u>";
// Strikethrough
if (formattedNode.Formatting == TextFormatting.Strikethrough)
return $"<s>{innerHtml}</s>";
// Spoiler
if (formattedNode.Formatting == TextFormatting.Spoiler)
return $"<span class=\"spoiler spoiler--hidden\" onclick=\"showSpoiler(event, this)\"><span class=\"spoiler-text\">{innerHtml}</span></span>";
// Quote
if (formattedNode.Formatting == TextFormatting.Quote)
return $"<div class=\"quote\">{innerHtml}</div>";
}
// Inline code block node
if (node is InlineCodeBlockNode inlineCodeBlockNode)
{
return $"<span class=\"pre pre--inline\">{HtmlEncode(inlineCodeBlockNode.Code)}</span>";
}
// Multi-line code block node
if (node is MultiLineCodeBlockNode multilineCodeBlockNode)
{
// Set CSS class for syntax highlighting
var highlightCssClass = !string.IsNullOrWhiteSpace(multilineCodeBlockNode.Language)
? $"language-{multilineCodeBlockNode.Language}"
: "nohighlight";
return $"<div class=\"pre pre--multiline {highlightCssClass}\">{HtmlEncode(multilineCodeBlockNode.Code)}</div>";
}
// Mention node
if (node is MentionNode mentionNode)
{
// Meta mention node
if (mentionNode.Type == MentionType.Meta)
{
return $"<span class=\"mention\">@{HtmlEncode(mentionNode.Id)}</span>";
}
// User mention node
if (mentionNode.Type == MentionType.User)
{
var user = context.MentionableUsers.FirstOrDefault(u => u.Id == mentionNode.Id) ??
User.CreateUnknownUser(mentionNode.Id);
var nick = Guild.GetUserNick(context.Guild, user);
return $"<span class=\"mention\" title=\"{HtmlEncode(user.FullName)}\">@{HtmlEncode(nick)}</span>";
}
// Channel mention node
if (mentionNode.Type == MentionType.Channel)
{
var channel = context.MentionableChannels.FirstOrDefault(c => c.Id == mentionNode.Id) ??
Channel.CreateDeletedChannel(mentionNode.Id);
return $"<span class=\"mention\">#{HtmlEncode(channel.Name)}</span>";
}
// Role mention node
if (mentionNode.Type == MentionType.Role)
{
var role = context.MentionableRoles.FirstOrDefault(r => r.Id == mentionNode.Id) ??
Role.CreateDeletedRole(mentionNode.Id);
var style = "";
if (role.Color != Color.Black)
style = $"style=\"color: {role.ColorAsHex}; background-color: rgba({role.ColorAsRgb}, 0.1); font-weight: 400;\"";
return $"<span class=\"mention\" {style}>@{HtmlEncode(role.Name)}</span>";
}
}
// Emoji node
if (node is EmojiNode emojiNode)
{
// Get emoji image URL
var emojiImageUrl = Emoji.GetImageUrl(emojiNode.Id, emojiNode.Name, emojiNode.IsAnimated);
// Make emoji large if it's jumbo
var jumboableCssClass = isJumbo ? "emoji--large" : null;
return
$"<img class=\"emoji {jumboableCssClass}\" alt=\"{emojiNode.Name}\" title=\"{emojiNode.Name}\" src=\"{emojiImageUrl}\" />";
}
// Link node
if (node is LinkNode linkNode)
{
// Extract message ID if the link points to a Discord message
var linkedMessageId = Regex.Match(linkNode.Url, "^https?://discordapp.com/channels/.*?/(\\d+)/?$").Groups[1].Value;
return string.IsNullOrWhiteSpace(linkedMessageId)
? $"<a href=\"{Uri.EscapeUriString(linkNode.Url)}\">{HtmlEncode(linkNode.Title)}</a>"
: $"<a href=\"{Uri.EscapeUriString(linkNode.Url)}\" onclick=\"scrollToMessage(event, '{linkedMessageId}')\">{HtmlEncode(linkNode.Title)}</a>";
}
// Throw on unexpected nodes
throw new InvalidOperationException($"Unexpected node [{node.GetType()}].");
}
private static string FormatMarkdownNodes(RenderContext context, IReadOnlyList<Node> nodes, bool isTopLevel)
{
// Emojis are jumbo if all top-level nodes are emoji nodes or whitespace text nodes
var isJumbo = isTopLevel && nodes.All(n => n is EmojiNode || n is TextNode textNode && string.IsNullOrWhiteSpace(textNode.Text));
return nodes.Select(n => FormatMarkdownNode(context, n, isJumbo)).JoinToString("");
}
public static string FormatMarkdown(RenderContext context, string markdown) =>
FormatMarkdownNodes(context, MarkdownParser.Parse(markdown), true);
}
}

246
DiscordChatExporter.Core.Rendering/Logic/PlainTextRenderingLogic.cs
View file

@ -1,246 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using DiscordChatExporter.Core.Markdown;
using DiscordChatExporter.Core.Markdown.Ast;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering.Internal;
using Tyrrrz.Extensions;
using static DiscordChatExporter.Core.Rendering.Logic.SharedRenderingLogic;
namespace DiscordChatExporter.Core.Rendering.Logic
{
public static class PlainTextRenderingLogic
{
public static string FormatPreamble(RenderContext context)
{
var buffer = new StringBuilder();
buffer.Append('=', 62).AppendLine();
buffer.AppendLine($"Guild: {context.Guild.Name}");
buffer.AppendLine($"Channel: {context.Channel.Name}");
if (!string.IsNullOrWhiteSpace(context.Channel.Topic))
buffer.AppendLine($"Topic: {context.Channel.Topic}");
if (context.After != null)
buffer.AppendLine($"After: {FormatDate(context.After.Value, context.DateFormat)}");
if (context.Before != null)
buffer.AppendLine($"Before: {FormatDate(context.Before.Value, context.DateFormat)}");
buffer.Append('=', 62).AppendLine();
return buffer.ToString();
}
public static string FormatPostamble(long messageCount)
{
var buffer = new StringBuilder();
buffer.Append('=', 62).AppendLine();
buffer.AppendLine($"Exported {messageCount:N0} message(s)");
buffer.Append('=', 62).AppendLine();
return buffer.ToString();
}
private static string FormatMarkdownNode(RenderContext context, Node node)
{
// Text node
if (node is TextNode textNode)
{
return textNode.Text;
}
// Mention node
if (node is MentionNode mentionNode)
{
// Meta mention node
if (mentionNode.Type == MentionType.Meta)
{
return $"@{mentionNode.Id}";
}
// User mention node
if (mentionNode.Type == MentionType.User)
{
var user = context.MentionableUsers.FirstOrDefault(u => u.Id == mentionNode.Id) ??
User.CreateUnknownUser(mentionNode.Id);
return $"@{user.Name}";
}
// Channel mention node
if (mentionNode.Type == MentionType.Channel)
{
var channel = context.MentionableChannels.FirstOrDefault(c => c.Id == mentionNode.Id) ??
Channel.CreateDeletedChannel(mentionNode.Id);
return $"#{channel.Name}";
}
// Role mention node
if (mentionNode.Type == MentionType.Role)
{
var role = context.MentionableRoles.FirstOrDefault(r => r.Id == mentionNode.Id) ??
Role.CreateDeletedRole(mentionNode.Id);
return $"@{role.Name}";
}
}
// Emoji node
if (node is EmojiNode emojiNode)
{
return emojiNode.IsCustomEmoji ? $":{emojiNode.Name}:" : emojiNode.Name;
}
// Throw on unexpected nodes
throw new InvalidOperationException($"Unexpected node [{node.GetType()}].");
}
public static string FormatMarkdown(RenderContext context, string markdown) =>
MarkdownParser.ParseMinimal(markdown).Select(n => FormatMarkdownNode(context, n)).JoinToString("");
public static string FormatMessageHeader(RenderContext context, Message message)
{
var buffer = new StringBuilder();
// Timestamp & author
buffer
.Append($"[{FormatDate(message.Timestamp, context.DateFormat)}]")
.Append(' ')
.Append($"{message.Author.FullName}");
// Whether the message is pinned
if (message.IsPinned)
{
buffer.Append(' ').Append("(pinned)");
}
return buffer.ToString();
}
public static string FormatMessageContent(RenderContext context, Message message)
{
if (string.IsNullOrWhiteSpace(message.Content))
return "";
return FormatMarkdown(context, message.Content);
}
public static string FormatAttachments(IReadOnlyList<Attachment> attachments)
{
if (!attachments.Any())
return "";
var buffer = new StringBuilder();
buffer
.AppendLine("{Attachments}")
.AppendJoin(Environment.NewLine, attachments.Select(a => a.Url))
.AppendLine();
return buffer.ToString();
}
public static string FormatEmbeds(RenderContext context, IReadOnlyList<Embed> embeds)
{
if (!embeds.Any())
return "";
var buffer = new StringBuilder();
foreach (var embed in embeds)
{
buffer.AppendLine("{Embed}");
// Author name
if (!string.IsNullOrWhiteSpace(embed.Author?.Name))
buffer.AppendLine(embed.Author.Name);
// URL
if (!string.IsNullOrWhiteSpace(embed.Url))
buffer.AppendLine(embed.Url);
// Title
if (!string.IsNullOrWhiteSpace(embed.Title))
buffer.AppendLine(FormatMarkdown(context, embed.Title));
// Description
if (!string.IsNullOrWhiteSpace(embed.Description))
buffer.AppendLine(FormatMarkdown(context, embed.Description));
// Fields
foreach (var field in embed.Fields)
{
// Name
if (!string.IsNullOrWhiteSpace(field.Name))
buffer.AppendLine(field.Name);
// Value
if (!string.IsNullOrWhiteSpace(field.Value))
buffer.AppendLine(field.Value);
}
// Thumbnail URL
if (!string.IsNullOrWhiteSpace(embed.Thumbnail?.Url))
buffer.AppendLine(embed.Thumbnail?.Url);
// Image URL
if (!string.IsNullOrWhiteSpace(embed.Image?.Url))
buffer.AppendLine(embed.Image?.Url);
// Footer text
if (!string.IsNullOrWhiteSpace(embed.Footer?.Text))
buffer.AppendLine(embed.Footer?.Text);
buffer.AppendLine();
}
return buffer.ToString();
}
public static string FormatReactions(IReadOnlyList<Reaction> reactions)
{
if (!reactions.Any())
return "";
var buffer = new StringBuilder();
buffer.AppendLine("{Reactions}");
foreach (var reaction in reactions)
{
buffer.Append(reaction.Emoji.Name);
if (reaction.Count > 1)
buffer.Append($" ({reaction.Count})");
buffer.Append(" ");
}
buffer.AppendLine();
return buffer.ToString();
}
public static string FormatMessage(RenderContext context, Message message)
{
var buffer = new StringBuilder();
buffer
.AppendLine(FormatMessageHeader(context, message))
.AppendLineIfNotEmpty(FormatMessageContent(context, message))
.AppendLine()
.AppendLineIfNotEmpty(FormatAttachments(message.Attachments))
.AppendLineIfNotEmpty(FormatEmbeds(context, message.Embeds))
.AppendLineIfNotEmpty(FormatReactions(message.Reactions));
return buffer.Trim().ToString();
}
}
}

11
DiscordChatExporter.Core.Rendering/Logic/SharedRenderingLogic.cs
View file

@ -1,11 +0,0 @@
using System;
using System.Globalization;
namespace DiscordChatExporter.Core.Rendering.Logic
{
public static class SharedRenderingLogic
{
public static string FormatDate(DateTimeOffset date, string dateFormat) =>
date.ToLocalTime().ToString(dateFormat, CultureInfo.InvariantCulture);
}
}

15
DiscordChatExporter.Core.Services/DiscordChatExporter.Core.Services.csproj
View file

@ -1,15 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<Import Project="../DiscordChatExporter.props" />
<ItemGroup>
<PackageReference Include="Polly" Version="7.2.0" />
<PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" />
<PackageReference Include="Tyrrrz.Settings" Version="1.3.4" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\DiscordChatExporter.Core.Models\DiscordChatExporter.Core.Models.csproj" />
<ProjectReference Include="..\DiscordChatExporter.Core.Rendering\DiscordChatExporter.Core.Rendering.csproj" />
</ItemGroup>
</Project>

19
DiscordChatExporter.Core.Services/Exceptions/HttpErrorStatusCodeException.cs
View file

@ -1,19 +0,0 @@
using System;
using System.Net;
namespace DiscordChatExporter.Core.Services.Exceptions
{
public class HttpErrorStatusCodeException : Exception
{
public HttpStatusCode StatusCode { get; }
public string ReasonPhrase { get; }
public HttpErrorStatusCodeException(HttpStatusCode statusCode, string reasonPhrase)
: base($"Error HTTP status code: {statusCode} - {reasonPhrase}")
{
StatusCode = statusCode;
ReasonPhrase = reasonPhrase;
}
}
}

96
DiscordChatExporter.Core.Services/ExportService.cs
View file

@ -1,96 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Models.Exceptions;
using DiscordChatExporter.Core.Rendering;
using DiscordChatExporter.Core.Services.Logic;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Services
{
public partial class ExportService
{
private readonly SettingsService _settingsService;
private readonly DataService _dataService;
public ExportService(SettingsService settingsService, DataService dataService)
{
_settingsService = settingsService;
_dataService = dataService;
}
public async Task ExportChatLogAsync(AuthToken token, Guild guild, Channel channel,
string outputPath, ExportFormat format, int? partitionLimit,
DateTimeOffset? after = null, DateTimeOffset? before = null, IProgress<double>? progress = null)
{
// Get base file path from output path
var baseFilePath = GetFilePathFromOutputPath(outputPath, format, guild, channel, after, before);
// Create options
var options = new RenderOptions(baseFilePath, format, partitionLimit);
// Create context
var mentionableUsers = new HashSet<User>(IdBasedEqualityComparer.Instance);
var mentionableChannels = await _dataService.GetGuildChannelsAsync(token, guild.Id);
var mentionableRoles = guild.Roles;
var context = new RenderContext
(
guild, channel, after, before, _settingsService.DateFormat,
mentionableUsers, mentionableChannels, mentionableRoles
);
// Create renderer
await using var renderer = new MessageRenderer(options, context);
// Render messages
var renderedAnything = false;
await foreach (var message in _dataService.GetMessagesAsync(token, channel.Id, after, before, progress))
{
// Add encountered users to the list of mentionable users
var encounteredUsers = new List<User>();
encounteredUsers.Add(message.Author);
encounteredUsers.AddRange(message.MentionedUsers);
mentionableUsers.AddRange(encounteredUsers);
foreach (User u in encounteredUsers)
{
if(!guild.Members.ContainsKey(u.Id))
{
var member = await _dataService.GetGuildMemberAsync(token, guild.Id, u.Id);
guild.Members[u.Id] = member;
}
}
// Render message
await renderer.RenderMessageAsync(message);
renderedAnything = true;
}
// Throw if no messages were rendered
if (!renderedAnything)
throw new DomainException($"Channel [{channel.Name}] contains no messages for specified period");
}
}
public partial class ExportService
{
private static string GetFilePathFromOutputPath(string outputPath, ExportFormat format, Guild guild, Channel channel,
DateTimeOffset? after, DateTimeOffset? before)
{
// Output is a directory
if (Directory.Exists(outputPath) || string.IsNullOrWhiteSpace(Path.GetExtension(outputPath)))
{
var fileName = ExportLogic.GetDefaultExportFileName(format, guild, channel, after, before);
return Path.Combine(outputPath, fileName);
}
// Output is a file
return outputPath;
}
}
}

9
DiscordChatExporter.Core.Services/Internal/Extensions/ColorExtensions.cs
View file

@ -1,9 +0,0 @@
using System.Drawing;
namespace DiscordChatExporter.Core.Services.Internal.Extensions
{
internal static class ColorExtensions
{
public static Color ResetAlpha(this Color color) => Color.FromArgb(1, color);
}
}

9
DiscordChatExporter.Core.Services/Internal/Extensions/GenericExtensions.cs
View file

@ -1,9 +0,0 @@
using System;
namespace DiscordChatExporter.Core.Services.Internal.Extensions
{
internal static class GenericExtensions
{
public static TOut Pipe<TIn, TOut>(this TIn input, Func<TIn, TOut> transform) => transform(input);
}
}

13
DiscordChatExporter.Core.Services/Internal/Json.cs
View file

@ -1,13 +0,0 @@
using System.Text.Json;
namespace DiscordChatExporter.Core.Services.Internal
{
internal static class Json
{
public static JsonElement Parse(string json)
{
using var document = JsonDocument.Parse(json);
return document.RootElement.Clone();
}
}
}

53
DiscordChatExporter.Core.Services/Logic/ExportLogic.cs
View file

@ -1,53 +0,0 @@
using System;
using System.IO;
using System.Text;
using DiscordChatExporter.Core.Models;
namespace DiscordChatExporter.Core.Services.Logic
{
public static class ExportLogic
{
public static string GetDefaultExportFileName(ExportFormat format,
Guild guild, Channel channel,
DateTimeOffset? after = null, DateTimeOffset? before = null)
{
var buffer = new StringBuilder();
// Append guild and channel names
buffer.Append($"{guild.Name} - {channel.Name} [{channel.Id}]");
// Append date range
if (after != null || before != null)
{
buffer.Append(" (");
// Both 'after' and 'before' are set
if (after != null && before != null)
{
buffer.Append($"{after:yyyy-MM-dd} to {before:yyyy-MM-dd}");
}
// Only 'after' is set
else if (after != null)
{
buffer.Append($"after {after:yyyy-MM-dd}");
}
// Only 'before' is set
else
{
buffer.Append($"before {before:yyyy-MM-dd}");
}
buffer.Append(")");
}
// Append extension
buffer.Append($".{format.GetFileExtension()}");
// Replace invalid chars
foreach (var invalidChar in Path.GetInvalidFileNameChars())
buffer.Replace(invalidChar, '_');
return buffer.ToString();
}
}
}

4
DiscordChatExporter.Core.Services/Extensions.cs → DiscordChatExporter.Domain/Discord/AccessibilityExtensions.cs
View file

@ -2,9 +2,9 @@
using System.Runtime.CompilerServices;
using System.Threading.Tasks;
namespace DiscordChatExporter.Core.Services
namespace DiscordChatExporter.Domain.Discord
{
public static class Extensions
public static class AccessibilityExtensions
{
private static async ValueTask<IReadOnlyList<T>> AggregateAsync<T>(this IAsyncEnumerable<T> asyncEnumerable)
{

29
DiscordChatExporter.Domain/Discord/AuthToken.cs Normal file
View file

@ -0,0 +1,29 @@
using System.Net.Http.Headers;
namespace DiscordChatExporter.Domain.Discord
{
public enum AuthTokenType
{
User,
Bot
}
public class AuthToken
{
public AuthTokenType Type { get; }
public string Value { get; }
public AuthToken(AuthTokenType type, string value)
{
Type = type;
Value = value;
}
public AuthenticationHeaderValue GetAuthenticationHeader() => Type == AuthTokenType.User
? new AuthenticationHeaderValue(Value)
: new AuthenticationHeaderValue("Bot", Value);
public override string ToString() => Value;
}
}

22
DiscordChatExporter.Core.Services/DataService.Parsers.cs → DiscordChatExporter.Domain/Discord/DiscordClient.Parsing.cs
View file

@ -2,13 +2,13 @@
using System.Drawing;
using System.Linq;
using System.Text.Json;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Services.Internal.Extensions;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Internal;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Services
namespace DiscordChatExporter.Domain.Discord
{
public partial class DataService
public partial class DiscordClient
{
private string ParseId(JsonElement json) =>
json.GetProperty("id").GetString();
@ -42,7 +42,7 @@ namespace DiscordChatExporter.Core.Services
var roles = json.GetPropertyOrNull("roles")?.EnumerateArray().Select(ParseRole).ToArray() ??
Array.Empty<Role>();
return new Guild(id, name, roles, iconHash);
return new Guild(id, name, iconHash, roles);
}
private Channel ParseChannel(JsonElement json)
@ -60,14 +60,14 @@ namespace DiscordChatExporter.Core.Services
json.GetPropertyOrNull("recipients")?.EnumerateArray().Select(ParseUser).Select(u => u.Name).JoinToString(", ") ??
id;
return new Channel(id, parentId, guildId, name, topic, type);
return new Channel(id, guildId, parentId, type, name, topic);
}
private Role ParseRole(JsonElement json)
{
var id = ParseId(json);
var name = json.GetProperty("name").GetString();
var color = json.GetProperty("color").GetInt32().Pipe(Color.FromArgb);
var color = json.GetPropertyOrNull("color")?.GetInt32().Pipe(Color.FromArgb).ResetAlpha().NullIf(c => c.ToRgb() <= 0);
var position = json.GetProperty("position").GetInt32();
return new Role(id, name, color, position);
@ -82,7 +82,7 @@ namespace DiscordChatExporter.Core.Services
var fileName = json.GetProperty("filename").GetString();
var fileSize = json.GetProperty("size").GetInt64().Pipe(FileSize.FromBytes);
return new Attachment(id, width, height, url, fileName, fileSize);
return new Attachment(id, url, fileName, width, height, fileSize);
}
private EmbedAuthor ParseEmbedAuthor(JsonElement json)
@ -153,7 +153,7 @@ namespace DiscordChatExporter.Core.Services
var count = json.GetProperty("count").GetInt32();
var emoji = json.GetProperty("emoji").Pipe(ParseEmoji);
return new Reaction(count, emoji);
return new Reaction(emoji, count);
}
private Message ParseMessage(JsonElement json)
@ -183,10 +183,10 @@ namespace DiscordChatExporter.Core.Services
Array.Empty<Attachment>();
var embeds = json.GetPropertyOrNull("embeds")?.EnumerateArray().Select(ParseEmbed).ToArray() ??
Array.Empty<Embed>();
Array.Empty<Embed>();
var reactions = json.GetPropertyOrNull("reactions")?.EnumerateArray().Select(ParseReaction).ToArray() ??
Array.Empty<Reaction>();
Array.Empty<Reaction>();
var mentionedUsers = json.GetPropertyOrNull("mentions")?.EnumerateArray().Select(ParseUser).ToArray() ??
Array.Empty<User>();

122
DiscordChatExporter.Core.Services/DataService.cs → DiscordChatExporter.Domain/Discord/DiscordClient.cs
View file

@ -3,29 +3,29 @@ using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text.Json;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Services.Exceptions;
using DiscordChatExporter.Core.Services.Internal;
using DiscordChatExporter.Core.Services.Internal.Extensions;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exceptions;
using DiscordChatExporter.Domain.Internal;
using Polly;
namespace DiscordChatExporter.Core.Services
namespace DiscordChatExporter.Domain.Discord
{
public partial class DataService : IDisposable
public partial class DiscordClient
{
private readonly HttpClient _httpClient = new HttpClient();
private readonly IAsyncPolicy<HttpResponseMessage> _httpPolicy;
private readonly AuthToken _token;
private readonly HttpClient _httpClient;
private readonly IAsyncPolicy<HttpResponseMessage> _httpRequestPolicy;
public DataService()
public DiscordClient(AuthToken token, HttpClient httpClient)
{
_httpClient.BaseAddress = new Uri("https://discordapp.com/api/v6");
_token = token;
_httpClient = httpClient;
// Discord seems to always respond 429 on our first request with unreasonable wait time (10+ minutes).
// For that reason the policy will start respecting their retry-after header only after Nth failed response.
_httpPolicy = Policy
_httpRequestPolicy = Policy
.HandleResult<HttpResponseMessage>(m => m.StatusCode == HttpStatusCode.TooManyRequests)
.OrResult(m => m.StatusCode >= HttpStatusCode.InternalServerError)
.WaitAndRetryAsync(6,
@ -42,64 +42,70 @@ namespace DiscordChatExporter.Core.Services
(response, timespan, retryCount, context) => Task.CompletedTask);
}
private async Task<JsonElement> GetApiResponseAsync(AuthToken token, string route)
public DiscordClient(AuthToken token)
: this(token, LazyHttpClient.Value)
{
return (await GetApiResponseAsync(token, route, true))!.Value;
}
private async Task<JsonElement?> GetApiResponseAsync(AuthToken token, string route, bool errorOnFail)
private async Task<JsonElement> GetApiResponseAsync(string url)
{
using var response = await _httpPolicy.ExecuteAsync(async () =>
using var response = await _httpRequestPolicy.ExecuteAsync(async () =>
{
using var request = new HttpRequestMessage(HttpMethod.Get, route);
using var request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.Authorization = _token.GetAuthenticationHeader();
request.Headers.Authorization = token.Type == AuthTokenType.Bot
? new AuthenticationHeaderValue("Bot", token.Value)
: new AuthenticationHeaderValue(token.Value);
return await _httpClient.SendAsync(request);
return await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
});
// We throw our own exception here because default one doesn't have status code
if (!response.IsSuccessStatusCode)
{
if (errorOnFail)
throw new HttpErrorStatusCodeException(response.StatusCode, response.ReasonPhrase);
if (response.StatusCode == HttpStatusCode.Unauthorized)
throw DiscordChatExporterException.Unauthorized();
return null;
}
if ((int) response.StatusCode >= 400)
throw DiscordChatExporterException.FailedHttpRequest(response);
var jsonRaw = await response.Content.ReadAsStringAsync();
return Json.Parse(jsonRaw);
return await response.Content.ReadAsJsonAsync();
}
public async Task<Guild> GetGuildAsync(AuthToken token, string guildId)
// TODO: do we need this?
private async Task<JsonElement?> TryGetApiResponseAsync(string url)
{
try
{
return await GetApiResponseAsync(url);
}
catch (DiscordChatExporterException)
{
return null;
}
}
public async Task<Guild> GetGuildAsync(string guildId)
{
// Special case for direct messages pseudo-guild
if (guildId == Guild.DirectMessages.Id)
return Guild.DirectMessages;
var response = await GetApiResponseAsync(token, $"guilds/{guildId}");
var response = await GetApiResponseAsync($"guilds/{guildId}");
var guild = ParseGuild(response);
return guild;
}
public async Task<Member?> GetGuildMemberAsync(AuthToken token, string guildId, string userId)
public async Task<Member?> GetGuildMemberAsync(string guildId, string userId)
{
var response = await GetApiResponseAsync(token, $"guilds/{guildId}/members/{userId}", false);
var response = await TryGetApiResponseAsync($"guilds/{guildId}/members/{userId}");
return response?.Pipe(ParseMember);
}
public async Task<Channel> GetChannelAsync(AuthToken token, string channelId)
public async Task<Channel> GetChannelAsync(string channelId)
{
var response = await GetApiResponseAsync(token, $"channels/{channelId}");
var response = await GetApiResponseAsync($"channels/{channelId}");
var channel = ParseChannel(response);
return channel;
}
public async IAsyncEnumerable<Guild> GetUserGuildsAsync(AuthToken token)
public async IAsyncEnumerable<Guild> GetUserGuildsAsync()
{
var afterId = "";
@ -109,7 +115,7 @@ namespace DiscordChatExporter.Core.Services
if (!string.IsNullOrWhiteSpace(afterId))
route += $"&after={afterId}";
var response = await GetApiResponseAsync(token, route);
var response = await GetApiResponseAsync(route);
var isEmpty = true;
@ -118,7 +124,7 @@ namespace DiscordChatExporter.Core.Services
{
var guildId = ParseId(guildJson);
yield return await GetGuildAsync(token, guildId);
yield return await GetGuildAsync(guildId);
afterId = guildId;
isEmpty = false;
@ -129,42 +135,42 @@ namespace DiscordChatExporter.Core.Services
}
}
public async Task<IReadOnlyList<Channel>> GetDirectMessageChannelsAsync(AuthToken token)
public async Task<IReadOnlyList<Channel>> GetDirectMessageChannelsAsync()
{
var response = await GetApiResponseAsync(token, "users/@me/channels");
var response = await GetApiResponseAsync("users/@me/channels");
var channels = response.EnumerateArray().Select(ParseChannel).ToArray();
return channels;
}
public async Task<IReadOnlyList<Channel>> GetGuildChannelsAsync(AuthToken token, string guildId)
public async Task<IReadOnlyList<Channel>> GetGuildChannelsAsync(string guildId)
{
// Special case for direct messages pseudo-guild
if (guildId == Guild.DirectMessages.Id)
return Array.Empty<Channel>();
var response = await GetApiResponseAsync(token, $"guilds/{guildId}/channels");
var response = await GetApiResponseAsync($"guilds/{guildId}/channels");
var channels = response.EnumerateArray().Select(ParseChannel).ToArray();
return channels;
}
private async Task<Message> GetLastMessageAsync(AuthToken token, string channelId, DateTimeOffset? before = null)
private async Task<Message> GetLastMessageAsync(string channelId, DateTimeOffset? before = null)
{
var route = $"channels/{channelId}/messages?limit=1";
if (before != null)
route += $"&before={before.Value.ToSnowflake()}";
var response = await GetApiResponseAsync(token, route);
var response = await GetApiResponseAsync(route);
return response.EnumerateArray().Select(ParseMessage).FirstOrDefault();
}
public async IAsyncEnumerable<Message> GetMessagesAsync(AuthToken token, string channelId,
public async IAsyncEnumerable<Message> GetMessagesAsync(string channelId,
DateTimeOffset? after = null, DateTimeOffset? before = null, IProgress<double>? progress = null)
{
// Get the last message
var lastMessage = await GetLastMessageAsync(token, channelId, before);
var lastMessage = await GetLastMessageAsync(channelId, before);
// If the last message doesn't exist or it's outside of range - return
if (lastMessage == null || lastMessage.Timestamp < after)
@ -180,7 +186,7 @@ namespace DiscordChatExporter.Core.Services
{
// Get message batch
var route = $"channels/{channelId}/messages?limit=100&after={afterId}";
var response = await GetApiResponseAsync(token, route);
var response = await GetApiResponseAsync(route);
// Parse
var messages = response
@ -221,7 +227,23 @@ namespace DiscordChatExporter.Core.Services
yield return lastMessage;
progress?.Report(1);
}
}
public void Dispose() => _httpClient.Dispose();
public partial class DiscordClient
{
private static readonly Lazy<HttpClient> LazyHttpClient = new Lazy<HttpClient>(() =>
{
var handler = new HttpClientHandler();
if (handler.SupportsAutomaticDecompression)
handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
handler.UseCookies = false;
return new HttpClient(handler, true)
{
BaseAddress = new Uri("https://discordapp.com/api/v6")
};
});
}
}

26
DiscordChatExporter.Core.Models/Attachment.cs → DiscordChatExporter.Domain/Discord/Models/Attachment.cs
View file

@ -2,7 +2,7 @@
using System.IO;
using System.Linq;
namespace DiscordChatExporter.Core.Models
namespace DiscordChatExporter.Domain.Discord.Models
{
// https://discordapp.com/developers/docs/resources/channel#attachment-object
@ -12,30 +12,26 @@ namespace DiscordChatExporter.Core.Models
public string Url { get; }
public string FileName { get; }
public int? Width { get; }
public int? Height { get; }
public string FileName { get; }
public bool IsImage => ImageFileExtensions.Contains(Path.GetExtension(FileName), StringComparer.OrdinalIgnoreCase);
public bool IsImage { get; }
public bool IsSpoiler { get; }
public bool IsSpoiler => IsImage && FileName.StartsWith("SPOILER_", StringComparison.Ordinal);
public FileSize FileSize { get; }
public Attachment(string id, int? width, int? height, string url, string fileName, FileSize fileSize)
public Attachment(string id, string url, string fileName, int? width, int? height, FileSize fileSize)
{
Id = id;
Url = url;
FileName = fileName;
Width = width;
Height = height;
FileName = fileName;
FileSize = fileSize;
IsImage = GetIsImage(fileName);
IsSpoiler = IsImage && FileName.StartsWith("SPOILER_", StringComparison.Ordinal);
}
public override string ToString() => FileName;
@ -43,12 +39,6 @@ namespace DiscordChatExporter.Core.Models
public partial class Attachment
{
private static readonly string[] ImageFileExtensions = { ".jpg", ".jpeg", ".png", ".gif", ".bmp" };
public static bool GetIsImage(string fileName)
{
var fileExtension = Path.GetExtension(fileName);
return ImageFileExtensions.Contains(fileExtension, StringComparer.OrdinalIgnoreCase);
}
private static readonly string[] ImageFileExtensions = {".jpg", ".jpeg", ".png", ".gif", ".bmp"};
}
}

58
DiscordChatExporter.Domain/Discord/Models/Channel.cs Normal file
View file

@ -0,0 +1,58 @@
namespace DiscordChatExporter.Domain.Discord.Models
{
// https://discordapp.com/developers/docs/resources/channel#channel-object-channel-types
// Order of enum fields needs to match the order in the docs.
public enum ChannelType
{
GuildTextChat,
DirectTextChat,
GuildVoiceChat,
DirectGroupTextChat,
GuildCategory,
GuildNews,
GuildStore
}
// https://discordapp.com/developers/docs/resources/channel#channel-object
public partial class Channel : IHasId
{
public string Id { get; }
public string GuildId { get; }
public string? ParentId { get; }
public ChannelType Type { get; }
public bool IsTextChannel =>
Type == ChannelType.GuildTextChat ||
Type == ChannelType.DirectTextChat ||
Type == ChannelType.DirectGroupTextChat ||
Type == ChannelType.GuildNews ||
Type == ChannelType.GuildStore;
public string Name { get; }
public string? Topic { get; }
public Channel(string id, string guildId, string? parentId, ChannelType type, string name, string? topic)
{
Id = id;
GuildId = guildId;
ParentId = parentId;
Type = type;
Name = name;
Topic = topic;
}
public override string ToString() => Name;
}
public partial class Channel
{
public static Channel CreateDeletedChannel(string id) =>
new Channel(id, "unknown-guild", null, ChannelType.GuildTextChat, "deleted-channel", null);
}
}

15
DiscordChatExporter.Core.Models/Embed.cs → DiscordChatExporter.Domain/Discord/Models/Embed.cs
View file

@ -2,7 +2,7 @@ using System;
using System.Collections.Generic;
using System.Drawing;
namespace DiscordChatExporter.Core.Models
namespace DiscordChatExporter.Domain.Discord.Models
{
// https://discordapp.com/developers/docs/resources/channel#embed-object
@ -28,8 +28,17 @@ namespace DiscordChatExporter.Core.Models
public EmbedFooter? Footer { get; }
public Embed(string? title, string? url, DateTimeOffset? timestamp, Color? color, EmbedAuthor? author, string? description,
IReadOnlyList<EmbedField> fields, EmbedImage? thumbnail, EmbedImage? image, EmbedFooter? footer)
public Embed(
string? title,
string? url,
DateTimeOffset? timestamp,
Color? color,
EmbedAuthor? author,
string? description,
IReadOnlyList<EmbedField> fields,
EmbedImage? thumbnail,
EmbedImage? image,
EmbedFooter? footer)
{
Title = title;
Url = url;

2
DiscordChatExporter.Core.Models/EmbedAuthor.cs → DiscordChatExporter.Domain/Discord/Models/EmbedAuthor.cs
View file

@ -1,4 +1,4 @@
namespace DiscordChatExporter.Core.Models
namespace DiscordChatExporter.Domain.Discord.Models
{
// https://discordapp.com/developers/docs/resources/channel#embed-object-embed-author-structure

2
DiscordChatExporter.Core.Models/EmbedField.cs → DiscordChatExporter.Domain/Discord/Models/EmbedField.cs
View file

@ -1,4 +1,4 @@
namespace DiscordChatExporter.Core.Models
namespace DiscordChatExporter.Domain.Discord.Models
{
// https://discordapp.com/developers/docs/resources/channel#embed-object-embed-field-structure

2
DiscordChatExporter.Core.Models/EmbedFooter.cs → DiscordChatExporter.Domain/Discord/Models/EmbedFooter.cs
View file

@ -1,4 +1,4 @@
namespace DiscordChatExporter.Core.Models
namespace DiscordChatExporter.Domain.Discord.Models
{
// https://discordapp.com/developers/docs/resources/channel#embed-object-embed-footer-structure

2
DiscordChatExporter.Core.Models/EmbedImage.cs → DiscordChatExporter.Domain/Discord/Models/EmbedImage.cs
View file

@ -1,4 +1,4 @@
namespace DiscordChatExporter.Core.Models
namespace DiscordChatExporter.Domain.Discord.Models
{
// https://discordapp.com/developers/docs/resources/channel#embed-object-embed-image-structure

17
DiscordChatExporter.Core.Models/Emoji.cs → DiscordChatExporter.Domain/Discord/Models/Emoji.cs
View file

@ -3,7 +3,7 @@ using System.Linq;
using System.Text;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Models
namespace DiscordChatExporter.Domain.Discord.Models
{
// https://discordapp.com/developers/docs/resources/emoji#emoji-object
@ -25,6 +25,8 @@ namespace DiscordChatExporter.Core.Models
ImageUrl = GetImageUrl(id, name, isAnimated);
}
public override string ToString() => Name;
}
public partial class Emoji
@ -58,17 +60,12 @@ namespace DiscordChatExporter.Core.Models
return $"https://cdn.discordapp.com/emojis/{id}.png";
}
// Standard unicode emoji
// Get runes
var emojiRunes = GetRunes(name).ToArray();
if (emojiRunes.Any())
{
// Get corresponding Twemoji image
var twemojiName = GetTwemojiName(emojiRunes);
return $"https://twemoji.maxcdn.com/2/72x72/{twemojiName}.png";
}
// Fallback in case of failure
return name;
// Get corresponding Twemoji image
var twemojiName = GetTwemojiName(emojiRunes);
return $"https://twemoji.maxcdn.com/2/72x72/{twemojiName}.png";
}
}
}

2
DiscordChatExporter.Core.Models/FileSize.cs → DiscordChatExporter.Domain/Discord/Models/FileSize.cs
View file

@ -1,6 +1,6 @@
using System;
namespace DiscordChatExporter.Core.Models
namespace DiscordChatExporter.Domain.Discord.Models
{
// Loosely based on https://github.com/omar/ByteSize (MIT license)

33
DiscordChatExporter.Core.Models/Guild.cs → DiscordChatExporter.Domain/Discord/Models/Guild.cs
View file

@ -1,9 +1,9 @@
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using DiscordChatExporter.Domain.Internal;
namespace DiscordChatExporter.Core.Models
namespace DiscordChatExporter.Domain.Discord.Models
{
// https://discordapp.com/developers/docs/resources/guild#guild-object
@ -15,21 +15,21 @@ namespace DiscordChatExporter.Core.Models
public string? IconHash { get; }
public string IconUrl => !string.IsNullOrWhiteSpace(IconHash)
? $"https://cdn.discordapp.com/icons/{Id}/{IconHash}.png"
: "https://cdn.discordapp.com/embed/avatars/0.png";
public IReadOnlyList<Role> Roles { get; }
public Dictionary<string, Member?> Members { get; }
public string IconUrl { get; }
public Guild(string id, string name, IReadOnlyList<Role> roles, string? iconHash)
public Guild(string id, string name, string? iconHash, IReadOnlyList<Role> roles)
{
Id = id;
Name = name;
IconHash = iconHash;
Roles = roles;
Members = new Dictionary<string, Member?>();
IconUrl = GetIconUrl(id, iconHash);
}
public override string ToString() => Name;
@ -38,22 +38,17 @@ namespace DiscordChatExporter.Core.Models
public partial class Guild
{
public static string GetUserColor(Guild guild, User user) =>
guild.Members.GetValueOrDefault(user.Id, null)
?.Roles
guild.Members.GetValueOrDefault(user.Id, null)?
.RoleIds
.Select(r => guild.Roles.FirstOrDefault(role => r == role.Id))
.Where(r => r != null)
.Where(r => r.Color != Color.Black)
.Where(r => r.Color.R + r.Color.G + r.Color.B > 0)
.Aggregate<Role, Role?>(null, (a, b) => (a?.Position ?? 0) > b.Position ? a : b)
?.ColorAsHex ?? "";
.Where(r => r.Color != null)
.Aggregate<Role, Role?>(null, (a, b) => (a?.Position ?? 0) > b.Position ? a : b)?
.Color?
.ToHexString() ?? "";
public static string GetUserNick(Guild guild, User user) => guild.Members.GetValueOrDefault(user.Id)?.Nick ?? user.Name;
public static string GetIconUrl(string id, string? iconHash) =>
!string.IsNullOrWhiteSpace(iconHash)
? $"https://cdn.discordapp.com/icons/{id}/{iconHash}.png"
: "https://cdn.discordapp.com/embed/avatars/0.png";
public static Guild DirectMessages { get; } = new Guild("@me", "Direct Messages", Array.Empty<Role>(), null);
public static Guild DirectMessages { get; } = new Guild("@me", "Direct Messages", null, Array.Empty<Role>());
}
}

2
DiscordChatExporter.Core.Models/IHasId.cs → DiscordChatExporter.Domain/Discord/Models/IHasId.cs
View file

@ -1,4 +1,4 @@
namespace DiscordChatExporter.Core.Models
namespace DiscordChatExporter.Domain.Discord.Models
{
public interface IHasId
{

2
DiscordChatExporter.Core.Models/IdBasedEqualityComparer.cs → DiscordChatExporter.Domain/Discord/Models/IdBasedEqualityComparer.cs
View file

@ -1,7 +1,7 @@
using System;
using System.Collections.Generic;
namespace DiscordChatExporter.Core.Models
namespace DiscordChatExporter.Domain.Discord.Models
{
public partial class IdBasedEqualityComparer : IEqualityComparer<IHasId>
{

8
DiscordChatExporter.Core.Models/Member.cs → DiscordChatExporter.Domain/Discord/Models/Member.cs
View file

@ -1,6 +1,6 @@
using System.Collections.Generic;
namespace DiscordChatExporter.Core.Models
namespace DiscordChatExporter.Domain.Discord.Models
{
// https://discordapp.com/developers/docs/resources/guild#guild-member-object
@ -10,13 +10,13 @@ namespace DiscordChatExporter.Core.Models
public string? Nick { get; }
public IReadOnlyList<string> Roles { get; }
public IReadOnlyList<string> RoleIds { get; }
public Member(string userId, string? nick, IReadOnlyList<string> roles)
public Member(string userId, string? nick, IReadOnlyList<string> roleIds)
{
UserId = userId;
Nick = nick;
Roles = roles;
RoleIds = roleIds;
}
}
}

38
DiscordChatExporter.Core.Models/Message.cs → DiscordChatExporter.Domain/Discord/Models/Message.cs
View file

@ -1,8 +1,23 @@
using System;
using System.Collections.Generic;
using System.Linq;
namespace DiscordChatExporter.Core.Models
namespace DiscordChatExporter.Domain.Discord.Models
{
// https://discordapp.com/developers/docs/resources/channel#message-object-message-types
public enum MessageType
{
Default,
RecipientAdd,
RecipientRemove,
Call,
ChannelNameChange,
ChannelIconChange,
ChannelPinnedMessage,
GuildMemberJoin
}
// https://discordapp.com/developers/docs/resources/channel#message-object
public class Message : IHasId
@ -21,7 +36,7 @@ namespace DiscordChatExporter.Core.Models
public bool IsPinned { get; }
public string? Content { get; }
public string Content { get; }
public IReadOnlyList<Attachment> Attachments { get; }
@ -31,10 +46,18 @@ namespace DiscordChatExporter.Core.Models
public IReadOnlyList<User> MentionedUsers { get; }
public Message(string id, string channelId, MessageType type, User author,
DateTimeOffset timestamp, DateTimeOffset? editedTimestamp, bool isPinned,
public Message(
string id,
string channelId,
MessageType type,
User author,
DateTimeOffset timestamp,
DateTimeOffset? editedTimestamp,
bool isPinned,
string content,
IReadOnlyList<Attachment> attachments,IReadOnlyList<Embed> embeds, IReadOnlyList<Reaction> reactions,
IReadOnlyList<Attachment> attachments,
IReadOnlyList<Embed> embeds,
IReadOnlyList<Reaction> reactions,
IReadOnlyList<User> mentionedUsers)
{
Id = id;
@ -51,6 +74,9 @@ namespace DiscordChatExporter.Core.Models
MentionedUsers = mentionedUsers;
}
public override string ToString() => Content ?? "<message without content>";
public override string ToString() =>
Content ?? (Embeds.Any()
? "<embed>"
: "<no content>");
}
}

12
DiscordChatExporter.Core.Models/Reaction.cs → DiscordChatExporter.Domain/Discord/Models/Reaction.cs
View file

@ -1,17 +1,19 @@
namespace DiscordChatExporter.Core.Models
namespace DiscordChatExporter.Domain.Discord.Models
{
// https://discordapp.com/developers/docs/resources/channel#reaction-object
public class Reaction
{
public int Count { get; }
public Emoji Emoji { get; }
public Reaction(int count, Emoji emoji)
public int Count { get; }
public Reaction(Emoji emoji, int count)
{
Count = count;
Emoji = emoji;
Count = count;
}
public override string ToString() => $"{Emoji} ({Count})";
}
}

12
DiscordChatExporter.Core.Models/Role.cs → DiscordChatExporter.Domain/Discord/Models/Role.cs
View file

@ -1,6 +1,6 @@
using System.Drawing;
namespace DiscordChatExporter.Core.Models
namespace DiscordChatExporter.Domain.Discord.Models
{
// https://discordapp.com/developers/docs/topics/permissions#role-object
@ -10,15 +10,11 @@ namespace DiscordChatExporter.Core.Models
public string Name { get; }
public Color Color { get; }
public string ColorAsHex => $"#{Color.ToArgb() & 0xffffff:X6}";
public string ColorAsRgb => $"{Color.R}, {Color.G}, {Color.B}";
public Color? Color { get; }
public int Position { get; }
public Role(string id, string name, Color color, int position)
public Role(string id, string name, Color? color, int position)
{
Id = id;
Name = name;
@ -31,6 +27,6 @@ namespace DiscordChatExporter.Core.Models
public partial class Role
{
public static Role CreateDeletedRole(string id) => new Role(id, "deleted-role", Color.Black, -1);
public static Role CreateDeletedRole(string id) => new Role(id, "deleted-role", null, -1);
}
}

9
DiscordChatExporter.Core.Models/User.cs → DiscordChatExporter.Domain/Discord/Models/User.cs
View file

@ -1,6 +1,6 @@
using System;
namespace DiscordChatExporter.Core.Models
namespace DiscordChatExporter.Domain.Discord.Models
{
// https://discordapp.com/developers/docs/resources/user#user-object
@ -12,7 +12,7 @@ namespace DiscordChatExporter.Core.Models
public string Name { get; }
public string FullName { get; }
public string FullName => $"{Name}#{Discriminator:0000}";
public string? AvatarHash { get; }
@ -28,7 +28,6 @@ namespace DiscordChatExporter.Core.Models
AvatarHash = avatarHash;
IsBot = isBot;
FullName = GetFullName(name, discriminator);
AvatarUrl = GetAvatarUrl(id, discriminator, avatarHash);
}
@ -37,9 +36,7 @@ namespace DiscordChatExporter.Core.Models
public partial class User
{
public static string GetFullName(string name, int discriminator) => $"{name}#{discriminator:0000}";
public static string GetAvatarUrl(string id, int discriminator, string? avatarHash)
private static string GetAvatarUrl(string id, int discriminator, string? avatarHash)
{
// Custom avatar
if (!string.IsNullOrWhiteSpace(avatarHash))

18
DiscordChatExporter.Domain/DiscordChatExporter.Domain.csproj Normal file
View file

@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">
<Import Project="../DiscordChatExporter.props" />
<ItemGroup>
<PackageReference Include="Polly" Version="7.2.0" />
<PackageReference Include="Scriban" Version="2.1.1" />
<PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" />
</ItemGroup>
<ItemGroup>
<EmbeddedResource Include="Exporting\Resources\HtmlCore.css" />
<EmbeddedResource Include="Exporting\Resources\HtmlDark.css" />
<EmbeddedResource Include="Exporting\Resources\HtmlLayoutTemplate.html" />
<EmbeddedResource Include="Exporting\Resources\HtmlLight.css" />
<EmbeddedResource Include="Exporting\Resources\HtmlMessageGroupTemplate.html" />
</ItemGroup>
</Project>

59
DiscordChatExporter.Domain/Exceptions/DiscordChatExporterException.cs Normal file
View file

@ -0,0 +1,59 @@
using System;
using System.Net.Http;
using DiscordChatExporter.Domain.Discord.Models;
namespace DiscordChatExporter.Domain.Exceptions
{
public partial class DiscordChatExporterException : Exception
{
public bool IsCritical { get; }
public DiscordChatExporterException(string message, bool isCritical = false)
: base(message)
{
IsCritical = isCritical;
}
}
public partial class DiscordChatExporterException
{
internal static DiscordChatExporterException FailedHttpRequest(HttpResponseMessage response)
{
var message = $@"
Failed to perform an HTTP request.
{response.RequestMessage}
{response}";
return new DiscordChatExporterException(message.Trim(), true);
}
internal static DiscordChatExporterException Unauthorized()
{
const string message = "Authentication token is invalid.";
return new DiscordChatExporterException(message);
}
internal static DiscordChatExporterException ChannelForbidden(string channel)
{
var message = $"Access to channel '{channel}' is forbidden.";
return new DiscordChatExporterException(message);
}
internal static DiscordChatExporterException ChannelDoesNotExist(string channel)
{
var message = $"Channel '{channel}' does not exist.";
return new DiscordChatExporterException(message);
}
internal static DiscordChatExporterException ChannelEmpty(string channel)
{
var message = $"Channel '{channel}' contains no messages for the specified period.";
return new DiscordChatExporterException(message);
}
internal static DiscordChatExporterException ChannelEmpty(Channel channel) =>
ChannelEmpty(channel.Name);
}
}

36
DiscordChatExporter.Domain/Exporting/ExportFormat.cs Normal file
View file

@ -0,0 +1,36 @@
using System;
namespace DiscordChatExporter.Domain.Exporting
{
public enum ExportFormat
{
PlainText,
HtmlDark,
HtmlLight,
Csv,
Json
}
public static class ExportFormatExtensions
{
public static string GetFileExtension(this ExportFormat format) => format switch
{
ExportFormat.PlainText => "txt",
ExportFormat.HtmlDark => "html",
ExportFormat.HtmlLight => "html",
ExportFormat.Csv => "csv",
ExportFormat.Json => "json",
_ => throw new ArgumentOutOfRangeException(nameof(format))
};
public static string GetDisplayName(this ExportFormat format) => format switch
{
ExportFormat.PlainText => "TXT",
ExportFormat.HtmlDark => "HTML (Dark)",
ExportFormat.HtmlLight => "HTML (Light)",
ExportFormat.Csv => "CSV",
ExportFormat.Json => "JSON",
_ => throw new ArgumentOutOfRangeException(nameof(format))
};
}
}

134
DiscordChatExporter.Domain/Exporting/Exporter.cs Normal file
View file

@ -0,0 +1,134 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Discord;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exceptions;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Domain.Exporting
{
public partial class Exporter
{
private readonly DiscordClient _discord;
public Exporter(DiscordClient discord) => _discord = discord;
public async Task ExportChatLogAsync(Guild guild, Channel channel,
string outputPath, ExportFormat format, string dateFormat, int? partitionLimit,
DateTimeOffset? after = null, DateTimeOffset? before = null, IProgress<double>? progress = null)
{
// Get base file path from output path
var baseFilePath = GetFilePathFromOutputPath(outputPath, format, guild, channel, after, before);
// Create options
var options = new RenderOptions(baseFilePath, format, partitionLimit);
// Create context
var mentionableUsers = new HashSet<User>(IdBasedEqualityComparer.Instance);
var mentionableChannels = await _discord.GetGuildChannelsAsync(guild.Id);
var mentionableRoles = guild.Roles;
var context = new RenderContext
(
guild, channel, after, before, dateFormat,
mentionableUsers, mentionableChannels, mentionableRoles
);
// Create renderer
await using var renderer = new MessageRenderer(options, context);
// Render messages
var renderedAnything = false;
await foreach (var message in _discord.GetMessagesAsync(channel.Id, after, before, progress))
{
// Add encountered users to the list of mentionable users
var encounteredUsers = new List<User>();
encounteredUsers.Add(message.Author);
encounteredUsers.AddRange(message.MentionedUsers);
mentionableUsers.AddRange(encounteredUsers);
foreach (User u in encounteredUsers)
{
if(!guild.Members.ContainsKey(u.Id))
{
var member = await _discord.GetGuildMemberAsync(guild.Id, u.Id);
guild.Members[u.Id] = member;
}
}
// Render message
await renderer.RenderMessageAsync(message);
renderedAnything = true;
}
// Throw if no messages were rendered
if (!renderedAnything)
throw DiscordChatExporterException.ChannelEmpty(channel);
}
}
public partial class Exporter
{
public static string GetDefaultExportFileName(ExportFormat format,
Guild guild, Channel channel,
DateTimeOffset? after = null, DateTimeOffset? before = null)
{
var buffer = new StringBuilder();
// Append guild and channel names
buffer.Append($"{guild.Name} - {channel.Name} [{channel.Id}]");
// Append date range
if (after != null || before != null)
{
buffer.Append(" (");
// Both 'after' and 'before' are set
if (after != null && before != null)
{
buffer.Append($"{after:yyyy-MM-dd} to {before:yyyy-MM-dd}");
}
// Only 'after' is set
else if (after != null)
{
buffer.Append($"after {after:yyyy-MM-dd}");
}
// Only 'before' is set
else
{
buffer.Append($"before {before:yyyy-MM-dd}");
}
buffer.Append(")");
}
// Append extension
buffer.Append($".{format.GetFileExtension()}");
// Replace invalid chars
foreach (var invalidChar in Path.GetInvalidFileNameChars())
buffer.Replace(invalidChar, '_');
return buffer.ToString();
}
private static string GetFilePathFromOutputPath(string outputPath, ExportFormat format, Guild guild, Channel channel,
DateTimeOffset? after, DateTimeOffset? before)
{
// Output is a directory
if (Directory.Exists(outputPath) || string.IsNullOrWhiteSpace(Path.GetExtension(outputPath)))
{
var fileName = GetDefaultExportFileName(format, guild, channel, after, before);
return Path.Combine(outputPath, fileName);
}
// Output is a file
return outputPath;
}
}
}

32
DiscordChatExporter.Domain/Exporting/MessageGroup.cs Normal file
View file

@ -0,0 +1,32 @@
using System;
using System.Collections.Generic;
using DiscordChatExporter.Domain.Discord.Models;
namespace DiscordChatExporter.Domain.Exporting
{
// Used for grouping contiguous messages in HTML export
internal partial class MessageGroup
{
public User Author { get; }
public DateTimeOffset Timestamp { get; }
public IReadOnlyList<Message> Messages { get; }
public MessageGroup(User author, DateTimeOffset timestamp, IReadOnlyList<Message> messages)
{
Author = author;
Timestamp = timestamp;
Messages = messages;
}
}
internal partial class MessageGroup
{
public static bool CanGroup(Message message1, Message message2) =>
string.Equals(message1.Author.Id, message2.Author.Id, StringComparison.Ordinal) &&
string.Equals(message1.Author.FullName, message2.Author.FullName, StringComparison.Ordinal) &&
(message2.Timestamp - message1.Timestamp).Duration().TotalMinutes <= 7;
}
}

47
DiscordChatExporter.Core.Rendering/MessageRenderer.cs → DiscordChatExporter.Domain/Exporting/MessageRenderer.cs
View file

@ -1,12 +1,12 @@
using System;
using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering.Formatters;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exporting.Writers;
namespace DiscordChatExporter.Core.Rendering
namespace DiscordChatExporter.Domain.Exporting
{
public partial class MessageRenderer : IAsyncDisposable
internal partial class MessageRenderer : IAsyncDisposable
{
private readonly RenderOptions _options;
private readonly RenderContext _context;
@ -21,7 +21,7 @@ namespace DiscordChatExporter.Core.Rendering
_context = context;
}
private async Task InitializeWriterAsync()
private async Task<MessageWriterBase> InitializeWriterAsync()
{
// Get partition file path
var filePath = GetPartitionFilePath(_options.BaseFilePath, _partitionIndex);
@ -32,10 +32,12 @@ namespace DiscordChatExporter.Core.Rendering
Directory.CreateDirectory(dirPath);
// Create writer
_writer = CreateMessageWriter(filePath, _options.Format, _context);
var writer = CreateMessageWriter(filePath, _options.Format, _context);
// Write preamble
await _writer.WritePreambleAsync();
await writer.WritePreambleAsync();
return _writer = writer;
}
private async Task ResetWriterAsync()
@ -54,8 +56,7 @@ namespace DiscordChatExporter.Core.Rendering
public async Task RenderMessageAsync(Message message)
{
// Ensure underlying writer is initialized
if (_writer == null)
await InitializeWriterAsync();
_writer ??= await InitializeWriterAsync();
// Render the actual message
await _writer!.WriteMessageAsync(message);
@ -76,7 +77,7 @@ namespace DiscordChatExporter.Core.Rendering
public async ValueTask DisposeAsync() => await ResetWriterAsync();
}
public partial class MessageRenderer
internal partial class MessageRenderer
{
private static string GetPartitionFilePath(string baseFilePath, int partitionIndex)
{
@ -102,23 +103,15 @@ namespace DiscordChatExporter.Core.Rendering
// Create a stream (it will get disposed by the writer)
var stream = File.Create(filePath);
// Create formatter
if (format == ExportFormat.PlainText)
return new PlainTextMessageWriter(stream, context);
if (format == ExportFormat.Csv)
return new CsvMessageWriter(stream, context);
if (format == ExportFormat.HtmlDark)
return new HtmlMessageWriter(stream, context, "Dark");
if (format == ExportFormat.HtmlLight)
return new HtmlMessageWriter(stream, context, "Light");
if (format == ExportFormat.Json)
return new JsonMessageWriter(stream, context);
throw new InvalidOperationException($"Unknown export format [{format}].");
return format switch
{
ExportFormat.PlainText => new PlainTextMessageWriter(stream, context),
ExportFormat.Csv => new CsvMessageWriter(stream, context),
ExportFormat.HtmlDark => new HtmlMessageWriter(stream, context, "Dark"),
ExportFormat.HtmlLight => new HtmlMessageWriter(stream, context, "Light"),
ExportFormat.Json => new JsonMessageWriter(stream, context),
_ => throw new ArgumentOutOfRangeException(nameof(format), $"Unknown export format '{format}'.")
};
}
}
}

16
DiscordChatExporter.Core.Rendering/RenderContext.cs → DiscordChatExporter.Domain/Exporting/RenderContext.cs
View file

@ -1,8 +1,8 @@
using System;
using System.Collections.Generic;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Domain.Discord.Models;
namespace DiscordChatExporter.Core.Rendering
namespace DiscordChatExporter.Domain.Exporting
{
public class RenderContext
{
@ -22,8 +22,16 @@ namespace DiscordChatExporter.Core.Rendering
public IReadOnlyCollection<Role> MentionableRoles { get; }
public RenderContext(Guild guild, Channel channel, DateTimeOffset? after, DateTimeOffset? before, string dateFormat,
IReadOnlyCollection<User> mentionableUsers, IReadOnlyCollection<Channel> mentionableChannels, IReadOnlyCollection<Role> mentionableRoles)
public RenderContext(
Guild guild,
Channel channel,
DateTimeOffset? after,
DateTimeOffset? before,
string dateFormat,
IReadOnlyCollection<User> mentionableUsers,
IReadOnlyCollection<Channel> mentionableChannels,
IReadOnlyCollection<Role> mentionableRoles)
{
Guild = guild;
Channel = channel;

4
DiscordChatExporter.Core.Rendering/RenderOptions.cs → DiscordChatExporter.Domain/Exporting/RenderOptions.cs
View file

@ -1,6 +1,4 @@
using DiscordChatExporter.Core.Models;
namespace DiscordChatExporter.Core.Rendering
namespace DiscordChatExporter.Domain.Exporting
{
public class RenderOptions
{

0
DiscordChatExporter.Core.Rendering/Resources/HtmlCore.css → DiscordChatExporter.Domain/Exporting/Resources/HtmlCore.css
View file

0
DiscordChatExporter.Core.Rendering/Resources/HtmlDark.css → DiscordChatExporter.Domain/Exporting/Resources/HtmlDark.css
View file

0
DiscordChatExporter.Core.Rendering/Resources/HtmlLayoutTemplate.html → DiscordChatExporter.Domain/Exporting/Resources/HtmlLayoutTemplate.html
View file

0
DiscordChatExporter.Core.Rendering/Resources/HtmlLight.css → DiscordChatExporter.Domain/Exporting/Resources/HtmlLight.css
View file

2
DiscordChatExporter.Core.Rendering/Resources/HtmlMessageGroupTemplate.html → DiscordChatExporter.Domain/Exporting/Resources/HtmlMessageGroupTemplate.html
View file

@ -5,7 +5,7 @@
</div>
<div class="chatlog__messages">
{{~ # Author name and timestamp ~}}
<span class="chatlog__author-name" title="{{ MessageGroup.Author.FullName | html.escape }}" data-user-id="{{ MessageGroup.Author.Id | html.escape }}" style="color: {{ GetUserColor Context.Guild MessageGroup.Author }}">{{ GetUserNick Context.Guild MessageGroup.Author | html.escape }}</span>
<span class="chatlog__author-name" title="{{ MessageGroup.Author.FullName | html.escape }}" data-user-id="{{ MessageGroup.Author.Id | html.escape }}" {{ if GetUserColor Context.Guild MessageGroup.Author }} style="color: {{ GetUserColor Context.Guild MessageGroup.Author }}" {{ end }}>{{ GetUserNick Context.Guild MessageGroup.Author | html.escape }}</span>
{{~ # Bot tag ~}}
{{~ if MessageGroup.Author.IsBot ~}}

58
DiscordChatExporter.Domain/Exporting/Writers/CsvMessageWriter.cs Normal file
View file

@ -0,0 +1,58 @@
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors;
using DiscordChatExporter.Domain.Internal;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Domain.Exporting.Writers
{
internal class CsvMessageWriter : MessageWriterBase
{
private readonly TextWriter _writer;
public CsvMessageWriter(Stream stream, RenderContext context)
: base(stream, context)
{
_writer = new StreamWriter(stream);
}
private string EncodeValue(string value)
{
value = value.Replace("\"", "\"\"");
return $"\"{value}\"";
}
private string FormatMarkdown(string markdown) =>
PlainTextMarkdownVisitor.Format(Context, markdown);
private string FormatMessage(Message message)
{
var buffer = new StringBuilder();
buffer
.Append(EncodeValue(message.Author.Id)).Append(',')
.Append(EncodeValue(message.Author.FullName)).Append(',')
.Append(EncodeValue(message.Timestamp.ToLocalString(Context.DateFormat))).Append(',')
.Append(EncodeValue(FormatMarkdown(message.Content))).Append(',')
.Append(EncodeValue(message.Attachments.Select(a => a.Url).JoinToString(","))).Append(',')
.Append(EncodeValue(message.Reactions.Select(r => $"{r.Emoji.Name} ({r.Count})").JoinToString(",")));
return buffer.ToString();
}
public override async Task WritePreambleAsync() =>
await _writer.WriteLineAsync("AuthorID,Author,Date,Content,Attachments,Reactions");
public override async Task WriteMessageAsync(Message message) =>
await _writer.WriteLineAsync(FormatMessage(message));
public override async ValueTask DisposeAsync()
{
await _writer.DisposeAsync();
await base.DisposeAsync();
}
}
}

33
DiscordChatExporter.Core.Rendering/Formatters/HtmlMessageWriter.cs → DiscordChatExporter.Domain/Exporting/Writers/HtmlMessageWriter.cs
View file

@ -1,18 +1,24 @@
using System;
using System.Collections.Generic;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering.Logic;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Markdown;
using DiscordChatExporter.Domain.Markdown.Ast;
using Scriban;
using Scriban.Runtime;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Rendering.Formatters
namespace DiscordChatExporter.Domain.Exporting.Writers
{
public partial class HtmlMessageWriter : MessageWriterBase
internal partial class HtmlMessageWriter : MessageWriterBase
{
private readonly TextWriter _writer;
private readonly string _themeName;
@ -70,13 +76,13 @@ namespace DiscordChatExporter.Core.Rendering.Formatters
// Functions
scriptObject.Import("FormatDate",
new Func<DateTimeOffset, string>(d => SharedRenderingLogic.FormatDate(d, Context.DateFormat)));
new Func<DateTimeOffset, string>(d => d.ToLocalString(Context.DateFormat)));
scriptObject.Import("FormatMarkdown",
new Func<string, string>(m => HtmlRenderingLogic.FormatMarkdown(Context, m)));
new Func<string, string>(FormatMarkdown));
scriptObject.Import("GetUserColor", new Func<Guild, User, string>(Guild.GetUserColor));
scriptObject.Import("GetUserNick", new Func<Guild, User, string>(Guild.GetUserNick));
// Push model
@ -88,6 +94,9 @@ namespace DiscordChatExporter.Core.Rendering.Formatters
return templateContext;
}
private string FormatMarkdown(string markdown) =>
HtmlMarkdownVisitor.Format(Context, markdown);
private async Task RenderCurrentMessageGroupAsync()
{
var templateContext = CreateTemplateContext(new Dictionary<string, object>
@ -107,7 +116,7 @@ namespace DiscordChatExporter.Core.Rendering.Formatters
public override async Task WriteMessageAsync(Message message)
{
// If message group is empty or the given message can be grouped, buffer the given message
if (!_messageGroupBuffer.Any() || HtmlRenderingLogic.CanBeGrouped(_messageGroupBuffer.Last(), message))
if (!_messageGroupBuffer.Any() || MessageGroup.CanGroup(_messageGroupBuffer.Last(), message))
{
_messageGroupBuffer.Add(message);
}
@ -145,10 +154,10 @@ namespace DiscordChatExporter.Core.Rendering.Formatters
}
}
public partial class HtmlMessageWriter
internal partial class HtmlMessageWriter
{
private static readonly Assembly ResourcesAssembly = typeof(HtmlRenderingLogic).Assembly;
private static readonly string ResourcesNamespace = $"{ResourcesAssembly.GetName().Name}.Resources";
private static readonly Assembly ResourcesAssembly = typeof(HtmlMessageWriter).Assembly;
private static readonly string ResourcesNamespace = $"{ResourcesAssembly.GetName().Name}.Exporting.Resources";
private static string GetCoreStyleSheetCode() =>
ResourcesAssembly
@ -171,5 +180,7 @@ namespace DiscordChatExporter.Core.Rendering.Formatters
ResourcesAssembly
.GetManifestResourceString($"{ResourcesNamespace}.HtmlLayoutTemplate.html")
.SubstringAfter("{{~ %SPLIT% ~}}");
private static string HtmlEncode(string s) => WebUtility.HtmlEncode(s);
}
}

12
DiscordChatExporter.Core.Rendering/Formatters/JsonMessageWriter.cs → DiscordChatExporter.Domain/Exporting/Writers/JsonMessageWriter.cs
View file

@ -1,13 +1,13 @@
using System.IO;
using System.Text.Json;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering.Internal;
using DiscordChatExporter.Core.Rendering.Logic;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors;
using DiscordChatExporter.Domain.Internal;
namespace DiscordChatExporter.Core.Rendering.Formatters
namespace DiscordChatExporter.Domain.Exporting.Writers
{
public class JsonMessageWriter : MessageWriterBase
internal class JsonMessageWriter : MessageWriterBase
{
private readonly Utf8JsonWriter _writer;
@ -66,7 +66,7 @@ namespace DiscordChatExporter.Core.Rendering.Formatters
_writer.WriteBoolean("isPinned", message.IsPinned);
// Content
var content = PlainTextRenderingLogic.FormatMessageContent(Context, message);
var content = PlainTextMarkdownVisitor.Format(Context, message.Content);
_writer.WriteString("content", content);
// Author

177
DiscordChatExporter.Domain/Exporting/Writers/MarkdownVisitors/HtmlMarkdownVisitor.cs Normal file
View file

@ -0,0 +1,177 @@
using System;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Markdown;
using DiscordChatExporter.Domain.Markdown.Ast;
namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
{
internal partial class HtmlMarkdownVisitor : MarkdownVisitor
{
private readonly RenderContext _context;
private readonly StringBuilder _buffer;
private readonly bool _isJumbo;
public HtmlMarkdownVisitor(RenderContext context, StringBuilder buffer, bool isJumbo)
{
_context = context;
_buffer = buffer;
_isJumbo = isJumbo;
}
public override MarkdownNode VisitText(TextNode text)
{
_buffer.Append(HtmlEncode(text.Text));
return base.VisitText(text);
}
public override MarkdownNode VisitFormatted(FormattedNode formatted)
{
var (tagOpen, tagClose) = formatted.Formatting switch
{
TextFormatting.Bold => ("<strong>", "</strong>"),
TextFormatting.Italic => ("<em>", "</em>"),
TextFormatting.Underline => ("<u>", "</u>"),
TextFormatting.Strikethrough => ("<s>", "</s>"),
TextFormatting.Spoiler => (
"<span class=\"spoiler spoiler--hidden\" onclick=\"showSpoiler(event, this)\"><span class=\"spoiler-text\">", "</span>"),
TextFormatting.Quote => ("<div class=\"quote\">", "</div>"),
_ => throw new ArgumentOutOfRangeException(nameof(formatted.Formatting))
};
_buffer.Append(tagOpen);
var result = base.VisitFormatted(formatted);
_buffer.Append(tagClose);
return result;
}
public override MarkdownNode VisitInlineCodeBlock(InlineCodeBlockNode inlineCodeBlock)
{
_buffer
.Append("<span class=\"pre pre--inline\">")
.Append(HtmlEncode(inlineCodeBlock.Code))
.Append("</span>");
return base.VisitInlineCodeBlock(inlineCodeBlock);
}
public override MarkdownNode VisitMultiLineCodeBlock(MultiLineCodeBlockNode multiLineCodeBlock)
{
var highlightCssClass = !string.IsNullOrWhiteSpace(multiLineCodeBlock.Language)
? $"language-{multiLineCodeBlock.Language}"
: "nohighlight";
_buffer
.Append($"<div class=\"pre pre--multiline {highlightCssClass}\">")
.Append(HtmlEncode(multiLineCodeBlock.Code))
.Append("</div>");
return base.VisitMultiLineCodeBlock(multiLineCodeBlock);
}
public override MarkdownNode VisitMention(MentionNode mention)
{
if (mention.Type == MentionType.Meta)
{
_buffer
.Append("<span class=\"mention\">")
.Append("@").Append(HtmlEncode(mention.Id))
.Append("</span>");
}
else if (mention.Type == MentionType.User)
{
var user = _context.MentionableUsers.FirstOrDefault(u => u.Id == mention.Id) ??
User.CreateUnknownUser(mention.Id);
var nick = Guild.GetUserNick(_context.Guild, user);
_buffer
.Append($"<span class=\"mention\" title=\"{HtmlEncode(user.FullName)}\">")
.Append("@").Append(HtmlEncode(nick))
.Append("</span>");
}
else if (mention.Type == MentionType.Channel)
{
var channel = _context.MentionableChannels.FirstOrDefault(c => c.Id == mention.Id) ??
Channel.CreateDeletedChannel(mention.Id);
_buffer
.Append("<span class=\"mention\">")
.Append("#").Append(HtmlEncode(channel.Name))
.Append("</span>");
}
else if (mention.Type == MentionType.Role)
{
var role = _context.MentionableRoles.FirstOrDefault(r => r.Id == mention.Id) ??
Role.CreateDeletedRole(mention.Id);
var style = role.Color != null
? $"color: {role.Color.Value.ToHexString()}; background-color: rgba({role.Color.Value.ToRgbString()}, 0.1);"
: "";
_buffer
.Append($"<span class=\"mention\" style=\"{style}>\"")
.Append("@").Append(HtmlEncode(role.Name))
.Append("</span>");
}
return base.VisitMention(mention);
}
public override MarkdownNode VisitEmoji(EmojiNode emoji)
{
var emojiImageUrl = Emoji.GetImageUrl(emoji.Id, emoji.Name, emoji.IsAnimated);
var jumboClass = _isJumbo ? "emoji--large" : "";
_buffer
.Append($"<img class=\"emoji {jumboClass}\" alt=\"{emoji.Name}\" title=\"{emoji.Name}\" src=\"{emojiImageUrl}\" />");
return base.VisitEmoji(emoji);
}
public override MarkdownNode VisitLink(LinkNode link)
{
// Extract message ID if the link points to a Discord message
var linkedMessageId = Regex.Match(link.Url, "^https?://discordapp.com/channels/.*?/(\\d+)/?$").Groups[1].Value;
if (!string.IsNullOrWhiteSpace(linkedMessageId))
{
_buffer
.Append($"<a href=\"{Uri.EscapeUriString(link.Url)}\" onclick=\"scrollToMessage(event, '{linkedMessageId}')\">")
.Append(HtmlEncode(link.Title))
.Append("</a>");
}
else
{
_buffer
.Append($"<a href=\"{Uri.EscapeUriString(link.Url)}\">")
.Append(HtmlEncode(link.Title))
.Append("</a>");
}
return base.VisitLink(link);
}
}
internal partial class HtmlMarkdownVisitor
{
private static string HtmlEncode(string text) => WebUtility.HtmlEncode(text);
public static string Format(RenderContext context, string markdown)
{
var nodes = MarkdownParser.Parse(markdown);
var isJumbo = nodes.All(n => n is EmojiNode || n is TextNode textNode && string.IsNullOrWhiteSpace(textNode.Text));
var buffer = new StringBuilder();
new HtmlMarkdownVisitor(context, buffer, isJumbo).Visit(nodes);
return buffer.ToString();
}
}
}

75
DiscordChatExporter.Domain/Exporting/Writers/MarkdownVisitors/PlainTextMarkdownVisitor.cs Normal file
View file

@ -0,0 +1,75 @@
using System.Linq;
using System.Text;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Markdown;
using DiscordChatExporter.Domain.Markdown.Ast;
namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
{
internal partial class PlainTextMarkdownVisitor : MarkdownVisitor
{
private readonly RenderContext _context;
private readonly StringBuilder _buffer;
public PlainTextMarkdownVisitor(RenderContext context, StringBuilder buffer)
{
_context = context;
_buffer = buffer;
}
public override MarkdownNode VisitText(TextNode text)
{
_buffer.Append(text.Text);
return base.VisitText(text);
}
public override MarkdownNode VisitMention(MentionNode mention)
{
if (mention.Type == MentionType.User)
{
var user = _context.MentionableUsers.FirstOrDefault(u => u.Id == mention.Id) ??
User.CreateUnknownUser(mention.Id);
_buffer.Append($"@{user.Name}");
}
else if (mention.Type == MentionType.Channel)
{
var channel = _context.MentionableChannels.FirstOrDefault(c => c.Id == mention.Id) ??
Channel.CreateDeletedChannel(mention.Id);
_buffer.Append($"#{channel.Name}");
}
else if (mention.Type == MentionType.Role)
{
var role = _context.MentionableRoles.FirstOrDefault(r => r.Id == mention.Id) ??
Role.CreateDeletedRole(mention.Id);
_buffer.Append($"@{role.Name}");
}
return base.VisitMention(mention);
}
public override MarkdownNode VisitEmoji(EmojiNode emoji)
{
_buffer.Append(emoji.IsCustomEmoji
? $":{emoji.Name}:"
: emoji.Name);
return base.VisitEmoji(emoji);
}
}
internal partial class PlainTextMarkdownVisitor
{
public static string Format(RenderContext context, string markdown)
{
var nodes = MarkdownParser.ParseMinimal(markdown);
var buffer = new StringBuilder();
new PlainTextMarkdownVisitor(context, buffer).Visit(nodes);
return buffer.ToString();
}
}
}

6
DiscordChatExporter.Core.Rendering/Formatters/MessageWriterBase.cs → DiscordChatExporter.Domain/Exporting/Writers/MessageWriterBase.cs
View file

@ -1,11 +1,11 @@
using System;
using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Domain.Discord.Models;
namespace DiscordChatExporter.Core.Rendering.Formatters
namespace DiscordChatExporter.Domain.Exporting.Writers
{
public abstract class MessageWriterBase : IAsyncDisposable
internal abstract class MessageWriterBase : IAsyncDisposable
{
protected Stream Stream { get; }

224
DiscordChatExporter.Domain/Exporting/Writers/PlainTextMessageWriter.cs Normal file
View file

@ -0,0 +1,224 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors;
using DiscordChatExporter.Domain.Internal;
namespace DiscordChatExporter.Domain.Exporting.Writers
{
internal class PlainTextMessageWriter : MessageWriterBase
{
private readonly TextWriter _writer;
private long _messageCount;
public PlainTextMessageWriter(Stream stream, RenderContext context)
: base(stream, context)
{
_writer = new StreamWriter(stream);
}
private string FormatPreamble()
{
var buffer = new StringBuilder();
buffer.Append('=', 62).AppendLine();
buffer.AppendLine($"Guild: {Context.Guild.Name}");
buffer.AppendLine($"Channel: {Context.Channel.Name}");
if (!string.IsNullOrWhiteSpace(Context.Channel.Topic))
buffer.AppendLine($"Topic: {Context.Channel.Topic}");
if (Context.After != null)
buffer.AppendLine($"After: {Context.After.Value.ToLocalString(Context.DateFormat)}");
if (Context.Before != null)
buffer.AppendLine($"Before: {Context.Before.Value.ToLocalString(Context.DateFormat)}");
buffer.Append('=', 62).AppendLine();
return buffer.ToString();
}
private string FormatPostamble()
{
var buffer = new StringBuilder();
buffer.Append('=', 62).AppendLine();
buffer.AppendLine($"Exported {_messageCount:N0} message(s)");
buffer.Append('=', 62).AppendLine();
return buffer.ToString();
}
private string FormatMarkdown(string markdown) =>
PlainTextMarkdownVisitor.Format(Context, markdown);
private string FormatMessageHeader(Message message)
{
var buffer = new StringBuilder();
// Timestamp & author
buffer
.Append($"[{message.Timestamp.ToLocalString(Context.DateFormat)}]")
.Append(' ')
.Append($"{message.Author.FullName}");
// Whether the message is pinned
if (message.IsPinned)
{
buffer.Append(' ').Append("(pinned)");
}
return buffer.ToString();
}
private string FormatMessageContent(Message message)
{
if (string.IsNullOrWhiteSpace(message.Content))
return "";
return FormatMarkdown(message.Content);
}
private string FormatAttachments(IReadOnlyList<Attachment> attachments)
{
if (!attachments.Any())
return "";
var buffer = new StringBuilder();
buffer
.AppendLine("{Attachments}")
.AppendJoin(Environment.NewLine, attachments.Select(a => a.Url))
.AppendLine();
return buffer.ToString();
}
private string FormatEmbeds(IReadOnlyList<Embed> embeds)
{
if (!embeds.Any())
return "";
var buffer = new StringBuilder();
foreach (var embed in embeds)
{
buffer.AppendLine("{Embed}");
// Author name
if (!string.IsNullOrWhiteSpace(embed.Author?.Name))
buffer.AppendLine(embed.Author.Name);
// URL
if (!string.IsNullOrWhiteSpace(embed.Url))
buffer.AppendLine(embed.Url);
// Title
if (!string.IsNullOrWhiteSpace(embed.Title))
buffer.AppendLine(FormatMarkdown(embed.Title));
// Description
if (!string.IsNullOrWhiteSpace(embed.Description))
buffer.AppendLine(FormatMarkdown(embed.Description));
// Fields
foreach (var field in embed.Fields)
{
// Name
if (!string.IsNullOrWhiteSpace(field.Name))
buffer.AppendLine(field.Name);
// Value
if (!string.IsNullOrWhiteSpace(field.Value))
buffer.AppendLine(field.Value);
}
// Thumbnail URL
if (!string.IsNullOrWhiteSpace(embed.Thumbnail?.Url))
buffer.AppendLine(embed.Thumbnail?.Url);
// Image URL
if (!string.IsNullOrWhiteSpace(embed.Image?.Url))
buffer.AppendLine(embed.Image?.Url);
// Footer text
if (!string.IsNullOrWhiteSpace(embed.Footer?.Text))
buffer.AppendLine(embed.Footer?.Text);
buffer.AppendLine();
}
return buffer.ToString();
}
private string FormatReactions(IReadOnlyList<Reaction> reactions)
{
if (!reactions.Any())
return "";
var buffer = new StringBuilder();
buffer.AppendLine("{Reactions}");
foreach (var reaction in reactions)
{
buffer.Append(reaction.Emoji.Name);
if (reaction.Count > 1)
buffer.Append($" ({reaction.Count})");
buffer.Append(" ");
}
buffer.AppendLine();
return buffer.ToString();
}
private string FormatMessage(Message message)
{
var buffer = new StringBuilder();
buffer
.AppendLine(FormatMessageHeader(message))
.AppendLineIfNotEmpty(FormatMessageContent(message))
.AppendLine()
.AppendLineIfNotEmpty(FormatAttachments(message.Attachments))
.AppendLineIfNotEmpty(FormatEmbeds(message.Embeds))
.AppendLineIfNotEmpty(FormatReactions(message.Reactions));
return buffer.Trim().ToString();
}
public override async Task WritePreambleAsync()
{
await _writer.WriteLineAsync(FormatPreamble());
}
public override async Task WriteMessageAsync(Message message)
{
await _writer.WriteLineAsync(FormatMessage(message));
await _writer.WriteLineAsync();
_messageCount++;
}
public override async Task WritePostambleAsync()
{
await _writer.WriteLineAsync();
await _writer.WriteLineAsync(FormatPostamble());
}
public override async ValueTask DisposeAsync()
{
await _writer.DisposeAsync();
await base.DisposeAsync();
}
}
}

15
DiscordChatExporter.Domain/Internal/ColorExtensions.cs Normal file
View file

@ -0,0 +1,15 @@
using System.Drawing;
namespace DiscordChatExporter.Domain.Internal
{
internal static class ColorExtensions
{
public static Color ResetAlpha(this Color color) => Color.FromArgb(1, color);
public static int ToRgb(this Color color) => color.ToArgb() & 0xffffff;
public static string ToHexString(this Color color) => $"#{color.ToRgb():x6}";
public static string ToRgbString(this Color color) => $"{color.R}, {color.G}, {color.B}";
}
}

6
DiscordChatExporter.Core.Services/Internal/Extensions/DateExtensions.cs → DiscordChatExporter.Domain/Internal/DateExtensions.cs
View file

@ -1,6 +1,7 @@
using System;
using System.Globalization;
namespace DiscordChatExporter.Core.Services.Internal.Extensions
namespace DiscordChatExporter.Domain.Internal
{
internal static class DateExtensions
{
@ -9,5 +10,8 @@ namespace DiscordChatExporter.Core.Services.Internal.Extensions
var value = ((ulong) dateTime.ToUnixTimeMilliseconds() - 1420070400000UL) << 22;
return value.ToString();
}
public static string ToLocalString(this DateTimeOffset dateTime, string format) =>
dateTime.ToLocalTime().ToString(format, CultureInfo.InvariantCulture);
}
}

14
DiscordChatExporter.Domain/Internal/GenericExtensions.cs Normal file
View file

@ -0,0 +1,14 @@
using System;
namespace DiscordChatExporter.Domain.Internal
{
internal static class GenericExtensions
{
public static TOut Pipe<TIn, TOut>(this TIn input, Func<TIn, TOut> transform) => transform(input);
public static T? NullIf<T>(this T value, Func<T, bool> predicate) where T : struct =>
!predicate(value)
? value
: (T?) null;
}
}

17
DiscordChatExporter.Domain/Internal/HttpClientExtensions.cs Normal file
View file

@ -0,0 +1,17 @@
using System.Net.Http;
using System.Text.Json;
using System.Threading.Tasks;
namespace DiscordChatExporter.Domain.Internal
{
internal static class HttpClientExtensions
{
public static async Task<JsonElement> ReadAsJsonAsync(this HttpContent content)
{
await using var stream = await content.ReadAsStreamAsync();
using var doc = await JsonDocument.ParseAsync(stream);
return doc.RootElement.Clone();
}
}
}

2
DiscordChatExporter.Core.Services/Internal/Extensions/JsonElementExtensions.cs → DiscordChatExporter.Domain/Internal/JsonElementExtensions.cs
View file

@ -1,6 +1,6 @@
using System.Text.Json;
namespace DiscordChatExporter.Core.Services.Internal.Extensions
namespace DiscordChatExporter.Domain.Internal
{
internal static class JsonElementExtensions
{

21
DiscordChatExporter.Domain/Internal/StringExtensions.cs Normal file
View file

@ -0,0 +1,21 @@
using System.Text;
namespace DiscordChatExporter.Domain.Internal
{
internal static class StringExtensions
{
public static StringBuilder AppendLineIfNotEmpty(this StringBuilder builder, string value) =>
!string.IsNullOrWhiteSpace(value) ? builder.AppendLine(value) : builder;
public static StringBuilder Trim(this StringBuilder builder)
{
while (builder.Length > 0 && char.IsWhiteSpace(builder[0]))
builder.Remove(0, 1);
while (builder.Length > 0 && char.IsWhiteSpace(builder[^1]))
builder.Remove(builder.Length - 1, 1);
return builder;
}
}
}

19
DiscordChatExporter.Core.Rendering/Internal/Extensions.cs → DiscordChatExporter.Domain/Internal/Utf8JsonWriterExtensions.cs
View file

@ -1,25 +1,10 @@
using System;
using System.Text;
using System.Text.Json;
namespace DiscordChatExporter.Core.Rendering.Internal
namespace DiscordChatExporter.Domain.Internal
{
internal static class Extensions
internal static class Utf8JsonWriterExtensions
{
public static StringBuilder AppendLineIfNotEmpty(this StringBuilder builder, string value) =>
!string.IsNullOrWhiteSpace(value) ? builder.AppendLine(value) : builder;
public static StringBuilder Trim(this StringBuilder builder)
{
while (builder.Length > 0 && char.IsWhiteSpace(builder[0]))
builder.Remove(0, 1);
while (builder.Length > 0 && char.IsWhiteSpace(builder[^1]))
builder.Remove(builder.Length - 1, 1);
return builder;
}
public static void WriteString(this Utf8JsonWriter writer, string propertyName, DateTimeOffset? value)
{
writer.WritePropertyName(propertyName);

4
DiscordChatExporter.Core.Markdown/Ast/EmojiNode.cs → DiscordChatExporter.Domain/Markdown/Ast/EmojiNode.cs
View file

@ -1,6 +1,6 @@
namespace DiscordChatExporter.Core.Markdown.Ast
namespace DiscordChatExporter.Domain.Markdown.Ast
{
public class EmojiNode : Node
internal class EmojiNode : MarkdownNode
{
public string? Id { get; }

29
DiscordChatExporter.Domain/Markdown/Ast/FormattedNode.cs Normal file
View file

@ -0,0 +1,29 @@
using System.Collections.Generic;
namespace DiscordChatExporter.Domain.Markdown.Ast
{
internal enum TextFormatting
{
Bold,
Italic,
Underline,
Strikethrough,
Spoiler,
Quote
}
internal class FormattedNode : MarkdownNode
{
public TextFormatting Formatting { get; }
public IReadOnlyList<MarkdownNode> Children { get; }
public FormattedNode(TextFormatting formatting, IReadOnlyList<MarkdownNode> children)
{
Formatting = formatting;
Children = children;
}
public override string ToString() => $"<{Formatting}> (+{Children.Count})";
}
}

4
DiscordChatExporter.Core.Markdown/Ast/InlineCodeBlockNode.cs → DiscordChatExporter.Domain/Markdown/Ast/InlineCodeBlockNode.cs
View file

@ -1,6 +1,6 @@
namespace DiscordChatExporter.Core.Markdown.Ast
namespace DiscordChatExporter.Domain.Markdown.Ast
{
public class InlineCodeBlockNode : Node
internal class InlineCodeBlockNode : MarkdownNode
{
public string Code { get; }

4
DiscordChatExporter.Core.Markdown/Ast/LinkNode.cs → DiscordChatExporter.Domain/Markdown/Ast/LinkNode.cs
View file

@ -1,6 +1,6 @@
namespace DiscordChatExporter.Core.Markdown.Ast
namespace DiscordChatExporter.Domain.Markdown.Ast
{
public class LinkNode : Node
internal class LinkNode : MarkdownNode
{
public string Url { get; }

6
DiscordChatExporter.Domain/Markdown/Ast/MarkdownNode.cs Normal file
View file

@ -0,0 +1,6 @@
namespace DiscordChatExporter.Domain.Markdown.Ast
{
internal abstract class MarkdownNode
{
}
}

12
DiscordChatExporter.Core.Markdown/Ast/MentionNode.cs → DiscordChatExporter.Domain/Markdown/Ast/MentionNode.cs
View file

@ -1,6 +1,14 @@
namespace DiscordChatExporter.Core.Markdown.Ast
namespace DiscordChatExporter.Domain.Markdown.Ast
{
public class MentionNode : Node
internal enum MentionType
{
Meta,
User,
Channel,
Role
}
internal class MentionNode : MarkdownNode
{
public string Id { get; }

6
DiscordChatExporter.Core.Markdown/Ast/MultiLineCodeBlockNode.cs → DiscordChatExporter.Domain/Markdown/Ast/MultiLineCodeBlockNode.cs
View file

@ -1,6 +1,6 @@
namespace DiscordChatExporter.Core.Markdown.Ast
namespace DiscordChatExporter.Domain.Markdown.Ast
{
public class MultiLineCodeBlockNode : Node
internal class MultiLineCodeBlockNode : MarkdownNode
{
public string Language { get; }
@ -12,6 +12,6 @@
Code = code;
}
public override string ToString() => $"<Code [{Language}]> {Code}";
public override string ToString() => $"<{Language}> {Code}";
}
}

4
DiscordChatExporter.Core.Markdown/Ast/TextNode.cs → DiscordChatExporter.Domain/Markdown/Ast/TextNode.cs
View file

@ -1,6 +1,6 @@
namespace DiscordChatExporter.Core.Markdown.Ast
namespace DiscordChatExporter.Domain.Markdown.Ast
{
public class TextNode : Node
internal class TextNode : MarkdownNode
{
public string Text { get; }

85
DiscordChatExporter.Core.Markdown/MarkdownParser.cs → DiscordChatExporter.Domain/Markdown/MarkdownParser.cs
View file

@ -1,70 +1,70 @@
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using DiscordChatExporter.Core.Markdown.Ast;
using DiscordChatExporter.Core.Markdown.Internal;
using DiscordChatExporter.Domain.Markdown.Ast;
using DiscordChatExporter.Domain.Markdown.Matching;
namespace DiscordChatExporter.Core.Markdown
namespace DiscordChatExporter.Domain.Markdown
{
// The following parsing logic is meant to replicate Discord's markdown grammar as close as possible
public static class MarkdownParser
internal static partial class MarkdownParser
{
private const RegexOptions DefaultRegexOptions = RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.Multiline;
/* Formatting */
// Capture any character until the earliest double asterisk not followed by an asterisk
private static readonly IMatcher<Node> BoldFormattedNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> BoldFormattedNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("\\*\\*(.+?)\\*\\*(?!\\*)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattedNode(TextFormatting.Bold, Parse(p.Slice(m.Groups[1]))));
// Capture any character until the earliest single asterisk not preceded or followed by an asterisk
// Opening asterisk must not be followed by whitespace
// Closing asterisk must not be preceded by whitespace
private static readonly IMatcher<Node> ItalicFormattedNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> ItalicFormattedNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("\\*(?!\\s)(.+?)(?<!\\s|\\*)\\*(?!\\*)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattedNode(TextFormatting.Italic, Parse(p.Slice(m.Groups[1]))));
// Capture any character until the earliest triple asterisk not followed by an asterisk
private static readonly IMatcher<Node> ItalicBoldFormattedNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> ItalicBoldFormattedNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("\\*(\\*\\*.+?\\*\\*)\\*(?!\\*)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattedNode(TextFormatting.Italic, Parse(p.Slice(m.Groups[1]), BoldFormattedNodeMatcher)));
// Capture any character except underscore until an underscore
// Closing underscore must not be followed by a word character
private static readonly IMatcher<Node> ItalicAltFormattedNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> ItalicAltFormattedNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("_([^_]+)_(?!\\w)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattedNode(TextFormatting.Italic, Parse(p.Slice(m.Groups[1]))));
// Capture any character until the earliest double underscore not followed by an underscore
private static readonly IMatcher<Node> UnderlineFormattedNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> UnderlineFormattedNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("__(.+?)__(?!_)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattedNode(TextFormatting.Underline, Parse(p.Slice(m.Groups[1]))));
// Capture any character until the earliest triple underscore not followed by an underscore
private static readonly IMatcher<Node> ItalicUnderlineFormattedNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> ItalicUnderlineFormattedNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("_(__.+?__)_(?!_)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattedNode(TextFormatting.Italic, Parse(p.Slice(m.Groups[1]), UnderlineFormattedNodeMatcher)));
// Capture any character until the earliest double tilde
private static readonly IMatcher<Node> StrikethroughFormattedNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> StrikethroughFormattedNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("~~(.+?)~~", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattedNode(TextFormatting.Strikethrough, Parse(p.Slice(m.Groups[1]))));
// Capture any character until the earliest double pipe
private static readonly IMatcher<Node> SpoilerFormattedNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> SpoilerFormattedNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("\\|\\|(.+?)\\|\\|", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattedNode(TextFormatting.Spoiler, Parse(p.Slice(m.Groups[1]))));
// Capture any character until the end of the line
// Opening 'greater than' character must be followed by whitespace
private static readonly IMatcher<Node> SingleLineQuoteNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> SingleLineQuoteNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("^>\\s(.+\n?)", DefaultRegexOptions),
(p, m) => new FormattedNode(TextFormatting.Quote, Parse(p.Slice(m.Groups[1]))));
// Repeatedly capture any character until the end of the line
// This one is tricky as it ends up producing multiple separate captures which need to be joined
private static readonly IMatcher<Node> RepeatedSingleLineQuoteNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> RepeatedSingleLineQuoteNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("(?:^>\\s(.+\n?)){2,}", DefaultRegexOptions),
(p, m) =>
{
@ -74,7 +74,7 @@ namespace DiscordChatExporter.Core.Markdown
// Capture any character until the end of the input
// Opening 'greater than' characters must be followed by whitespace
private static readonly IMatcher<Node> MultiLineQuoteNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> MultiLineQuoteNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("^>>>\\s(.+)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattedNode(TextFormatting.Quote, Parse(p.Slice(m.Groups[1]))));
@ -82,41 +82,42 @@ namespace DiscordChatExporter.Core.Markdown
// Capture any character except backtick until a backtick
// Blank lines at the beginning and end of content are trimmed
private static readonly IMatcher<Node> InlineCodeBlockNodeMatcher = new RegexMatcher<Node>(
// There can be either one or two backticks, but equal number on both sides
private static readonly IMatcher<MarkdownNode> InlineCodeBlockNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("`([^`]+)`", DefaultRegexOptions | RegexOptions.Singleline),
m => new InlineCodeBlockNode(m.Groups[1].Value.Trim('\r', '\n')));
// Capture language identifier and then any character until the earliest triple backtick
// Language identifier is one word immediately after opening backticks, followed immediately by newline
// Blank lines at the beginning and end of content are trimmed
private static readonly IMatcher<Node> MultiLineCodeBlockNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> MultiLineCodeBlockNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("```(?:(\\w*)\\n)?(.+?)```", DefaultRegexOptions | RegexOptions.Singleline),
m => new MultiLineCodeBlockNode(m.Groups[1].Value, m.Groups[2].Value.Trim('\r', '\n')));
/* Mentions */
// Capture @everyone
private static readonly IMatcher<Node> EveryoneMentionNodeMatcher = new StringMatcher<Node>(
private static readonly IMatcher<MarkdownNode> EveryoneMentionNodeMatcher = new StringMatcher<MarkdownNode>(
"@everyone",
p => new MentionNode("everyone", MentionType.Meta));
// Capture @here
private static readonly IMatcher<Node> HereMentionNodeMatcher = new StringMatcher<Node>(
private static readonly IMatcher<MarkdownNode> HereMentionNodeMatcher = new StringMatcher<MarkdownNode>(
"@here",
p => new MentionNode("here", MentionType.Meta));
// Capture <@123456> or <@!123456>
private static readonly IMatcher<Node> UserMentionNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> UserMentionNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("<@!?(\\d+)>", DefaultRegexOptions),
m => new MentionNode(m.Groups[1].Value, MentionType.User));
// Capture <#123456>
private static readonly IMatcher<Node> ChannelMentionNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> ChannelMentionNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("<#(\\d+)>", DefaultRegexOptions),
m => new MentionNode(m.Groups[1].Value, MentionType.Channel));
// Capture <@&123456>
private static readonly IMatcher<Node> RoleMentionNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> RoleMentionNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("<@&(\\d+)>", DefaultRegexOptions),
m => new MentionNode(m.Groups[1].Value, MentionType.Role));
@ -127,29 +128,29 @@ namespace DiscordChatExporter.Core.Markdown
// ... or surrogate pair
// ... or digit followed by enclosing mark
// (this does not match all emojis in Discord but it's reasonably accurate enough)
private static readonly IMatcher<Node> StandardEmojiNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> StandardEmojiNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("((?:[\\uD83C][\\uDDE6-\\uDDFF]){2}|[\\u2600-\\u26FF]|\\p{Cs}{2}|\\d\\p{Me})", DefaultRegexOptions),
m => new EmojiNode(m.Groups[1].Value));
// Capture <:lul:123456> or <a:lul:123456>
private static readonly IMatcher<Node> CustomEmojiNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> CustomEmojiNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("<(a)?:(.+?):(\\d+?)>", DefaultRegexOptions),
m => new EmojiNode(m.Groups[3].Value, m.Groups[2].Value, !string.IsNullOrWhiteSpace(m.Groups[1].Value)));
/* Links */
// Capture [title](link)
private static readonly IMatcher<Node> TitledLinkNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> TitledLinkNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("\\[(.+?)\\]\\((.+?)\\)", DefaultRegexOptions),
m => new LinkNode(m.Groups[2].Value, m.Groups[1].Value));
// Capture any non-whitespace character after http:// or https:// until the last punctuation character or whitespace
private static readonly IMatcher<Node> AutoLinkNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> AutoLinkNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("(https?://\\S*[^\\.,:;\"\'\\s])", DefaultRegexOptions),
m => new LinkNode(m.Groups[1].Value));
// Same as auto link but also surrounded by angular brackets
private static readonly IMatcher<Node> HiddenLinkNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> HiddenLinkNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("<(https?://\\S*[^\\.,:;\"\'\\s])>", DefaultRegexOptions),
m => new LinkNode(m.Groups[1].Value));
@ -157,31 +158,31 @@ namespace DiscordChatExporter.Core.Markdown
// Capture the shrug emoticon
// This escapes it from matching for formatting
private static readonly IMatcher<Node> ShrugTextNodeMatcher = new StringMatcher<Node>(
private static readonly IMatcher<MarkdownNode> ShrugTextNodeMatcher = new StringMatcher<MarkdownNode>(
@"¯\_(ツ)_/¯",
p => new TextNode(p.ToString()));
// Capture some specific emojis that don't get rendered
// This escapes it from matching for emoji
private static readonly IMatcher<Node> IgnoredEmojiTextNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> IgnoredEmojiTextNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("(\\u26A7|\\u2640|\\u2642|\\u2695|\\u267E|\\u00A9|\\u00AE|\\u2122)", DefaultRegexOptions),
m => new TextNode(m.Groups[1].Value));
// Capture any "symbol/other" character or surrogate pair preceded by a backslash
// This escapes it from matching for emoji
private static readonly IMatcher<Node> EscapedSymbolTextNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> EscapedSymbolTextNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("\\\\(\\p{So}|\\p{Cs}{2})", DefaultRegexOptions),
m => new TextNode(m.Groups[1].Value));
// Capture any non-whitespace, non latin alphanumeric character preceded by a backslash
// This escapes it from matching for formatting or other tokens
private static readonly IMatcher<Node> EscapedCharacterTextNodeMatcher = new RegexMatcher<Node>(
private static readonly IMatcher<MarkdownNode> EscapedCharacterTextNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("\\\\([^a-zA-Z0-9\\s])", DefaultRegexOptions),
m => new TextNode(m.Groups[1].Value));
// Combine all matchers into one
// Matchers that have similar patterns are ordered from most specific to least specific
private static readonly IMatcher<Node> AggregateNodeMatcher = new AggregateMatcher<Node>(
private static readonly IMatcher<MarkdownNode> AggregateNodeMatcher = new AggregateMatcher<MarkdownNode>(
// Escaped text
ShrugTextNodeMatcher,
IgnoredEmojiTextNodeMatcher,
@ -223,7 +224,7 @@ namespace DiscordChatExporter.Core.Markdown
);
// Minimal set of matchers for non-multimedia formats (e.g. plain text)
private static readonly IMatcher<Node> MinimalAggregateNodeMatcher = new AggregateMatcher<Node>(
private static readonly IMatcher<MarkdownNode> MinimalAggregateNodeMatcher = new AggregateMatcher<MarkdownNode>(
// Mentions
EveryoneMentionNodeMatcher,
HereMentionNodeMatcher,
@ -235,15 +236,21 @@ namespace DiscordChatExporter.Core.Markdown
CustomEmojiNodeMatcher
);
private static IReadOnlyList<Node> Parse(StringPart stringPart, IMatcher<Node> matcher) =>
matcher.MatchAll(stringPart, p => new TextNode(p.ToString())).Select(r => r.Value).ToArray();
private static IReadOnlyList<MarkdownNode> Parse(StringPart stringPart, IMatcher<MarkdownNode> matcher) =>
matcher
.MatchAll(stringPart, p => new TextNode(p.ToString()))
.Select(r => r.Value)
.ToArray();
}
private static IReadOnlyList<Node> Parse(StringPart stringPart) => Parse(stringPart, AggregateNodeMatcher);
internal static partial class MarkdownParser
{
private static IReadOnlyList<MarkdownNode> Parse(StringPart stringPart) => Parse(stringPart, AggregateNodeMatcher);
private static IReadOnlyList<Node> ParseMinimal(StringPart stringPart) => Parse(stringPart, MinimalAggregateNodeMatcher);
private static IReadOnlyList<MarkdownNode> ParseMinimal(StringPart stringPart) => Parse(stringPart, MinimalAggregateNodeMatcher);
public static IReadOnlyList<Node> Parse(string input) => Parse(new StringPart(input));
public static IReadOnlyList<MarkdownNode> Parse(string input) => Parse(new StringPart(input));
public static IReadOnlyList<Node> ParseMinimal(string input) => ParseMinimal(new StringPart(input));
public static IReadOnlyList<MarkdownNode> ParseMinimal(string input) => ParseMinimal(new StringPart(input));
}
}

45
DiscordChatExporter.Domain/Markdown/MarkdownVisitor.cs Normal file
View file

@ -0,0 +1,45 @@
using System;
using System.Collections.Generic;
using DiscordChatExporter.Domain.Markdown.Ast;
namespace DiscordChatExporter.Domain.Markdown
{
internal abstract class MarkdownVisitor
{
public virtual MarkdownNode VisitText(TextNode text) => text;
public virtual MarkdownNode VisitFormatted(FormattedNode formatted)
{
Visit(formatted.Children);
return formatted;
}
public virtual MarkdownNode VisitInlineCodeBlock(InlineCodeBlockNode inlineCodeBlock) => inlineCodeBlock;
public virtual MarkdownNode VisitMultiLineCodeBlock(MultiLineCodeBlockNode multiLineCodeBlock) => multiLineCodeBlock;
public virtual MarkdownNode VisitLink(LinkNode link) => link;
public virtual MarkdownNode VisitEmoji(EmojiNode emoji) => emoji;
public virtual MarkdownNode VisitMention(MentionNode mention) => mention;
public MarkdownNode Visit(MarkdownNode node) => node switch
{
TextNode text => VisitText(text),
FormattedNode formatted => VisitFormatted(formatted),
InlineCodeBlockNode inlineCodeBlock => VisitInlineCodeBlock(inlineCodeBlock),
MultiLineCodeBlockNode multiLineCodeBlock => VisitMultiLineCodeBlock(multiLineCodeBlock),
LinkNode link => VisitLink(link),
EmojiNode emoji => VisitEmoji(emoji),
MentionNode mention => VisitMention(mention),
_ => throw new ArgumentOutOfRangeException(nameof(node))
};
public void Visit(IEnumerable<MarkdownNode> nodes)
{
foreach (var node in nodes)
Visit(node);
}
}
}

8
DiscordChatExporter.Core.Markdown/Internal/AggregateMatcher.cs → DiscordChatExporter.Domain/Markdown/Matching/AggregateMatcher.cs
View file

@ -1,6 +1,6 @@
using System.Collections.Generic;
namespace DiscordChatExporter.Core.Markdown.Internal
namespace DiscordChatExporter.Domain.Markdown.Matching
{
internal class AggregateMatcher<T> : IMatcher<T>
{
@ -12,11 +12,11 @@ namespace DiscordChatExporter.Core.Markdown.Internal
}
public AggregateMatcher(params IMatcher<T>[] matchers)
: this((IReadOnlyList<IMatcher<T>>)matchers)
: this((IReadOnlyList<IMatcher<T>>) matchers)
{
}
public ParsedMatch<T>? Match(StringPart stringPart)
public ParsedMatch<T>? TryMatch(StringPart stringPart)
{
ParsedMatch<T>? earliestMatch = null;
@ -24,7 +24,7 @@ namespace DiscordChatExporter.Core.Markdown.Internal
foreach (var matcher in _matchers)
{
// Try to match
var match = matcher.Match(stringPart);
var match = matcher.TryMatch(stringPart);
// If there's no match - continue
if (match == null)

Some files were not shown because too many files have changed in this diff Show more