From 4bfb2ec7fdcb34c5c0910c55711a76d78383e44a Mon Sep 17 00:00:00 2001 From: Alexey Golub Date: Wed, 10 Apr 2019 23:45:21 +0300 Subject: [PATCH] Improve performance (#162) --- DiscordChatExporter.Cli/Container.cs | 2 +- .../DiscordChatExporter.Cli.csproj | 35 +-- .../Verbs/ExportChannelVerb.cs | 6 +- .../Verbs/ExportDirectMessagesVerb.cs | 6 +- .../Verbs/ExportGuildVerb.cs | 6 +- .../DiscordChatExporter.Core.Markdown.csproj | 5 +- .../Internal/AggregateMatcher.cs | 46 ++++ .../Internal/Extensions.cs | 50 ++++ .../Internal/Grammar.cs | 178 -------------- .../Internal/IMatcher.cs | 7 + .../Internal/ParsedMatch.cs | 18 ++ .../Internal/RegexMatcher.cs | 23 ++ .../Internal/StringMatcher.cs | 29 +++ .../MarkdownParser.cs | 179 +++++++++++++- DiscordChatExporter.Core.Markdown/Node.cs | 12 - .../{ => Nodes}/EmojiNode.cs | 14 +- .../{ => Nodes}/FormattedNode.cs | 6 +- .../{ => Nodes}/InlineCodeBlockNode.cs | 6 +- .../{ => Nodes}/LinkNode.cs | 8 +- .../{ => Nodes}/MentionNode.cs | 6 +- .../{ => Nodes}/MentionType.cs | 2 +- .../{ => Nodes}/MultilineCodeBlockNode.cs | 6 +- .../Nodes/Node.cs | 12 + .../{ => Nodes}/TextFormatting.cs | 2 +- .../{ => Nodes}/TextNode.cs | 6 +- .../Attachment.cs | 23 +- .../AuthToken.cs | 0 .../AuthTokenType.cs | 0 .../Channel.cs | 0 .../ChannelType.cs | 0 .../ChatLog.cs | 0 .../DiscordChatExporter.Core.Models.csproj | 11 + .../Embed.cs | 0 .../EmbedAuthor.cs | 0 .../EmbedField.cs | 0 .../EmbedFooter.cs | 0 .../EmbedImage.cs | 0 .../Emoji.cs | 44 ++-- .../ExportFormat.cs | 0 .../Extensions.cs | 0 .../FileSize.cs | 0 .../Guild.cs | 17 +- .../Mentionables.cs | 0 .../Message.cs | 0 .../MessageType.cs | 0 .../Reaction.cs | 0 .../Role.cs | 3 +- DiscordChatExporter.Core.Models/User.cs | 58 +++++ .../CsvChatLogRenderer.cs | 112 +++++++++ .../DiscordChatExporter.Core.Rendering.csproj | 25 ++ .../HtmlChatLogRenderer.MessageGroup.cs | 8 +- .../HtmlChatLogRenderer.TemplateLoader.cs | 27 +++ .../HtmlChatLogRenderer.cs | 197 ++++++++++++++++ .../IChatLogRenderer.cs | 10 + .../PlainTextChatLogRenderer.cs | 128 ++++++++++ .../Resources/HtmlDark.css | 0 .../Resources/HtmlDark.html | 3 + .../Resources/HtmlLight.css | 0 .../Resources/HtmlLight.html | 3 + .../Resources/HtmlShared.css | 0 .../Resources/HtmlShared.html | 4 +- .../DataService.Parsers.cs | 6 +- .../DataService.cs | 10 +- .../DiscordChatExporter.Core.Services.csproj | 20 ++ .../HttpErrorStatusCodeException.cs | 2 +- .../ExportService.cs | 89 +++++++ .../Helpers/ExportHelper.cs | 11 +- .../Internal/Extensions.cs | 18 ++ .../SettingsService.cs | 0 .../UpdateService.cs | 0 .../DiscordChatExporter.Core.csproj | 31 --- .../Internal/Extensions.cs | 52 ---- DiscordChatExporter.Core/Models/User.cs | 61 ----- .../ExportTemplates/Csv/Template.csv | 10 - .../ExportTemplates/HtmlDark/Template.html | 3 - .../ExportTemplates/HtmlLight/Template.html | 3 - .../ExportTemplates/PlainText/Template.txt | 21 -- .../Services/ExportService.TemplateLoader.cs | 43 ---- .../Services/ExportService.TemplateModel.cs | 222 ------------------ .../Services/ExportService.cs | 107 --------- DiscordChatExporter.Gui/Bootstrapper.cs | 2 +- .../DiscordChatExporter.Gui.csproj | 10 +- .../Dialogs/ExportSetupViewModel.cs | 5 +- .../ViewModels/RootViewModel.cs | 12 +- DiscordChatExporter.sln | 48 ++-- Dockerfile | 13 +- 86 files changed, 1242 insertions(+), 900 deletions(-) create mode 100644 DiscordChatExporter.Core.Markdown/Internal/AggregateMatcher.cs create mode 100644 DiscordChatExporter.Core.Markdown/Internal/Extensions.cs delete mode 100644 DiscordChatExporter.Core.Markdown/Internal/Grammar.cs create mode 100644 DiscordChatExporter.Core.Markdown/Internal/IMatcher.cs create mode 100644 DiscordChatExporter.Core.Markdown/Internal/ParsedMatch.cs create mode 100644 DiscordChatExporter.Core.Markdown/Internal/RegexMatcher.cs create mode 100644 DiscordChatExporter.Core.Markdown/Internal/StringMatcher.cs delete mode 100644 DiscordChatExporter.Core.Markdown/Node.cs rename DiscordChatExporter.Core.Markdown/{ => Nodes}/EmojiNode.cs (53%) rename DiscordChatExporter.Core.Markdown/{ => Nodes}/FormattedNode.cs (77%) rename DiscordChatExporter.Core.Markdown/{ => Nodes}/InlineCodeBlockNode.cs (58%) rename DiscordChatExporter.Core.Markdown/{ => Nodes}/LinkNode.cs (58%) rename DiscordChatExporter.Core.Markdown/{ => Nodes}/MentionNode.cs (65%) rename DiscordChatExporter.Core.Markdown/{ => Nodes}/MentionType.cs (64%) rename DiscordChatExporter.Core.Markdown/{ => Nodes}/MultilineCodeBlockNode.cs (68%) create mode 100644 DiscordChatExporter.Core.Markdown/Nodes/Node.cs rename DiscordChatExporter.Core.Markdown/{ => Nodes}/TextFormatting.cs (71%) rename DiscordChatExporter.Core.Markdown/{ => Nodes}/TextNode.cs (65%) rename {DiscordChatExporter.Core/Models => DiscordChatExporter.Core.Models}/Attachment.cs (57%) rename {DiscordChatExporter.Core/Models => DiscordChatExporter.Core.Models}/AuthToken.cs (100%) rename {DiscordChatExporter.Core/Models => DiscordChatExporter.Core.Models}/AuthTokenType.cs (100%) rename {DiscordChatExporter.Core/Models => DiscordChatExporter.Core.Models}/Channel.cs (100%) rename {DiscordChatExporter.Core/Models => DiscordChatExporter.Core.Models}/ChannelType.cs (100%) rename {DiscordChatExporter.Core/Models => DiscordChatExporter.Core.Models}/ChatLog.cs (100%) create mode 100644 DiscordChatExporter.Core.Models/DiscordChatExporter.Core.Models.csproj rename {DiscordChatExporter.Core/Models => DiscordChatExporter.Core.Models}/Embed.cs (100%) rename {DiscordChatExporter.Core/Models => DiscordChatExporter.Core.Models}/EmbedAuthor.cs (100%) rename {DiscordChatExporter.Core/Models => DiscordChatExporter.Core.Models}/EmbedField.cs (100%) rename {DiscordChatExporter.Core/Models => DiscordChatExporter.Core.Models}/EmbedFooter.cs (100%) rename {DiscordChatExporter.Core/Models => DiscordChatExporter.Core.Models}/EmbedImage.cs (100%) rename {DiscordChatExporter.Core/Models => DiscordChatExporter.Core.Models}/Emoji.cs (55%) rename {DiscordChatExporter.Core/Models => DiscordChatExporter.Core.Models}/ExportFormat.cs (100%) rename {DiscordChatExporter.Core/Models => DiscordChatExporter.Core.Models}/Extensions.cs (100%) rename {DiscordChatExporter.Core/Models => DiscordChatExporter.Core.Models}/FileSize.cs (100%) rename {DiscordChatExporter.Core/Models => DiscordChatExporter.Core.Models}/Guild.cs (59%) rename {DiscordChatExporter.Core/Models => DiscordChatExporter.Core.Models}/Mentionables.cs (100%) rename {DiscordChatExporter.Core/Models => DiscordChatExporter.Core.Models}/Message.cs (100%) rename {DiscordChatExporter.Core/Models => DiscordChatExporter.Core.Models}/MessageType.cs (100%) rename {DiscordChatExporter.Core/Models => DiscordChatExporter.Core.Models}/Reaction.cs (100%) rename {DiscordChatExporter.Core/Models => DiscordChatExporter.Core.Models}/Role.cs (81%) create mode 100644 DiscordChatExporter.Core.Models/User.cs create mode 100644 DiscordChatExporter.Core.Rendering/CsvChatLogRenderer.cs create mode 100644 DiscordChatExporter.Core.Rendering/DiscordChatExporter.Core.Rendering.csproj rename DiscordChatExporter.Core/Services/ExportService.MessageGroup.cs => DiscordChatExporter.Core.Rendering/HtmlChatLogRenderer.MessageGroup.cs (76%) create mode 100644 DiscordChatExporter.Core.Rendering/HtmlChatLogRenderer.TemplateLoader.cs create mode 100644 DiscordChatExporter.Core.Rendering/HtmlChatLogRenderer.cs create mode 100644 DiscordChatExporter.Core.Rendering/IChatLogRenderer.cs create mode 100644 DiscordChatExporter.Core.Rendering/PlainTextChatLogRenderer.cs rename DiscordChatExporter.Core/Resources/ExportTemplates/HtmlDark/Theme.css => DiscordChatExporter.Core.Rendering/Resources/HtmlDark.css (100%) create mode 100644 DiscordChatExporter.Core.Rendering/Resources/HtmlDark.html rename DiscordChatExporter.Core/Resources/ExportTemplates/HtmlLight/Theme.css => DiscordChatExporter.Core.Rendering/Resources/HtmlLight.css (100%) create mode 100644 DiscordChatExporter.Core.Rendering/Resources/HtmlLight.html rename DiscordChatExporter.Core/Resources/ExportTemplates/HtmlShared/Main.css => DiscordChatExporter.Core.Rendering/Resources/HtmlShared.css (100%) rename DiscordChatExporter.Core/Resources/ExportTemplates/HtmlShared/Main.html => DiscordChatExporter.Core.Rendering/Resources/HtmlShared.html (99%) rename {DiscordChatExporter.Core/Services => DiscordChatExporter.Core.Services}/DataService.Parsers.cs (98%) rename {DiscordChatExporter.Core/Services => DiscordChatExporter.Core.Services}/DataService.cs (97%) create mode 100644 DiscordChatExporter.Core.Services/DiscordChatExporter.Core.Services.csproj rename {DiscordChatExporter.Core => DiscordChatExporter.Core.Services}/Exceptions/HttpErrorStatusCodeException.cs (89%) create mode 100644 DiscordChatExporter.Core.Services/ExportService.cs rename {DiscordChatExporter.Core => DiscordChatExporter.Core.Services}/Helpers/ExportHelper.cs (83%) create mode 100644 DiscordChatExporter.Core.Services/Internal/Extensions.cs rename {DiscordChatExporter.Core/Services => DiscordChatExporter.Core.Services}/SettingsService.cs (100%) rename {DiscordChatExporter.Core/Services => DiscordChatExporter.Core.Services}/UpdateService.cs (100%) delete mode 100644 DiscordChatExporter.Core/DiscordChatExporter.Core.csproj delete mode 100644 DiscordChatExporter.Core/Internal/Extensions.cs delete mode 100644 DiscordChatExporter.Core/Models/User.cs delete mode 100644 DiscordChatExporter.Core/Resources/ExportTemplates/Csv/Template.csv delete mode 100644 DiscordChatExporter.Core/Resources/ExportTemplates/HtmlDark/Template.html delete mode 100644 DiscordChatExporter.Core/Resources/ExportTemplates/HtmlLight/Template.html delete mode 100644 DiscordChatExporter.Core/Resources/ExportTemplates/PlainText/Template.txt delete mode 100644 DiscordChatExporter.Core/Services/ExportService.TemplateLoader.cs delete mode 100644 DiscordChatExporter.Core/Services/ExportService.TemplateModel.cs delete mode 100644 DiscordChatExporter.Core/Services/ExportService.cs diff --git a/DiscordChatExporter.Cli/Container.cs b/DiscordChatExporter.Cli/Container.cs index 091c60e4..d7e44f19 100644 --- a/DiscordChatExporter.Cli/Container.cs +++ b/DiscordChatExporter.Cli/Container.cs @@ -11,7 +11,7 @@ namespace DiscordChatExporter.Cli { var builder = new StyletIoCBuilder(); - // Autobind services in the .Core assembly + // Autobind the .Services assembly builder.Autobind(typeof(DataService).Assembly); // Bind settings as singleton diff --git a/DiscordChatExporter.Cli/DiscordChatExporter.Cli.csproj b/DiscordChatExporter.Cli/DiscordChatExporter.Cli.csproj index d18c970c..f20c8d22 100644 --- a/DiscordChatExporter.Cli/DiscordChatExporter.Cli.csproj +++ b/DiscordChatExporter.Cli/DiscordChatExporter.Cli.csproj @@ -1,23 +1,24 @@  - - Exe - net46;netcoreapp2.1 - 2.11 - Tyrrrz - Copyright (c) Alexey Golub - ..\favicon.ico - true - + + Exe + net46;netcoreapp2.1 + 2.11 + Tyrrrz + Copyright (c) Alexey Golub + ..\favicon.ico + true + - - - - - + + + + + - - - + + + + \ No newline at end of file diff --git a/DiscordChatExporter.Cli/Verbs/ExportChannelVerb.cs b/DiscordChatExporter.Cli/Verbs/ExportChannelVerb.cs index c1a3cc0e..f407e680 100644 --- a/DiscordChatExporter.Cli/Verbs/ExportChannelVerb.cs +++ b/DiscordChatExporter.Cli/Verbs/ExportChannelVerb.cs @@ -3,8 +3,8 @@ using System.IO; using System.Threading.Tasks; using DiscordChatExporter.Cli.Internal; using DiscordChatExporter.Cli.Verbs.Options; -using DiscordChatExporter.Core.Helpers; using DiscordChatExporter.Core.Services; +using DiscordChatExporter.Core.Services.Helpers; using Tyrrrz.Extensions; namespace DiscordChatExporter.Cli.Verbs @@ -24,7 +24,7 @@ namespace DiscordChatExporter.Cli.Verbs var exportService = Container.Instance.Get(); // Configure settings - if (Options.DateFormat.IsNotBlank()) + if (!Options.DateFormat.EmptyIfNull().IsWhiteSpace()) settingsService.DateFormat = Options.DateFormat; // Track progress @@ -37,7 +37,7 @@ namespace DiscordChatExporter.Cli.Verbs // Generate file path if not set or is a directory var filePath = Options.OutputPath; - if (filePath.IsBlank() || ExportHelper.IsDirectoryPath(filePath)) + if (filePath.EmptyIfNull().IsWhiteSpace() || ExportHelper.IsDirectoryPath(filePath)) { // Generate default file name var fileName = ExportHelper.GetDefaultExportFileName(Options.ExportFormat, chatLog.Guild, diff --git a/DiscordChatExporter.Cli/Verbs/ExportDirectMessagesVerb.cs b/DiscordChatExporter.Cli/Verbs/ExportDirectMessagesVerb.cs index d34d1989..08fa3014 100644 --- a/DiscordChatExporter.Cli/Verbs/ExportDirectMessagesVerb.cs +++ b/DiscordChatExporter.Cli/Verbs/ExportDirectMessagesVerb.cs @@ -5,9 +5,9 @@ using System.Net; using System.Threading.Tasks; using DiscordChatExporter.Cli.Internal; using DiscordChatExporter.Cli.Verbs.Options; -using DiscordChatExporter.Core.Exceptions; -using DiscordChatExporter.Core.Helpers; using DiscordChatExporter.Core.Services; +using DiscordChatExporter.Core.Services.Exceptions; +using DiscordChatExporter.Core.Services.Helpers; using Tyrrrz.Extensions; namespace DiscordChatExporter.Cli.Verbs @@ -27,7 +27,7 @@ namespace DiscordChatExporter.Cli.Verbs var exportService = Container.Instance.Get(); // Configure settings - if (Options.DateFormat.IsNotBlank()) + if (!Options.DateFormat.EmptyIfNull().IsWhiteSpace()) settingsService.DateFormat = Options.DateFormat; // Get channels diff --git a/DiscordChatExporter.Cli/Verbs/ExportGuildVerb.cs b/DiscordChatExporter.Cli/Verbs/ExportGuildVerb.cs index 6ef9a7f8..454cdc12 100644 --- a/DiscordChatExporter.Cli/Verbs/ExportGuildVerb.cs +++ b/DiscordChatExporter.Cli/Verbs/ExportGuildVerb.cs @@ -5,10 +5,10 @@ using System.Net; using System.Threading.Tasks; using DiscordChatExporter.Cli.Internal; using DiscordChatExporter.Cli.Verbs.Options; -using DiscordChatExporter.Core.Exceptions; -using DiscordChatExporter.Core.Helpers; using DiscordChatExporter.Core.Models; using DiscordChatExporter.Core.Services; +using DiscordChatExporter.Core.Services.Exceptions; +using DiscordChatExporter.Core.Services.Helpers; using Tyrrrz.Extensions; namespace DiscordChatExporter.Cli.Verbs @@ -28,7 +28,7 @@ namespace DiscordChatExporter.Cli.Verbs var exportService = Container.Instance.Get(); // Configure settings - if (Options.DateFormat.IsNotBlank()) + if (!Options.DateFormat.EmptyIfNull().IsWhiteSpace()) settingsService.DateFormat = Options.DateFormat; // Get channels diff --git a/DiscordChatExporter.Core.Markdown/DiscordChatExporter.Core.Markdown.csproj b/DiscordChatExporter.Core.Markdown/DiscordChatExporter.Core.Markdown.csproj index 82548696..03f624f8 100644 --- a/DiscordChatExporter.Core.Markdown/DiscordChatExporter.Core.Markdown.csproj +++ b/DiscordChatExporter.Core.Markdown/DiscordChatExporter.Core.Markdown.csproj @@ -5,8 +5,7 @@ - - + - + \ No newline at end of file diff --git a/DiscordChatExporter.Core.Markdown/Internal/AggregateMatcher.cs b/DiscordChatExporter.Core.Markdown/Internal/AggregateMatcher.cs new file mode 100644 index 00000000..449bcb55 --- /dev/null +++ b/DiscordChatExporter.Core.Markdown/Internal/AggregateMatcher.cs @@ -0,0 +1,46 @@ +using System.Collections.Generic; + +namespace DiscordChatExporter.Core.Markdown.Internal +{ + internal class AggregateMatcher : IMatcher + { + private readonly IReadOnlyList> _matchers; + + public AggregateMatcher(IReadOnlyList> matchers) + { + _matchers = matchers; + } + + public AggregateMatcher(params IMatcher[] matchers) + : this((IReadOnlyList>)matchers) + { + } + + public ParsedMatch Match(string input, int startIndex, int length) + { + ParsedMatch earliestMatch = null; + + // Try to match the input with each matcher and get the match with the lowest start index + foreach (var matcher in _matchers) + { + // Try to match + var match = matcher.Match(input, startIndex, length); + + // If there's no match - continue + if (match == null) + continue; + + // If this match is earlier than previous earliest - replace + if (earliestMatch == null || match.StartIndex < earliestMatch.StartIndex) + earliestMatch = match; + + // If the earliest match starts at the very beginning - break, + // because it's impossible to find a match earlier than that + if (earliestMatch.StartIndex == startIndex) + break; + } + + return earliestMatch; + } + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Core.Markdown/Internal/Extensions.cs b/DiscordChatExporter.Core.Markdown/Internal/Extensions.cs new file mode 100644 index 00000000..dfff64c9 --- /dev/null +++ b/DiscordChatExporter.Core.Markdown/Internal/Extensions.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; + +namespace DiscordChatExporter.Core.Markdown.Internal +{ + internal static class Extensions + { + public static IEnumerable> MatchAll(this IMatcher matcher, string input, + int startIndex, int length, Func fallbackTransform) + { + // Get end index for simplicity + var endIndex = startIndex + length; + + // Loop through segments divided by individual matches + var currentIndex = startIndex; + while (currentIndex < endIndex) + { + // Find a match within this segment + var match = matcher.Match(input, currentIndex, endIndex - currentIndex); + + // If there's no match - break + if (match == null) + break; + + // If this match doesn't start immediately at current index - transform and yield fallback first + if (match.StartIndex > currentIndex) + { + var fallback = input.Substring(currentIndex, match.StartIndex - currentIndex); + yield return new ParsedMatch(currentIndex, fallback.Length, fallbackTransform(fallback)); + } + + // Yield match + yield return match; + + // Shift current index to the end of the match + currentIndex = match.StartIndex + match.Length; + } + + // If EOL wasn't reached - transform and yield remaining part as fallback + if (currentIndex < endIndex) + { + var fallback = input.Substring(currentIndex); + yield return new ParsedMatch(currentIndex, fallback.Length, fallbackTransform(fallback)); + } + } + + public static IEnumerable> MatchAll(this IMatcher matcher, string input, + Func fallbackTransform) => matcher.MatchAll(input, 0, input.Length, fallbackTransform); + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Core.Markdown/Internal/Grammar.cs b/DiscordChatExporter.Core.Markdown/Internal/Grammar.cs deleted file mode 100644 index da626550..00000000 --- a/DiscordChatExporter.Core.Markdown/Internal/Grammar.cs +++ /dev/null @@ -1,178 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using System.Text.RegularExpressions; -using Sprache; -using Tyrrrz.Extensions; - -namespace DiscordChatExporter.Core.Markdown.Internal -{ - // The following parsing logic is meant to replicate Discord's markdown grammar as close as possible - internal static class Grammar - { - /* Formatting */ - - // Capture until the earliest double asterisk not followed by an asterisk - private static readonly Parser BoldFormattedNode = - Parse.RegexMatch(new Regex("\\*\\*(.+?)\\*\\*(?!\\*)", RegexOptions.Singleline)) - .Select(m => new FormattedNode(m.Value, "**", TextFormatting.Bold, BuildTree(m.Groups[1].Value))); - - // Capture until the earliest single asterisk not preceded or followed by an asterisk - // Can't have whitespace right after opening or right before closing asterisk - private static readonly Parser ItalicFormattedNode = - Parse.RegexMatch(new Regex("\\*(?!\\s)(.+?)(? new FormattedNode(m.Value, "*", TextFormatting.Italic, BuildTree(m.Groups[1].Value))); - - // Can't have underscores inside - // Can't have word characters right after closing underscore - private static readonly Parser ItalicAltFormattedNode = - Parse.RegexMatch(new Regex("_([^_]+?)_(?!\\w)", RegexOptions.Singleline)) - .Select(m => new FormattedNode(m.Value, "_", TextFormatting.Italic, BuildTree(m.Groups[1].Value))); - - // Treated as a separate entity for simplicity - // Capture until the earliest triple asterisk not preceded or followed by an asterisk - private static readonly Parser ItalicBoldFormattedNode = - Parse.RegexMatch(new Regex("\\*(\\*\\*(?:.+?)\\*\\*)\\*(?!\\*)", RegexOptions.Singleline)) - .Select(m => new FormattedNode(m.Value, "*", TextFormatting.Italic, BuildTree(m.Groups[1].Value))); - - // Capture until the earliest double underscore not followed by an underscore - private static readonly Parser UnderlineFormattedNode = - Parse.RegexMatch(new Regex("__(.+?)__(?!_)", RegexOptions.Singleline)) - .Select(m => new FormattedNode(m.Value, "__", TextFormatting.Underline, BuildTree(m.Groups[1].Value))); - - // Treated as a separate entity for simplicity - // Capture until the earliest triple underscore not preceded or followed by an underscore - private static readonly Parser ItalicUnderlineFormattedNode = - Parse.RegexMatch(new Regex("_(__(?:.+?)__)_(?!_)", RegexOptions.Singleline)) - .Select(m => new FormattedNode(m.Value, "_", TextFormatting.Italic, BuildTree(m.Groups[1].Value))); - - // Strikethrough is safe - private static readonly Parser StrikethroughFormattedNode = - Parse.RegexMatch(new Regex("~~(.+?)~~", RegexOptions.Singleline)) - .Select(m => new FormattedNode(m.Value, "~~", TextFormatting.Strikethrough, BuildTree(m.Groups[1].Value))); - - // Spoiler is safe - private static readonly Parser SpoilerFormattedNode = - Parse.RegexMatch(new Regex("\\|\\|(.+?)\\|\\|", RegexOptions.Singleline)) - .Select(m => new FormattedNode(m.Value, "||", TextFormatting.Spoiler, BuildTree(m.Groups[1].Value))); - - // Combinator, order matters - private static readonly Parser AnyFormattedNode = - ItalicBoldFormattedNode.Or(ItalicUnderlineFormattedNode) - .Or(BoldFormattedNode).Or(ItalicFormattedNode) - .Or(UnderlineFormattedNode).Or(ItalicAltFormattedNode) - .Or(StrikethroughFormattedNode).Or(SpoilerFormattedNode); - - /* Code blocks */ - - // Can't have backticks inside and surrounding whitespace is trimmed - private static readonly Parser InlineCodeBlockNode = - Parse.RegexMatch(new Regex("`\\s*([^`]+?)\\s*`", RegexOptions.Singleline)) - .Select(m => new InlineCodeBlockNode(m.Value, m.Groups[1].Value)); - - // The first word is a language identifier if it's the only word followed by a newline, the rest is code - private static readonly Parser MultilineCodeBlockNode = - Parse.RegexMatch(new Regex("```(?:(\\w*?)?(?:\\s*?\\n))?(.+?)```", RegexOptions.Singleline)) - .Select(m => new MultilineCodeBlockNode(m.Value, m.Groups[1].Value, m.Groups[2].Value)); - - // Combinator, order matters - private static readonly Parser AnyCodeBlockNode = MultilineCodeBlockNode.Or(InlineCodeBlockNode); - - /* Mentions */ - - // @everyone or @here - private static readonly Parser MetaMentionNode = Parse.RegexMatch("@(everyone|here)") - .Select(m => new MentionNode(m.Value, m.Groups[1].Value, MentionType.Meta)); - - // <@123456> or <@!123456> - private static readonly Parser UserMentionNode = Parse.RegexMatch("<@!?(\\d+)>") - .Select(m => new MentionNode(m.Value, m.Groups[1].Value, MentionType.User)); - - // <#123456> - private static readonly Parser ChannelMentionNode = Parse.RegexMatch("<#(\\d+)>") - .Select(m => new MentionNode(m.Value, m.Groups[1].Value, MentionType.Channel)); - - // <@&123456> - private static readonly Parser RoleMentionNode = Parse.RegexMatch("<@&(\\d+)>") - .Select(m => new MentionNode(m.Value, m.Groups[1].Value, MentionType.Role)); - - // Combinator, order matters - private static readonly Parser AnyMentionNode = - MetaMentionNode.Or(UserMentionNode).Or(ChannelMentionNode).Or(RoleMentionNode); - - /* Emojis */ - - // Matches all standard unicode emojis - private static readonly Parser StandardEmojiNode = Parse.RegexMatch( - "([\\u2700-\\u27bf]|" + - "(?:\\ud83c[\\udde6-\\uddff]){2}|" + - "[\\ud800-\\udbff][\\udc00-\\udfff]|" + - "[\\u0023-\\u0039]\\u20e3|" + - "\\u3299|\\u3297|\\u303d|\\u3030|\\u24c2|\\ud83c[\\udd70-\\udd71]|\\ud83c[\\udd7e-\\udd7f]|\\ud83c\\udd8e|\\ud83c[\\udd91-\\udd9a]|\\ud83c[\\udde6-\\uddff]|" + - "\\ud83c[\\ude01-\\ude02]|\\ud83c\\ude1a|\\ud83c\\ude2f|\\ud83c[\\ude32-\\ude3a]|\\ud83c[\\ude50-\\ude51]|\\u203c|\\u2049|[\\u25aa-\\u25ab]|" + - "\\u25b6|\\u25c0|[\\u25fb-\\u25fe]|\\u00a9|\\u00ae|\\u2122|\\u2139|\\ud83c\\udc04|[\\u2600-\\u26FF]|\\u2b05|\\u2b06|\\u2b07|\\u2b1b|\\u2b1c|\\u2b50|" + - "\\u2b55|\\u231a|\\u231b|\\u2328|\\u23cf|[\\u23e9-\\u23f3]|[\\u23f8-\\u23fa]|\\ud83c\\udccf|\\u2934|\\u2935|[\\u2190-\\u21ff])") - .Select(m => new EmojiNode(m.Value, m.Groups[1].Value)); - - // <:lul:123456> or - private static readonly Parser CustomEmojiNode = Parse.RegexMatch("<(a)?:(.+?):(\\d+)>") - .Select(m => new EmojiNode(m.Value, m.Groups[3].Value, m.Groups[2].Value, m.Groups[1].Value.IsNotBlank())); - - // Combinator, order matters - private static readonly Parser AnyEmojiNode = StandardEmojiNode.Or(CustomEmojiNode); - - /* Links */ - - // [title](link) - private static readonly Parser TitledLinkNode = Parse.RegexMatch("\\[(.+?)\\]\\((.+?)\\)") - .Select(m => new LinkNode(m.Value, m.Groups[2].Value, m.Groups[1].Value)); - - // Starts with http:// or https://, stops at the last non-whitespace character followed by whitespace or punctuation character - private static readonly Parser AutoLinkNode = Parse.RegexMatch("(https?://\\S*[^\\.,:;\"\'\\s])") - .Select(m => new LinkNode(m.Value, m.Groups[1].Value)); - - // Autolink surrounded by angular brackets - private static readonly Parser HiddenLinkNode = Parse.RegexMatch("<(https?://\\S*[^\\.,:;\"\'\\s])>") - .Select(m => new LinkNode(m.Value, m.Groups[1].Value)); - - // Combinator, order matters - private static readonly Parser AnyLinkNode = TitledLinkNode.Or(HiddenLinkNode).Or(AutoLinkNode); - - /* Text */ - - // Shrug is an exception and needs to be exempt from formatting - private static readonly Parser ShrugTextNode = - Parse.String("¯\\_(ツ)_/¯").Text().Select(s => new TextNode(s)); - - // Backslash escapes any following unicode surrogate pair - private static readonly Parser EscapedSurrogateTextNode = - from slash in Parse.Char('\\') - from high in Parse.AnyChar.Where(char.IsHighSurrogate) - from low in Parse.AnyChar - let lexeme = $"{slash}{high}{low}" - let text = $"{high}{low}" - select new TextNode(lexeme, text); - - // Backslash escapes any following non-whitespace character except for digits and latin letters - private static readonly Parser EscapedTextNode = - Parse.RegexMatch("\\\\([^a-zA-Z0-9\\s])").Select(m => new TextNode(m.Value, m.Groups[1].Value)); - - // Combinator, order matters - private static readonly Parser AnyTextNode = ShrugTextNode.Or(EscapedSurrogateTextNode).Or(EscapedTextNode); - - /* Aggregator and fallback */ - - // Any node recognized by above patterns - private static readonly Parser AnyRecognizedNode = AnyFormattedNode.Or(AnyCodeBlockNode) - .Or(AnyMentionNode).Or(AnyEmojiNode).Or(AnyLinkNode).Or(AnyTextNode); - - // Any node not recognized by above patterns (treated as plain text) - private static readonly Parser FallbackNode = - Parse.AnyChar.Except(AnyRecognizedNode).AtLeastOnce().Text().Select(s => new TextNode(s)); - - // Any node - private static readonly Parser AnyNode = AnyRecognizedNode.Or(FallbackNode); - - // Entry point - public static IReadOnlyList BuildTree(string input) => AnyNode.Many().Parse(input).ToArray(); - } -} \ No newline at end of file diff --git a/DiscordChatExporter.Core.Markdown/Internal/IMatcher.cs b/DiscordChatExporter.Core.Markdown/Internal/IMatcher.cs new file mode 100644 index 00000000..fa0c7754 --- /dev/null +++ b/DiscordChatExporter.Core.Markdown/Internal/IMatcher.cs @@ -0,0 +1,7 @@ +namespace DiscordChatExporter.Core.Markdown.Internal +{ + internal interface IMatcher + { + ParsedMatch Match(string input, int startIndex, int length); + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Core.Markdown/Internal/ParsedMatch.cs b/DiscordChatExporter.Core.Markdown/Internal/ParsedMatch.cs new file mode 100644 index 00000000..dcb14a13 --- /dev/null +++ b/DiscordChatExporter.Core.Markdown/Internal/ParsedMatch.cs @@ -0,0 +1,18 @@ +namespace DiscordChatExporter.Core.Markdown.Internal +{ + internal partial class ParsedMatch + { + public int StartIndex { get; } + + public int Length { get; } + + public T Value { get; } + + public ParsedMatch(int startIndex, int length, T value) + { + StartIndex = startIndex; + Length = length; + Value = value; + } + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Core.Markdown/Internal/RegexMatcher.cs b/DiscordChatExporter.Core.Markdown/Internal/RegexMatcher.cs new file mode 100644 index 00000000..9977410c --- /dev/null +++ b/DiscordChatExporter.Core.Markdown/Internal/RegexMatcher.cs @@ -0,0 +1,23 @@ +using System; +using System.Text.RegularExpressions; + +namespace DiscordChatExporter.Core.Markdown.Internal +{ + internal class RegexMatcher : IMatcher + { + private readonly Regex _regex; + private readonly Func _transform; + + public RegexMatcher(Regex regex, Func transform) + { + _regex = regex; + _transform = transform; + } + + public ParsedMatch Match(string input, int startIndex, int length) + { + var match = _regex.Match(input, startIndex, length); + return match.Success ? new ParsedMatch(match.Index, match.Length, _transform(match)) : null; + } + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Core.Markdown/Internal/StringMatcher.cs b/DiscordChatExporter.Core.Markdown/Internal/StringMatcher.cs new file mode 100644 index 00000000..e757d6b4 --- /dev/null +++ b/DiscordChatExporter.Core.Markdown/Internal/StringMatcher.cs @@ -0,0 +1,29 @@ +using System; + +namespace DiscordChatExporter.Core.Markdown.Internal +{ + internal class StringMatcher : IMatcher + { + private readonly string _needle; + private readonly StringComparison _comparison; + private readonly Func _transform; + + public StringMatcher(string needle, StringComparison comparison, Func transform) + { + _needle = needle; + _comparison = comparison; + _transform = transform; + } + + public StringMatcher(string needle, Func transform) + : this(needle, StringComparison.Ordinal, transform) + { + } + + public ParsedMatch Match(string input, int startIndex, int length) + { + var index = input.IndexOf(_needle, startIndex, length, _comparison); + return index >= 0 ? new ParsedMatch(index, _needle.Length, _transform(_needle)) : null; + } + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Core.Markdown/MarkdownParser.cs b/DiscordChatExporter.Core.Markdown/MarkdownParser.cs index 679a6ad4..e9e6a295 100644 --- a/DiscordChatExporter.Core.Markdown/MarkdownParser.cs +++ b/DiscordChatExporter.Core.Markdown/MarkdownParser.cs @@ -1,10 +1,187 @@ using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; using DiscordChatExporter.Core.Markdown.Internal; +using DiscordChatExporter.Core.Markdown.Nodes; +using Tyrrrz.Extensions; namespace DiscordChatExporter.Core.Markdown { + // The following parsing logic is meant to replicate Discord's markdown grammar as close as possible public static class MarkdownParser { - public static IReadOnlyList Parse(string input) => Grammar.BuildTree(input); + private const RegexOptions DefaultRegexOptions = RegexOptions.Compiled | RegexOptions.CultureInvariant; + + /* Formatting */ + + // Capture any character until the earliest double asterisk not followed by an asterisk + private static readonly IMatcher BoldFormattedNodeMatcher = new RegexMatcher( + new Regex("\\*\\*(.+?)\\*\\*(?!\\*)", DefaultRegexOptions | RegexOptions.Singleline), + m => new FormattedNode(m.Value, "**", TextFormatting.Bold, Parse(m.Groups[1].Value))); + + // Capture any character until the earliest single asterisk not preceded or followed by an asterisk + // Opening asterisk must not be followed by whitespace + // Closing asterisk must not be preceeded by whitespace + private static readonly IMatcher ItalicFormattedNodeMatcher = new RegexMatcher( + new Regex("\\*(?!\\s)(.+?)(? new FormattedNode(m.Value, "*", TextFormatting.Italic, Parse(m.Groups[1].Value))); + + // Capture any character until the earliest triple asterisk not followed by an asterisk + private static readonly IMatcher ItalicBoldFormattedNodeMatcher = new RegexMatcher( + new Regex("\\*(\\*\\*.+?\\*\\*)\\*(?!\\*)", DefaultRegexOptions | RegexOptions.Singleline), + m => new FormattedNode(m.Value, "*", TextFormatting.Italic, Parse(m.Groups[1].Value, BoldFormattedNodeMatcher))); + + // Capture any character except underscore until an underscore + // Closing underscore must not be followed by a word character + private static readonly IMatcher ItalicAltFormattedNodeMatcher = new RegexMatcher( + new Regex("_([^_]+)_(?!\\w)", DefaultRegexOptions | RegexOptions.Singleline), + m => new FormattedNode(m.Value, "_", TextFormatting.Italic, Parse(m.Groups[1].Value))); + + // Capture any character until the earliest double underscore not followed by an underscore + private static readonly IMatcher UnderlineFormattedNodeMatcher = new RegexMatcher( + new Regex("__(.+?)__(?!_)", DefaultRegexOptions | RegexOptions.Singleline), + m => new FormattedNode(m.Value, "__", TextFormatting.Underline, Parse(m.Groups[1].Value))); + + // Capture any character until the earliest triple underscore not followed by an underscore + private static readonly IMatcher ItalicUnderlineFormattedNodeMatcher = new RegexMatcher( + new Regex("_(__.+?__)_(?!_)", DefaultRegexOptions | RegexOptions.Singleline), + m => new FormattedNode(m.Value, "_", TextFormatting.Italic, Parse(m.Groups[1].Value, UnderlineFormattedNodeMatcher))); + + // Capture any character until the earliest double tilde + private static readonly IMatcher StrikethroughFormattedNodeMatcher = new RegexMatcher( + new Regex("~~(.+?)~~", DefaultRegexOptions | RegexOptions.Singleline), + m => new FormattedNode(m.Value, "~~", TextFormatting.Strikethrough, Parse(m.Groups[1].Value))); + + // Capture any character until the earliest double pipe + private static readonly IMatcher SpoilerFormattedNodeMatcher = new RegexMatcher( + new Regex("\\|\\|(.+?)\\|\\|", DefaultRegexOptions | RegexOptions.Singleline), + m => new FormattedNode(m.Value, "||", TextFormatting.Spoiler, Parse(m.Groups[1].Value))); + + /* Code blocks */ + + // Capture any character except backtick until a backtick + // Whitespace surrounding content inside backticks is trimmed + private static readonly IMatcher InlineCodeBlockNodeMatcher = new RegexMatcher( + new Regex("`([^`]+)`", DefaultRegexOptions | RegexOptions.Singleline), + m => new InlineCodeBlockNode(m.Value, m.Groups[1].Value.Trim())); + + // Capture language identifier and then any character until the earliest triple backtick + // Languge identifier is one word immediately after opening backticks, followed immediately by newline + // Whitespace surrounding content inside backticks is trimmed + private static readonly IMatcher MultilineCodeBlockNodeMatcher = new RegexMatcher( + new Regex("```(?:(\\w*)\\n)?(.+?)```", DefaultRegexOptions | RegexOptions.Singleline), + m => new MultilineCodeBlockNode(m.Value, m.Groups[1].Value, m.Groups[2].Value.Trim())); + + /* Mentions */ + + // Capture @everyone + private static readonly IMatcher EveryoneMentionNodeMatcher = new StringMatcher( + "@everyone", + s => new MentionNode(s, "everyone", MentionType.Meta)); + + // Capture @here + private static readonly IMatcher HereMentionNodeMatcher = new StringMatcher( + "@here", + s => new MentionNode(s, "here", MentionType.Meta)); + + // Capture <@123456> or <@!123456> + private static readonly IMatcher UserMentionNodeMatcher = new RegexMatcher( + new Regex("<@!?(\\d+)>", DefaultRegexOptions), + m => new MentionNode(m.Value, m.Groups[1].Value, MentionType.User)); + + // Capture <#123456> + private static readonly IMatcher ChannelMentionNodeMatcher = new RegexMatcher( + new Regex("<#(\\d+)>", DefaultRegexOptions), + m => new MentionNode(m.Value, m.Groups[1].Value, MentionType.Channel)); + + // Capture <@&123456> + private static readonly IMatcher RoleMentionNodeMatcher = new RegexMatcher( + new Regex("<@&(\\d+)>", DefaultRegexOptions), + m => new MentionNode(m.Value, m.Groups[1].Value, MentionType.Role)); + + /* Emojis */ + + // Capture any country flag emoji (two regional indicator surrogate pairs) + // ... or "symbol/other" character + // ... or surrogate pair + // ... or digit followed by enclosing mark + // (this does not match all emojis in Discord but it's reasonably accurate enough) + private static readonly IMatcher StandardEmojiNodeMatcher = new RegexMatcher( + new Regex("((?:[\\uD83C][\\uDDE6-\\uDDFF]){2}|\\p{So}|\\p{Cs}{2}|\\d\\p{Me})", DefaultRegexOptions), + m => new EmojiNode(m.Value, m.Groups[1].Value)); + + // Capture <:lul:123456> or + private static readonly IMatcher CustomEmojiNodeMatcher = new RegexMatcher( + new Regex("<(a)?:(.+?):(\\d+?)>", DefaultRegexOptions), + m => new EmojiNode(m.Value, m.Groups[3].Value, m.Groups[2].Value, !m.Groups[1].Value.IsEmpty())); + + /* Links */ + + // Capture [title](link) + private static readonly IMatcher TitledLinkNodeMatcher = new RegexMatcher( + new Regex("\\[(.+?)\\]\\((.+?)\\)", DefaultRegexOptions), + m => new LinkNode(m.Value, m.Groups[2].Value, m.Groups[1].Value)); + + // Capture any non-whitespace character after http:// or https:// until the last punctuation character or whitespace + private static readonly IMatcher AutoLinkNodeMatcher = new RegexMatcher( + new Regex("(https?://\\S*[^\\.,:;\"\'\\s])", DefaultRegexOptions), + m => new LinkNode(m.Value, m.Groups[1].Value)); + + // Same as auto link but also surrounded by angular brackets + private static readonly IMatcher HiddenLinkNodeMatcher = new RegexMatcher( + new Regex("<(https?://\\S*[^\\.,:;\"\'\\s])>", DefaultRegexOptions), + m => new LinkNode(m.Value, m.Groups[1].Value)); + + /* Text */ + + // Capture the shrug emoticon + // This escapes it from matching for formatting + private static readonly IMatcher ShrugTextNodeMatcher = new StringMatcher( + @"¯\_(ツ)_/¯", + s => new TextNode(s)); + + // Capture any "symbol/other" character or surrogate pair preceeded by a backslash + // This escapes it from matching for emoji + private static readonly IMatcher EscapedSymbolTextNodeMatcher = new RegexMatcher( + new Regex("\\\\(\\p{So}|\\p{Cs}{2})", DefaultRegexOptions), + m => new TextNode(m.Value, m.Groups[1].Value)); + + // Capture any non-whitespace, non latin alphanumeric character preceeded by a backslash + // This escapes it from matching for formatting or other tokens + private static readonly IMatcher EscapedCharacterTextNodeMatcher = new RegexMatcher( + new Regex("\\\\([^a-zA-Z0-9\\s])", DefaultRegexOptions), + m => new TextNode(m.Value, m.Groups[1].Value)); + + // Combine all matchers into one + // Matchers that have similar patterns are ordered from most specific to least specific + private static readonly IMatcher AggregateNodeMatcher = new AggregateMatcher( + ItalicBoldFormattedNodeMatcher, + ItalicUnderlineFormattedNodeMatcher, + BoldFormattedNodeMatcher, + ItalicFormattedNodeMatcher, + UnderlineFormattedNodeMatcher, + ItalicAltFormattedNodeMatcher, + StrikethroughFormattedNodeMatcher, + SpoilerFormattedNodeMatcher, + MultilineCodeBlockNodeMatcher, + InlineCodeBlockNodeMatcher, + EveryoneMentionNodeMatcher, + HereMentionNodeMatcher, + UserMentionNodeMatcher, + ChannelMentionNodeMatcher, + RoleMentionNodeMatcher, + StandardEmojiNodeMatcher, + CustomEmojiNodeMatcher, + TitledLinkNodeMatcher, + AutoLinkNodeMatcher, + HiddenLinkNodeMatcher, + ShrugTextNodeMatcher, + EscapedSymbolTextNodeMatcher, + EscapedCharacterTextNodeMatcher); + + private static IReadOnlyList Parse(string input, IMatcher matcher) => + matcher.MatchAll(input, s => new TextNode(s)).Select(r => r.Value).ToArray(); + + public static IReadOnlyList Parse(string input) => Parse(input, AggregateNodeMatcher); } } \ No newline at end of file diff --git a/DiscordChatExporter.Core.Markdown/Node.cs b/DiscordChatExporter.Core.Markdown/Node.cs deleted file mode 100644 index 2b7908a6..00000000 --- a/DiscordChatExporter.Core.Markdown/Node.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace DiscordChatExporter.Core.Markdown -{ - public abstract class Node - { - public string Lexeme { get; } - - protected Node(string lexeme) - { - Lexeme = lexeme; - } - } -} \ No newline at end of file diff --git a/DiscordChatExporter.Core.Markdown/EmojiNode.cs b/DiscordChatExporter.Core.Markdown/Nodes/EmojiNode.cs similarity index 53% rename from DiscordChatExporter.Core.Markdown/EmojiNode.cs rename to DiscordChatExporter.Core.Markdown/Nodes/EmojiNode.cs index ab7b7395..f372c4a5 100644 --- a/DiscordChatExporter.Core.Markdown/EmojiNode.cs +++ b/DiscordChatExporter.Core.Markdown/Nodes/EmojiNode.cs @@ -1,6 +1,4 @@ -using Tyrrrz.Extensions; - -namespace DiscordChatExporter.Core.Markdown +namespace DiscordChatExporter.Core.Markdown.Nodes { public class EmojiNode : Node { @@ -10,18 +8,18 @@ namespace DiscordChatExporter.Core.Markdown public bool IsAnimated { get; } - public bool IsCustomEmoji => Id.IsNotBlank(); + public bool IsCustomEmoji => Id != null; - public EmojiNode(string lexeme, string id, string name, bool isAnimated) - : base(lexeme) + public EmojiNode(string source, string id, string name, bool isAnimated) + : base(source) { Id = id; Name = name; IsAnimated = isAnimated; } - public EmojiNode(string lexeme, string name) - : this(lexeme, null, name, false) + public EmojiNode(string source, string name) + : this(source, null, name, false) { } diff --git a/DiscordChatExporter.Core.Markdown/FormattedNode.cs b/DiscordChatExporter.Core.Markdown/Nodes/FormattedNode.cs similarity index 77% rename from DiscordChatExporter.Core.Markdown/FormattedNode.cs rename to DiscordChatExporter.Core.Markdown/Nodes/FormattedNode.cs index 131b8114..30808da1 100644 --- a/DiscordChatExporter.Core.Markdown/FormattedNode.cs +++ b/DiscordChatExporter.Core.Markdown/Nodes/FormattedNode.cs @@ -1,6 +1,6 @@ using System.Collections.Generic; -namespace DiscordChatExporter.Core.Markdown +namespace DiscordChatExporter.Core.Markdown.Nodes { public class FormattedNode : Node { @@ -10,8 +10,8 @@ namespace DiscordChatExporter.Core.Markdown public IReadOnlyList Children { get; } - public FormattedNode(string lexeme, string token, TextFormatting formatting, IReadOnlyList children) - : base(lexeme) + public FormattedNode(string source, string token, TextFormatting formatting, IReadOnlyList children) + : base(source) { Token = token; Formatting = formatting; diff --git a/DiscordChatExporter.Core.Markdown/InlineCodeBlockNode.cs b/DiscordChatExporter.Core.Markdown/Nodes/InlineCodeBlockNode.cs similarity index 58% rename from DiscordChatExporter.Core.Markdown/InlineCodeBlockNode.cs rename to DiscordChatExporter.Core.Markdown/Nodes/InlineCodeBlockNode.cs index 588b1a7e..e5b85d99 100644 --- a/DiscordChatExporter.Core.Markdown/InlineCodeBlockNode.cs +++ b/DiscordChatExporter.Core.Markdown/Nodes/InlineCodeBlockNode.cs @@ -1,11 +1,11 @@ -namespace DiscordChatExporter.Core.Markdown +namespace DiscordChatExporter.Core.Markdown.Nodes { public class InlineCodeBlockNode : Node { public string Code { get; } - public InlineCodeBlockNode(string lexeme, string code) - : base(lexeme) + public InlineCodeBlockNode(string source, string code) + : base(source) { Code = code; } diff --git a/DiscordChatExporter.Core.Markdown/LinkNode.cs b/DiscordChatExporter.Core.Markdown/Nodes/LinkNode.cs similarity index 58% rename from DiscordChatExporter.Core.Markdown/LinkNode.cs rename to DiscordChatExporter.Core.Markdown/Nodes/LinkNode.cs index 637eb783..9c2e1770 100644 --- a/DiscordChatExporter.Core.Markdown/LinkNode.cs +++ b/DiscordChatExporter.Core.Markdown/Nodes/LinkNode.cs @@ -1,4 +1,4 @@ -namespace DiscordChatExporter.Core.Markdown +namespace DiscordChatExporter.Core.Markdown.Nodes { public class LinkNode : Node { @@ -6,14 +6,14 @@ public string Title { get; } - public LinkNode(string lexeme, string url, string title) - : base(lexeme) + public LinkNode(string source, string url, string title) + : base(source) { Url = url; Title = title; } - public LinkNode(string lexeme, string url) : this(lexeme, url, url) + public LinkNode(string source, string url) : this(source, url, url) { } diff --git a/DiscordChatExporter.Core.Markdown/MentionNode.cs b/DiscordChatExporter.Core.Markdown/Nodes/MentionNode.cs similarity index 65% rename from DiscordChatExporter.Core.Markdown/MentionNode.cs rename to DiscordChatExporter.Core.Markdown/Nodes/MentionNode.cs index 4031baac..3a054425 100644 --- a/DiscordChatExporter.Core.Markdown/MentionNode.cs +++ b/DiscordChatExporter.Core.Markdown/Nodes/MentionNode.cs @@ -1,4 +1,4 @@ -namespace DiscordChatExporter.Core.Markdown +namespace DiscordChatExporter.Core.Markdown.Nodes { public class MentionNode : Node { @@ -6,8 +6,8 @@ public MentionType Type { get; } - public MentionNode(string lexeme, string id, MentionType type) - : base(lexeme) + public MentionNode(string source, string id, MentionType type) + : base(source) { Id = id; Type = type; diff --git a/DiscordChatExporter.Core.Markdown/MentionType.cs b/DiscordChatExporter.Core.Markdown/Nodes/MentionType.cs similarity index 64% rename from DiscordChatExporter.Core.Markdown/MentionType.cs rename to DiscordChatExporter.Core.Markdown/Nodes/MentionType.cs index c5cef3c7..6559ebd7 100644 --- a/DiscordChatExporter.Core.Markdown/MentionType.cs +++ b/DiscordChatExporter.Core.Markdown/Nodes/MentionType.cs @@ -1,4 +1,4 @@ -namespace DiscordChatExporter.Core.Markdown +namespace DiscordChatExporter.Core.Markdown.Nodes { public enum MentionType { diff --git a/DiscordChatExporter.Core.Markdown/MultilineCodeBlockNode.cs b/DiscordChatExporter.Core.Markdown/Nodes/MultilineCodeBlockNode.cs similarity index 68% rename from DiscordChatExporter.Core.Markdown/MultilineCodeBlockNode.cs rename to DiscordChatExporter.Core.Markdown/Nodes/MultilineCodeBlockNode.cs index 62bc9694..3f647bef 100644 --- a/DiscordChatExporter.Core.Markdown/MultilineCodeBlockNode.cs +++ b/DiscordChatExporter.Core.Markdown/Nodes/MultilineCodeBlockNode.cs @@ -1,4 +1,4 @@ -namespace DiscordChatExporter.Core.Markdown +namespace DiscordChatExporter.Core.Markdown.Nodes { public class MultilineCodeBlockNode : Node { @@ -6,8 +6,8 @@ public string Code { get; } - public MultilineCodeBlockNode(string lexeme, string language, string code) - : base(lexeme) + public MultilineCodeBlockNode(string source, string language, string code) + : base(source) { Language = language; Code = code; diff --git a/DiscordChatExporter.Core.Markdown/Nodes/Node.cs b/DiscordChatExporter.Core.Markdown/Nodes/Node.cs new file mode 100644 index 00000000..22f6462a --- /dev/null +++ b/DiscordChatExporter.Core.Markdown/Nodes/Node.cs @@ -0,0 +1,12 @@ +namespace DiscordChatExporter.Core.Markdown.Nodes +{ + public abstract class Node + { + public string Source { get; } + + protected Node(string source) + { + Source = source; + } + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Core.Markdown/TextFormatting.cs b/DiscordChatExporter.Core.Markdown/Nodes/TextFormatting.cs similarity index 71% rename from DiscordChatExporter.Core.Markdown/TextFormatting.cs rename to DiscordChatExporter.Core.Markdown/Nodes/TextFormatting.cs index 175aece5..9a35c5a0 100644 --- a/DiscordChatExporter.Core.Markdown/TextFormatting.cs +++ b/DiscordChatExporter.Core.Markdown/Nodes/TextFormatting.cs @@ -1,4 +1,4 @@ -namespace DiscordChatExporter.Core.Markdown +namespace DiscordChatExporter.Core.Markdown.Nodes { public enum TextFormatting { diff --git a/DiscordChatExporter.Core.Markdown/TextNode.cs b/DiscordChatExporter.Core.Markdown/Nodes/TextNode.cs similarity index 65% rename from DiscordChatExporter.Core.Markdown/TextNode.cs rename to DiscordChatExporter.Core.Markdown/Nodes/TextNode.cs index a7da397e..8bc77778 100644 --- a/DiscordChatExporter.Core.Markdown/TextNode.cs +++ b/DiscordChatExporter.Core.Markdown/Nodes/TextNode.cs @@ -1,11 +1,11 @@ -namespace DiscordChatExporter.Core.Markdown +namespace DiscordChatExporter.Core.Markdown.Nodes { public class TextNode : Node { public string Text { get; } - public TextNode(string lexeme, string text) - : base(lexeme) + public TextNode(string source, string text) + : base(source) { Text = text; } diff --git a/DiscordChatExporter.Core/Models/Attachment.cs b/DiscordChatExporter.Core.Models/Attachment.cs similarity index 57% rename from DiscordChatExporter.Core/Models/Attachment.cs rename to DiscordChatExporter.Core.Models/Attachment.cs index 4e54314e..8a7d10ea 100644 --- a/DiscordChatExporter.Core/Models/Attachment.cs +++ b/DiscordChatExporter.Core.Models/Attachment.cs @@ -1,10 +1,12 @@ using System; +using System.IO; +using System.Linq; namespace DiscordChatExporter.Core.Models { // https://discordapp.com/developers/docs/resources/channel#attachment-object - public class Attachment + public partial class Attachment { public string Id { get; } @@ -16,11 +18,7 @@ namespace DiscordChatExporter.Core.Models public string FileName { get; } - public bool IsImage => FileName.EndsWith(".jpg", StringComparison.OrdinalIgnoreCase) || - FileName.EndsWith(".jpeg", StringComparison.OrdinalIgnoreCase) || - FileName.EndsWith(".png", StringComparison.OrdinalIgnoreCase) || - FileName.EndsWith(".gif", StringComparison.OrdinalIgnoreCase) || - FileName.EndsWith(".bmp", StringComparison.OrdinalIgnoreCase); + public bool IsImage { get; } public FileSize FileSize { get; } @@ -32,8 +30,21 @@ namespace DiscordChatExporter.Core.Models Height = height; FileName = fileName; FileSize = fileSize; + + IsImage = GetIsImage(fileName); } public override string ToString() => FileName; } + + public partial class Attachment + { + private static readonly string[] ImageFileExtensions = { ".jpg", ".jpeg", ".png", ".gif", ".bmp" }; + + public static bool GetIsImage(string fileName) + { + var fileExtension = Path.GetExtension(fileName); + return ImageFileExtensions.Contains(fileExtension, StringComparer.OrdinalIgnoreCase); + } + } } \ No newline at end of file diff --git a/DiscordChatExporter.Core/Models/AuthToken.cs b/DiscordChatExporter.Core.Models/AuthToken.cs similarity index 100% rename from DiscordChatExporter.Core/Models/AuthToken.cs rename to DiscordChatExporter.Core.Models/AuthToken.cs diff --git a/DiscordChatExporter.Core/Models/AuthTokenType.cs b/DiscordChatExporter.Core.Models/AuthTokenType.cs similarity index 100% rename from DiscordChatExporter.Core/Models/AuthTokenType.cs rename to DiscordChatExporter.Core.Models/AuthTokenType.cs diff --git a/DiscordChatExporter.Core/Models/Channel.cs b/DiscordChatExporter.Core.Models/Channel.cs similarity index 100% rename from DiscordChatExporter.Core/Models/Channel.cs rename to DiscordChatExporter.Core.Models/Channel.cs diff --git a/DiscordChatExporter.Core/Models/ChannelType.cs b/DiscordChatExporter.Core.Models/ChannelType.cs similarity index 100% rename from DiscordChatExporter.Core/Models/ChannelType.cs rename to DiscordChatExporter.Core.Models/ChannelType.cs diff --git a/DiscordChatExporter.Core/Models/ChatLog.cs b/DiscordChatExporter.Core.Models/ChatLog.cs similarity index 100% rename from DiscordChatExporter.Core/Models/ChatLog.cs rename to DiscordChatExporter.Core.Models/ChatLog.cs diff --git a/DiscordChatExporter.Core.Models/DiscordChatExporter.Core.Models.csproj b/DiscordChatExporter.Core.Models/DiscordChatExporter.Core.Models.csproj new file mode 100644 index 00000000..03f624f8 --- /dev/null +++ b/DiscordChatExporter.Core.Models/DiscordChatExporter.Core.Models.csproj @@ -0,0 +1,11 @@ + + + + net46;netstandard2.0 + + + + + + + \ No newline at end of file diff --git a/DiscordChatExporter.Core/Models/Embed.cs b/DiscordChatExporter.Core.Models/Embed.cs similarity index 100% rename from DiscordChatExporter.Core/Models/Embed.cs rename to DiscordChatExporter.Core.Models/Embed.cs diff --git a/DiscordChatExporter.Core/Models/EmbedAuthor.cs b/DiscordChatExporter.Core.Models/EmbedAuthor.cs similarity index 100% rename from DiscordChatExporter.Core/Models/EmbedAuthor.cs rename to DiscordChatExporter.Core.Models/EmbedAuthor.cs diff --git a/DiscordChatExporter.Core/Models/EmbedField.cs b/DiscordChatExporter.Core.Models/EmbedField.cs similarity index 100% rename from DiscordChatExporter.Core/Models/EmbedField.cs rename to DiscordChatExporter.Core.Models/EmbedField.cs diff --git a/DiscordChatExporter.Core/Models/EmbedFooter.cs b/DiscordChatExporter.Core.Models/EmbedFooter.cs similarity index 100% rename from DiscordChatExporter.Core/Models/EmbedFooter.cs rename to DiscordChatExporter.Core.Models/EmbedFooter.cs diff --git a/DiscordChatExporter.Core/Models/EmbedImage.cs b/DiscordChatExporter.Core.Models/EmbedImage.cs similarity index 100% rename from DiscordChatExporter.Core/Models/EmbedImage.cs rename to DiscordChatExporter.Core.Models/EmbedImage.cs diff --git a/DiscordChatExporter.Core/Models/Emoji.cs b/DiscordChatExporter.Core.Models/Emoji.cs similarity index 55% rename from DiscordChatExporter.Core/Models/Emoji.cs rename to DiscordChatExporter.Core.Models/Emoji.cs index 24abe4ce..b976e1fb 100644 --- a/DiscordChatExporter.Core/Models/Emoji.cs +++ b/DiscordChatExporter.Core.Models/Emoji.cs @@ -14,31 +14,15 @@ namespace DiscordChatExporter.Core.Models public bool IsAnimated { get; } - public string ImageUrl - { - get - { - // Custom emoji - if (Id.IsNotBlank()) - { - // Animated - if (IsAnimated) - return $"https://cdn.discordapp.com/emojis/{Id}.gif"; - - // Non-animated - return $"https://cdn.discordapp.com/emojis/{Id}.png"; - } - - // Standard unicode emoji (via twemoji) - return $"https://twemoji.maxcdn.com/2/72x72/{GetTwemojiName(Name)}.png"; - } - } + public string ImageUrl { get; } public Emoji(string id, string name, bool isAnimated) { Id = id; Name = name; IsAnimated = isAnimated; + + ImageUrl = GetImageUrl(id, name, isAnimated); } } @@ -50,7 +34,25 @@ namespace DiscordChatExporter.Core.Models yield return char.ConvertToUtf32(emoji, i); } - private static string GetTwemojiName(string emoji) - => GetCodePoints(emoji).Select(i => i.ToString("x")).JoinToString("-"); + private static string GetTwemojiName(string emoji) => + GetCodePoints(emoji).Select(i => i.ToString("x")).JoinToString("-"); + + public static string GetImageUrl(string id, string name, bool isAnimated) + { + // Custom emoji + if (id != null) + { + // Animated + if (isAnimated) + return $"https://cdn.discordapp.com/emojis/{id}.gif"; + + // Non-animated + return $"https://cdn.discordapp.com/emojis/{id}.png"; + } + + // Standard unicode emoji (via twemoji) + var twemojiName = GetTwemojiName(name); + return $"https://twemoji.maxcdn.com/2/72x72/{twemojiName}.png"; + } } } \ No newline at end of file diff --git a/DiscordChatExporter.Core/Models/ExportFormat.cs b/DiscordChatExporter.Core.Models/ExportFormat.cs similarity index 100% rename from DiscordChatExporter.Core/Models/ExportFormat.cs rename to DiscordChatExporter.Core.Models/ExportFormat.cs diff --git a/DiscordChatExporter.Core/Models/Extensions.cs b/DiscordChatExporter.Core.Models/Extensions.cs similarity index 100% rename from DiscordChatExporter.Core/Models/Extensions.cs rename to DiscordChatExporter.Core.Models/Extensions.cs diff --git a/DiscordChatExporter.Core/Models/FileSize.cs b/DiscordChatExporter.Core.Models/FileSize.cs similarity index 100% rename from DiscordChatExporter.Core/Models/FileSize.cs rename to DiscordChatExporter.Core.Models/FileSize.cs diff --git a/DiscordChatExporter.Core/Models/Guild.cs b/DiscordChatExporter.Core.Models/Guild.cs similarity index 59% rename from DiscordChatExporter.Core/Models/Guild.cs rename to DiscordChatExporter.Core.Models/Guild.cs index 3babd4b2..cb70a19f 100644 --- a/DiscordChatExporter.Core/Models/Guild.cs +++ b/DiscordChatExporter.Core.Models/Guild.cs @@ -1,6 +1,4 @@ -using Tyrrrz.Extensions; - -namespace DiscordChatExporter.Core.Models +namespace DiscordChatExporter.Core.Models { // https://discordapp.com/developers/docs/resources/guild#guild-object @@ -12,15 +10,15 @@ namespace DiscordChatExporter.Core.Models public string IconHash { get; } - public string IconUrl => IconHash.IsNotBlank() - ? $"https://cdn.discordapp.com/icons/{Id}/{IconHash}.png" - : "https://cdn.discordapp.com/embed/avatars/0.png"; + public string IconUrl { get; } public Guild(string id, string name, string iconHash) { Id = id; Name = name; IconHash = iconHash; + + IconUrl = GetIconUrl(id, iconHash); } public override string ToString() => Name; @@ -28,6 +26,13 @@ namespace DiscordChatExporter.Core.Models public partial class Guild { + public static string GetIconUrl(string id, string iconHash) + { + return iconHash != null + ? $"https://cdn.discordapp.com/icons/{id}/{iconHash}.png" + : "https://cdn.discordapp.com/embed/avatars/0.png"; + } + public static Guild DirectMessages { get; } = new Guild("@me", "Direct Messages", null); } } \ No newline at end of file diff --git a/DiscordChatExporter.Core/Models/Mentionables.cs b/DiscordChatExporter.Core.Models/Mentionables.cs similarity index 100% rename from DiscordChatExporter.Core/Models/Mentionables.cs rename to DiscordChatExporter.Core.Models/Mentionables.cs diff --git a/DiscordChatExporter.Core/Models/Message.cs b/DiscordChatExporter.Core.Models/Message.cs similarity index 100% rename from DiscordChatExporter.Core/Models/Message.cs rename to DiscordChatExporter.Core.Models/Message.cs diff --git a/DiscordChatExporter.Core/Models/MessageType.cs b/DiscordChatExporter.Core.Models/MessageType.cs similarity index 100% rename from DiscordChatExporter.Core/Models/MessageType.cs rename to DiscordChatExporter.Core.Models/MessageType.cs diff --git a/DiscordChatExporter.Core/Models/Reaction.cs b/DiscordChatExporter.Core.Models/Reaction.cs similarity index 100% rename from DiscordChatExporter.Core/Models/Reaction.cs rename to DiscordChatExporter.Core.Models/Reaction.cs diff --git a/DiscordChatExporter.Core/Models/Role.cs b/DiscordChatExporter.Core.Models/Role.cs similarity index 81% rename from DiscordChatExporter.Core/Models/Role.cs rename to DiscordChatExporter.Core.Models/Role.cs index 0f54ac14..4eb1b20c 100644 --- a/DiscordChatExporter.Core/Models/Role.cs +++ b/DiscordChatExporter.Core.Models/Role.cs @@ -19,7 +19,6 @@ public partial class Role { - public static Role CreateDeletedRole(string id) => - new Role(id, "deleted-role"); + public static Role CreateDeletedRole(string id) => new Role(id, "deleted-role"); } } \ No newline at end of file diff --git a/DiscordChatExporter.Core.Models/User.cs b/DiscordChatExporter.Core.Models/User.cs new file mode 100644 index 00000000..2a5154cf --- /dev/null +++ b/DiscordChatExporter.Core.Models/User.cs @@ -0,0 +1,58 @@ +using System; + +namespace DiscordChatExporter.Core.Models +{ + // https://discordapp.com/developers/docs/topics/permissions#role-object + + public partial class User + { + public string Id { get; } + + public int Discriminator { get; } + + public string Name { get; } + + public string FullName { get; } + + public string AvatarHash { get; } + + public string AvatarUrl { get; } + + public User(string id, int discriminator, string name, string avatarHash) + { + Id = id; + Discriminator = discriminator; + Name = name; + AvatarHash = avatarHash; + + FullName = GetFullName(name, discriminator); + AvatarUrl = GetAvatarUrl(id, discriminator, avatarHash); + } + + public override string ToString() => FullName; + } + + public partial class User + { + public static string GetFullName(string name, int discriminator) => $"{name}#{discriminator:0000}"; + + public static string GetAvatarUrl(string id, int discriminator, string avatarHash) + { + // Custom avatar + if (avatarHash != null) + { + // Animated + if (avatarHash.StartsWith("a_", StringComparison.Ordinal)) + return $"https://cdn.discordapp.com/avatars/{id}/{avatarHash}.gif"; + + // Non-animated + return $"https://cdn.discordapp.com/avatars/{id}/{avatarHash}.png"; + } + + // Default avatar + return $"https://cdn.discordapp.com/embed/avatars/{discriminator % 5}.png"; + } + + public static User CreateUnknownUser(string id) => new User(id, 0, "Unknown", null); + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Core.Rendering/CsvChatLogRenderer.cs b/DiscordChatExporter.Core.Rendering/CsvChatLogRenderer.cs new file mode 100644 index 00000000..b8afe18d --- /dev/null +++ b/DiscordChatExporter.Core.Rendering/CsvChatLogRenderer.cs @@ -0,0 +1,112 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using DiscordChatExporter.Core.Markdown; +using DiscordChatExporter.Core.Markdown.Nodes; +using DiscordChatExporter.Core.Models; +using Tyrrrz.Extensions; + +namespace DiscordChatExporter.Core.Rendering +{ + public class CsvChatLogRenderer : IChatLogRenderer + { + private readonly ChatLog _chatLog; + private readonly string _dateFormat; + + public CsvChatLogRenderer(ChatLog chatLog, string dateFormat) + { + _chatLog = chatLog; + _dateFormat = dateFormat; + } + + private string FormatDate(DateTime date) => date.ToString(_dateFormat, CultureInfo.InvariantCulture); + + private string FormatMarkdown(Node node) + { + // Formatted node + if (node is FormattedNode formattedNode) + { + // Recursively get inner text + var innerText = FormatMarkdown(formattedNode.Children); + + return $"{formattedNode.Token}{innerText}{formattedNode.Token}"; + } + + // Non-meta mention node + if (node is MentionNode mentionNode && mentionNode.Type != MentionType.Meta) + { + // User mention node + if (mentionNode.Type == MentionType.User) + { + var user = _chatLog.Mentionables.GetUser(mentionNode.Id); + return $"@{user.Name}"; + } + + // Channel mention node + if (mentionNode.Type == MentionType.Channel) + { + var channel = _chatLog.Mentionables.GetChannel(mentionNode.Id); + return $"#{channel.Name}"; + } + + // Role mention node + if (mentionNode.Type == MentionType.Role) + { + var role = _chatLog.Mentionables.GetRole(mentionNode.Id); + return $"@{role.Name}"; + } + } + + // Custom emoji node + if (node is EmojiNode emojiNode && emojiNode.IsCustomEmoji) + { + return $":{emojiNode.Name}:"; + } + + // All other nodes - simply return source + return node.Source; + } + + private string FormatMarkdown(IEnumerable nodes) => nodes.Select(FormatMarkdown).JoinToString(""); + + private string FormatMarkdown(string markdown) => FormatMarkdown(MarkdownParser.Parse(markdown)); + + private async Task RenderFieldAsync(TextWriter writer, string value) + { + var encodedValue = value.Replace("\"", "\"\""); + await writer.WriteAsync($"\"{encodedValue}\";"); + } + + private async Task RenderMessageAsync(TextWriter writer, Message message) + { + // Author + await RenderFieldAsync(writer, message.Author.FullName); + + // Timestamp + await RenderFieldAsync(writer, FormatDate(message.Timestamp)); + + // Content + await RenderFieldAsync(writer, FormatMarkdown(message.Content)); + + // Attachments + var formattedAttachments = message.Attachments.Select(a => a.Url).JoinToString(","); + await RenderFieldAsync(writer, formattedAttachments); + + // Line break + await writer.WriteLineAsync(); + } + + public async Task RenderAsync(TextWriter writer) + { + // Headers + await writer.WriteLineAsync("Author;Date;Content;Attachments;"); + + // Log + foreach (var message in _chatLog.Messages) + await RenderMessageAsync(writer, message); + } + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Core.Rendering/DiscordChatExporter.Core.Rendering.csproj b/DiscordChatExporter.Core.Rendering/DiscordChatExporter.Core.Rendering.csproj new file mode 100644 index 00000000..bd5c30ca --- /dev/null +++ b/DiscordChatExporter.Core.Rendering/DiscordChatExporter.Core.Rendering.csproj @@ -0,0 +1,25 @@ + + + + net46;netstandard2.0 + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/DiscordChatExporter.Core/Services/ExportService.MessageGroup.cs b/DiscordChatExporter.Core.Rendering/HtmlChatLogRenderer.MessageGroup.cs similarity index 76% rename from DiscordChatExporter.Core/Services/ExportService.MessageGroup.cs rename to DiscordChatExporter.Core.Rendering/HtmlChatLogRenderer.MessageGroup.cs index 05775407..e5609558 100644 --- a/DiscordChatExporter.Core/Services/ExportService.MessageGroup.cs +++ b/DiscordChatExporter.Core.Rendering/HtmlChatLogRenderer.MessageGroup.cs @@ -1,10 +1,10 @@ -using System; +using DiscordChatExporter.Core.Models; +using System; using System.Collections.Generic; -using DiscordChatExporter.Core.Models; -namespace DiscordChatExporter.Core.Services +namespace DiscordChatExporter.Core.Rendering { - public partial class ExportService + public partial class HtmlChatLogRenderer { private class MessageGroup { diff --git a/DiscordChatExporter.Core.Rendering/HtmlChatLogRenderer.TemplateLoader.cs b/DiscordChatExporter.Core.Rendering/HtmlChatLogRenderer.TemplateLoader.cs new file mode 100644 index 00000000..21504c4d --- /dev/null +++ b/DiscordChatExporter.Core.Rendering/HtmlChatLogRenderer.TemplateLoader.cs @@ -0,0 +1,27 @@ +using Scriban.Parsing; +using Scriban.Runtime; +using Scriban; +using System.Reflection; +using System.Threading.Tasks; +using Tyrrrz.Extensions; + +namespace DiscordChatExporter.Core.Rendering +{ + public partial class HtmlChatLogRenderer + { + private class TemplateLoader : ITemplateLoader + { + private const string ResourceRootNamespace = "DiscordChatExporter.Core.Rendering.Resources"; + + public string Load(string templatePath) => + Assembly.GetExecutingAssembly().GetManifestResourceString($"{ResourceRootNamespace}.{templatePath}"); + + public string GetPath(TemplateContext context, SourceSpan callerSpan, string templateName) => templateName; + + public string Load(TemplateContext context, SourceSpan callerSpan, string templatePath) => Load(templatePath); + + public ValueTask LoadAsync(TemplateContext context, SourceSpan callerSpan, string templatePath) => + new ValueTask(Load(templatePath)); + } + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Core.Rendering/HtmlChatLogRenderer.cs b/DiscordChatExporter.Core.Rendering/HtmlChatLogRenderer.cs new file mode 100644 index 00000000..530b8250 --- /dev/null +++ b/DiscordChatExporter.Core.Rendering/HtmlChatLogRenderer.cs @@ -0,0 +1,197 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using DiscordChatExporter.Core.Markdown; +using DiscordChatExporter.Core.Markdown.Nodes; +using DiscordChatExporter.Core.Models; +using Scriban; +using Scriban.Runtime; +using Tyrrrz.Extensions; + +namespace DiscordChatExporter.Core.Rendering +{ + public partial class HtmlChatLogRenderer : IChatLogRenderer + { + private readonly ChatLog _chatLog; + private readonly string _themeName; + private readonly string _dateFormat; + + public HtmlChatLogRenderer(ChatLog chatLog, string themeName, string dateFormat) + { + _chatLog = chatLog; + _themeName = themeName; + _dateFormat = dateFormat; + } + + private string HtmlEncode(string s) => WebUtility.HtmlEncode(s); + + private string FormatDate(DateTime date) => date.ToString(_dateFormat, CultureInfo.InvariantCulture); + + private IEnumerable GroupMessages(IEnumerable messages) => + messages.GroupContiguous((buffer, message) => + { + // Break group if the author changed + if (buffer.Last().Author.Id != message.Author.Id) + return false; + + // Break group if last message was more than 7 minutes ago + if ((message.Timestamp - buffer.Last().Timestamp).TotalMinutes > 7) + return false; + + return true; + }).Select(g => new MessageGroup(g.First().Author, g.First().Timestamp, g)); + + private string FormatMarkdown(Node node, bool isTopLevel, bool isSingle) + { + // Text node + if (node is TextNode textNode) + { + // Return HTML-encoded text + return HtmlEncode(textNode.Text); + } + + // Formatted node + if (node is FormattedNode formattedNode) + { + // Recursively get inner html + var innerHtml = FormatMarkdown(formattedNode.Children, false); + + // Bold + if (formattedNode.Formatting == TextFormatting.Bold) + return $"{innerHtml}"; + + // Italic + if (formattedNode.Formatting == TextFormatting.Italic) + return $"{innerHtml}"; + + // Underline + if (formattedNode.Formatting == TextFormatting.Underline) + return $"{innerHtml}"; + + // Strikethrough + if (formattedNode.Formatting == TextFormatting.Strikethrough) + return $"{innerHtml}"; + + // Spoiler + if (formattedNode.Formatting == TextFormatting.Spoiler) + return $"{innerHtml}"; + } + + // Inline code block node + if (node is InlineCodeBlockNode inlineCodeBlockNode) + { + return $"{HtmlEncode(inlineCodeBlockNode.Code)}"; + } + + // Multi-line code block node + if (node is MultilineCodeBlockNode multilineCodeBlockNode) + { + // Set language class for syntax highlighting + var languageCssClass = multilineCodeBlockNode.Language != null + ? "language-" + multilineCodeBlockNode.Language + : null; + + return $"
{HtmlEncode(multilineCodeBlockNode.Code)}
"; + } + + // Mention node + if (node is MentionNode mentionNode) + { + // Meta mention node + if (mentionNode.Type == MentionType.Meta) + { + return $"@{HtmlEncode(mentionNode.Id)}"; + } + + // User mention node + if (mentionNode.Type == MentionType.User) + { + var user = _chatLog.Mentionables.GetUser(mentionNode.Id); + return $"@{HtmlEncode(user.Name)}"; + } + + // Channel mention node + if (mentionNode.Type == MentionType.Channel) + { + var channel = _chatLog.Mentionables.GetChannel(mentionNode.Id); + return $"#{HtmlEncode(channel.Name)}"; + } + + // Role mention node + if (mentionNode.Type == MentionType.Role) + { + var role = _chatLog.Mentionables.GetRole(mentionNode.Id); + return $"@{HtmlEncode(role.Name)}"; + } + } + + // Emoji node + if (node is EmojiNode emojiNode) + { + // Get emoji image URL + var emojiImageUrl = Emoji.GetImageUrl(emojiNode.Id, emojiNode.Name, emojiNode.IsAnimated); + + // Emoji can be jumboable if it's the only top-level node + var jumboableCssClass = isTopLevel && isSingle ? "emoji--large" : null; + + return $"\"{emojiNode.Name}\""; + } + + // Link node + if (node is LinkNode linkNode) + { + return $"{HtmlEncode(linkNode.Title)}"; + } + + // All other nodes - simply return source + return node.Source; + } + + private string FormatMarkdown(IReadOnlyList nodes, bool isTopLevel) + { + var isSingle = nodes.Count == 1; + return nodes.Select(n => FormatMarkdown(n, isTopLevel, isSingle)).JoinToString(""); + } + + private string FormatMarkdown(string markdown) => FormatMarkdown(MarkdownParser.Parse(markdown), true); + + public async Task RenderAsync(TextWriter writer) + { + // Create template loader + var loader = new TemplateLoader(); + + // Get template + var templateCode = loader.Load($"Html{_themeName}.html"); + var template = Template.Parse(templateCode); + + // Create template context + var context = new TemplateContext + { + TemplateLoader = loader, + MemberRenamer = m => m.Name, + MemberFilter = m => true, + LoopLimit = int.MaxValue, + StrictVariables = true + }; + + // Create template model + var model = new ScriptObject(); + model.SetValue("Model", _chatLog, true); + model.Import(nameof(GroupMessages), new Func, IEnumerable>(GroupMessages)); + model.Import(nameof(FormatDate), new Func(FormatDate)); + model.Import(nameof(FormatMarkdown), new Func(FormatMarkdown)); + context.PushGlobal(model); + + // Configure output + context.PushOutput(new TextWriterOutput(writer)); + + // HACK: Render output in a separate thread + // (even though Scriban has async API, it still makes a lot of blocking CPU-bound calls) + await Task.Run(async () => await context.EvaluateAsync(template.Page)); + } + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Core.Rendering/IChatLogRenderer.cs b/DiscordChatExporter.Core.Rendering/IChatLogRenderer.cs new file mode 100644 index 00000000..8f280c6a --- /dev/null +++ b/DiscordChatExporter.Core.Rendering/IChatLogRenderer.cs @@ -0,0 +1,10 @@ +using System.IO; +using System.Threading.Tasks; + +namespace DiscordChatExporter.Core.Rendering +{ + public interface IChatLogRenderer + { + Task RenderAsync(TextWriter writer); + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Core.Rendering/PlainTextChatLogRenderer.cs b/DiscordChatExporter.Core.Rendering/PlainTextChatLogRenderer.cs new file mode 100644 index 00000000..31b6a3b7 --- /dev/null +++ b/DiscordChatExporter.Core.Rendering/PlainTextChatLogRenderer.cs @@ -0,0 +1,128 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using DiscordChatExporter.Core.Markdown; +using DiscordChatExporter.Core.Markdown.Nodes; +using DiscordChatExporter.Core.Models; +using Tyrrrz.Extensions; + +namespace DiscordChatExporter.Core.Rendering +{ + public class PlainTextChatLogRenderer : IChatLogRenderer + { + private readonly ChatLog _chatLog; + private readonly string _dateFormat; + + public PlainTextChatLogRenderer(ChatLog chatLog, string dateFormat) + { + _chatLog = chatLog; + _dateFormat = dateFormat; + } + + private string FormatDate(DateTime date) => date.ToString(_dateFormat, CultureInfo.InvariantCulture); + + private string FormatDateRange(DateTime? from, DateTime? to) + { + // Both 'from' and 'to' + if (from.HasValue && to.HasValue) + return $"{FormatDate(from.Value)} to {FormatDate(to.Value)}"; + + // Just 'from' + if (from.HasValue) + return $"after {FormatDate(from.Value)}"; + + // Just 'to' + if (to.HasValue) + return $"before {FormatDate(to.Value)}"; + + // Neither + return null; + } + + private string FormatMarkdown(Node node) + { + // Formatted node + if (node is FormattedNode formattedNode) + { + // Recursively get inner text + var innerText = FormatMarkdown(formattedNode.Children); + + return $"{formattedNode.Token}{innerText}{formattedNode.Token}"; + } + + // Non-meta mention node + if (node is MentionNode mentionNode && mentionNode.Type != MentionType.Meta) + { + // User mention node + if (mentionNode.Type == MentionType.User) + { + var user = _chatLog.Mentionables.GetUser(mentionNode.Id); + return $"@{user.Name}"; + } + + // Channel mention node + if (mentionNode.Type == MentionType.Channel) + { + var channel = _chatLog.Mentionables.GetChannel(mentionNode.Id); + return $"#{channel.Name}"; + } + + // Role mention node + if (mentionNode.Type == MentionType.Role) + { + var role = _chatLog.Mentionables.GetRole(mentionNode.Id); + return $"@{role.Name}"; + } + } + + // Custom emoji node + if (node is EmojiNode emojiNode && emojiNode.IsCustomEmoji) + { + return $":{emojiNode.Name}:"; + } + + // All other nodes - simply return source + return node.Source; + } + + private string FormatMarkdown(IEnumerable nodes) => nodes.Select(FormatMarkdown).JoinToString(""); + + private string FormatMarkdown(string markdown) => FormatMarkdown(MarkdownParser.Parse(markdown)); + + private async Task RenderMessageAsync(TextWriter writer, Message message) + { + // Timestamp and author + await writer.WriteLineAsync($"[{FormatDate(message.Timestamp)}] {message.Author.FullName}"); + + // Content + await writer.WriteLineAsync(FormatMarkdown(message.Content)); + + // Attachments + foreach (var attachment in message.Attachments) + await writer.WriteLineAsync(attachment.Url); + } + + public async Task RenderAsync(TextWriter writer) + { + // Metadata + await writer.WriteLineAsync('='.Repeat(62)); + await writer.WriteLineAsync($"Guild: {_chatLog.Guild.Name}"); + await writer.WriteLineAsync($"Channel: {_chatLog.Channel.Name}"); + await writer.WriteLineAsync($"Topic: {_chatLog.Channel.Topic}"); + await writer.WriteLineAsync($"Messages: {_chatLog.Messages.Count:N0}"); + await writer.WriteLineAsync($"Range: {FormatDateRange(_chatLog.From, _chatLog.To)}"); + await writer.WriteLineAsync('='.Repeat(62)); + await writer.WriteLineAsync(); + + // Log + foreach (var message in _chatLog.Messages) + { + await RenderMessageAsync(writer, message); + await writer.WriteLineAsync(); + } + } + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Core/Resources/ExportTemplates/HtmlDark/Theme.css b/DiscordChatExporter.Core.Rendering/Resources/HtmlDark.css similarity index 100% rename from DiscordChatExporter.Core/Resources/ExportTemplates/HtmlDark/Theme.css rename to DiscordChatExporter.Core.Rendering/Resources/HtmlDark.css diff --git a/DiscordChatExporter.Core.Rendering/Resources/HtmlDark.html b/DiscordChatExporter.Core.Rendering/Resources/HtmlDark.html new file mode 100644 index 00000000..5cafcc86 --- /dev/null +++ b/DiscordChatExporter.Core.Rendering/Resources/HtmlDark.html @@ -0,0 +1,3 @@ +{{~ ThemeStyleSheet = include "HtmlDark.css" ~}} +{{~ HighlightJsStyleName = "solarized-dark" ~}} +{{~ include "HtmlShared.html" ~}} \ No newline at end of file diff --git a/DiscordChatExporter.Core/Resources/ExportTemplates/HtmlLight/Theme.css b/DiscordChatExporter.Core.Rendering/Resources/HtmlLight.css similarity index 100% rename from DiscordChatExporter.Core/Resources/ExportTemplates/HtmlLight/Theme.css rename to DiscordChatExporter.Core.Rendering/Resources/HtmlLight.css diff --git a/DiscordChatExporter.Core.Rendering/Resources/HtmlLight.html b/DiscordChatExporter.Core.Rendering/Resources/HtmlLight.html new file mode 100644 index 00000000..49c1b931 --- /dev/null +++ b/DiscordChatExporter.Core.Rendering/Resources/HtmlLight.html @@ -0,0 +1,3 @@ +{{~ ThemeStyleSheet = include "HtmlLight.css" ~}} +{{~ HighlightJsStyleName = "solarized-light" ~}} +{{~ include "HtmlShared.html" ~}} \ No newline at end of file diff --git a/DiscordChatExporter.Core/Resources/ExportTemplates/HtmlShared/Main.css b/DiscordChatExporter.Core.Rendering/Resources/HtmlShared.css similarity index 100% rename from DiscordChatExporter.Core/Resources/ExportTemplates/HtmlShared/Main.css rename to DiscordChatExporter.Core.Rendering/Resources/HtmlShared.css diff --git a/DiscordChatExporter.Core/Resources/ExportTemplates/HtmlShared/Main.html b/DiscordChatExporter.Core.Rendering/Resources/HtmlShared.html similarity index 99% rename from DiscordChatExporter.Core/Resources/ExportTemplates/HtmlShared/Main.html rename to DiscordChatExporter.Core.Rendering/Resources/HtmlShared.html index 2b173087..a2a3fc56 100644 --- a/DiscordChatExporter.Core/Resources/ExportTemplates/HtmlShared/Main.html +++ b/DiscordChatExporter.Core.Rendering/Resources/HtmlShared.html @@ -9,7 +9,7 @@ {{~ # Styles ~}}