mirror of
https://github.com/Tyrrrz/DiscordChatExporter.git
synced 2025-05-22 10:55:15 -04:00
Refactor
This commit is contained in:
parent
e1726683f8
commit
650c55bbd1
47 changed files with 280 additions and 266 deletions
|
@ -33,11 +33,11 @@ namespace DiscordChatExporter.Cli.Commands.Base
|
|||
[CommandOption("before", Description = "Only include messages sent before this date or message ID.")]
|
||||
public Snowflake? Before { get; init; }
|
||||
|
||||
[CommandOption("partition", 'p', Description = "Split output into partitions, each limited to this number of messages (e.g. 100) or file size (e.g. 10mb).")]
|
||||
public PartitionLimit PartitionLimit { get; init; } = NullPartitionLimit.Instance;
|
||||
[CommandOption("partition", 'p', Description = "Split output into partitions, each limited to this number of messages (e.g. '100') or file size (e.g. '10mb').")]
|
||||
public PartitionLimit PartitionLimit { get; init; } = PartitionLimit.Null;
|
||||
|
||||
[CommandOption("filter", Description = "Only include messages that satisfy this filter (e.g. from:foo#1234).")]
|
||||
public MessageFilter MessageFilter { get; init; } = NullMessageFilter.Instance;
|
||||
[CommandOption("filter", Description = "Only include messages that satisfy this filter (e.g. 'from:foo#1234' or 'has:image').")]
|
||||
public MessageFilter MessageFilter { get; init; } = MessageFilter.Null;
|
||||
|
||||
[CommandOption("parallel", Description = "Limits how many channels can be exported in parallel.")]
|
||||
public int ParallelLimit { get; init; } = 1;
|
||||
|
@ -133,8 +133,6 @@ namespace DiscordChatExporter.Cli.Commands.Base
|
|||
{
|
||||
throw new CommandException("Export failed.");
|
||||
}
|
||||
|
||||
await console.Output.WriteLineAsync("Done.");
|
||||
}
|
||||
|
||||
public override ValueTask ExecuteAsync(IConsole console)
|
||||
|
|
|
@ -6,8 +6,8 @@
|
|||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="CliFx" Version="2.0.5" />
|
||||
<PackageReference Include="Spectre.Console" Version="0.40.0" />
|
||||
<PackageReference Include="CliFx" Version="2.0.6" />
|
||||
<PackageReference Include="Spectre.Console" Version="0.41.0" />
|
||||
<PackageReference Include="Gress" Version="1.2.0" />
|
||||
<PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" />
|
||||
</ItemGroup>
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
namespace DiscordChatExporter.Core.Exporting.Filtering
|
||||
{
|
||||
public enum BinaryExpressionKind
|
||||
internal enum BinaryExpressionKind
|
||||
{
|
||||
Or,
|
||||
And
|
||||
|
|
|
@ -1,9 +1,9 @@
|
|||
using DiscordChatExporter.Core.Discord.Data;
|
||||
using System;
|
||||
using System;
|
||||
using DiscordChatExporter.Core.Discord.Data;
|
||||
|
||||
namespace DiscordChatExporter.Core.Exporting.Filtering
|
||||
{
|
||||
public class BinaryExpressionMessageFilter : MessageFilter
|
||||
internal class BinaryExpressionMessageFilter : MessageFilter
|
||||
{
|
||||
private readonly MessageFilter _first;
|
||||
private readonly MessageFilter _second;
|
||||
|
@ -19,8 +19,8 @@ namespace DiscordChatExporter.Core.Exporting.Filtering
|
|||
public override bool Filter(Message message) => _kind switch
|
||||
{
|
||||
BinaryExpressionKind.Or => _first.Filter(message) || _second.Filter(message),
|
||||
BinaryExpressionKind.And => _first.Filter(message) && _second.Filter(message),
|
||||
BinaryExpressionKind.And => _first.Filter(message) && _second.Filter(message),
|
||||
_ => throw new InvalidOperationException($"Unknown binary expression kind '{_kind}'.")
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,15 +1,18 @@
|
|||
using DiscordChatExporter.Core.Discord.Data;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Text.RegularExpressions;
|
||||
using DiscordChatExporter.Core.Discord.Data;
|
||||
|
||||
namespace DiscordChatExporter.Core.Exporting.Filtering
|
||||
{
|
||||
public class ContainsMessageFilter : MessageFilter
|
||||
internal class ContainsMessageFilter : MessageFilter
|
||||
{
|
||||
private readonly string _value;
|
||||
private readonly string _text;
|
||||
|
||||
public ContainsMessageFilter(string value) => _value = value;
|
||||
public ContainsMessageFilter(string text) => _text = text;
|
||||
|
||||
public override bool Filter(Message message) =>
|
||||
Regex.IsMatch(message.Content, $@"\b{Regex.Escape(_value)}\b", RegexOptions.IgnoreCase | DefaultRegexOptions);
|
||||
public override bool Filter(Message message) => Regex.IsMatch(
|
||||
message.Content,
|
||||
"\\b" + _text + "\\b",
|
||||
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant
|
||||
);
|
||||
}
|
||||
}
|
|
@ -1,9 +1,9 @@
|
|||
using DiscordChatExporter.Core.Discord.Data;
|
||||
using System;
|
||||
using System;
|
||||
using DiscordChatExporter.Core.Discord.Data;
|
||||
|
||||
namespace DiscordChatExporter.Core.Exporting.Filtering
|
||||
{
|
||||
public class FromMessageFilter : MessageFilter
|
||||
internal class FromMessageFilter : MessageFilter
|
||||
{
|
||||
private readonly string _value;
|
||||
|
||||
|
@ -14,4 +14,4 @@ namespace DiscordChatExporter.Core.Exporting.Filtering
|
|||
string.Equals(_value, message.Author.FullName, StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(_value, message.Author.Id.ToString(), StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,26 +1,25 @@
|
|||
using DiscordChatExporter.Core.Discord.Data;
|
||||
using System;
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using DiscordChatExporter.Core.Discord.Data;
|
||||
|
||||
namespace DiscordChatExporter.Core.Exporting.Filtering
|
||||
{
|
||||
public class HasMessageFilter : MessageFilter
|
||||
internal class HasMessageFilter : MessageFilter
|
||||
{
|
||||
private readonly string _value;
|
||||
private readonly MessageContentMatchKind _kind;
|
||||
|
||||
public HasMessageFilter(string value) => _value = value;
|
||||
public HasMessageFilter(MessageContentMatchKind kind) => _kind = kind;
|
||||
|
||||
public override bool Filter(Message message) =>
|
||||
_value switch
|
||||
{
|
||||
"link" => Regex.IsMatch(message.Content, "https?://\\S*[^\\.,:;\"\'\\s]", DefaultRegexOptions),
|
||||
"embed" => message.Embeds.Any(),
|
||||
"file" => message.Attachments.Any(),
|
||||
"video" => message.Attachments.Any(file => file.IsVideo),
|
||||
"image" => message.Attachments.Any(file => file.IsImage),
|
||||
"sound" => message.Attachments.Any(file => file.IsAudio),
|
||||
_ => throw new InvalidOperationException($"Invalid value provided for the 'has' message filter: '{_value}'")
|
||||
};
|
||||
public override bool Filter(Message message) => _kind switch
|
||||
{
|
||||
MessageContentMatchKind.Link => Regex.IsMatch(message.Content, "https?://\\S*[^\\.,:;\"\'\\s]"),
|
||||
MessageContentMatchKind.Embed => message.Embeds.Any(),
|
||||
MessageContentMatchKind.File => message.Attachments.Any(),
|
||||
MessageContentMatchKind.Video => message.Attachments.Any(file => file.IsVideo),
|
||||
MessageContentMatchKind.Image => message.Attachments.Any(file => file.IsImage),
|
||||
MessageContentMatchKind.Sound => message.Attachments.Any(file => file.IsAudio),
|
||||
_ => throw new InvalidOperationException($"Unknown message content match kind '{_kind}'.")
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,19 +1,19 @@
|
|||
using DiscordChatExporter.Core.Discord.Data;
|
||||
using System;
|
||||
using System;
|
||||
using System.Linq;
|
||||
using DiscordChatExporter.Core.Discord.Data;
|
||||
|
||||
namespace DiscordChatExporter.Core.Exporting.Filtering
|
||||
{
|
||||
public class MentionsMessageFilter : MessageFilter
|
||||
internal class MentionsMessageFilter : MessageFilter
|
||||
{
|
||||
private readonly string _value;
|
||||
|
||||
public MentionsMessageFilter(string value) => _value = value;
|
||||
|
||||
public override bool Filter(Message message) =>
|
||||
message.MentionedUsers.Any(user =>
|
||||
string.Equals(_value, user.Name, StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(_value, user.FullName, StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(_value, user.Id.ToString(), StringComparison.OrdinalIgnoreCase));
|
||||
public override bool Filter(Message message) => message.MentionedUsers.Any(user =>
|
||||
string.Equals(_value, user.Name, StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(_value, user.FullName, StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(_value, user.Id.ToString(), StringComparison.OrdinalIgnoreCase)
|
||||
);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,12 @@
|
|||
namespace DiscordChatExporter.Core.Exporting.Filtering
|
||||
{
|
||||
internal enum MessageContentMatchKind
|
||||
{
|
||||
Link,
|
||||
Embed,
|
||||
File,
|
||||
Video,
|
||||
Image,
|
||||
Sound
|
||||
}
|
||||
}
|
|
@ -1,6 +1,4 @@
|
|||
using System;
|
||||
using System.Text.RegularExpressions;
|
||||
using DiscordChatExporter.Core.Discord.Data;
|
||||
using DiscordChatExporter.Core.Discord.Data;
|
||||
using DiscordChatExporter.Core.Exporting.Filtering.Parsing;
|
||||
using Superpower;
|
||||
|
||||
|
@ -13,26 +11,8 @@ namespace DiscordChatExporter.Core.Exporting.Filtering
|
|||
|
||||
public partial class MessageFilter
|
||||
{
|
||||
protected const RegexOptions DefaultRegexOptions = RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.Multiline;
|
||||
public static MessageFilter Null { get; } = new NullMessageFilter();
|
||||
|
||||
internal static MessageFilter CreateFilter(string text) => new ContainsMessageFilter(text);
|
||||
|
||||
internal static MessageFilter CreateFilter(string key, string value)
|
||||
{
|
||||
return key.ToLowerInvariant() switch
|
||||
{
|
||||
"from" => new FromMessageFilter(value),
|
||||
"has" => new HasMessageFilter(value),
|
||||
"mentions" => new MentionsMessageFilter(value),
|
||||
_ => throw new ArgumentException($"Invalid filter type '{key}'.", nameof(key))
|
||||
};
|
||||
}
|
||||
|
||||
public static MessageFilter Parse(string value, IFormatProvider? formatProvider = null)
|
||||
{
|
||||
var tokens = FilterTokenizer.Instance.Tokenize(value);
|
||||
var parsed = FilterParser.Instance.Parse(tokens);
|
||||
return parsed;
|
||||
}
|
||||
public static MessageFilter Parse(string value) => FilterGrammar.Filter.Parse(value);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
namespace DiscordChatExporter.Core.Exporting.Filtering
|
||||
{
|
||||
public class NegatedMessageFilter : MessageFilter
|
||||
internal class NegatedMessageFilter : MessageFilter
|
||||
{
|
||||
private readonly MessageFilter _filter;
|
||||
|
||||
|
|
|
@ -2,10 +2,8 @@
|
|||
|
||||
namespace DiscordChatExporter.Core.Exporting.Filtering
|
||||
{
|
||||
public class NullMessageFilter : MessageFilter
|
||||
internal class NullMessageFilter : MessageFilter
|
||||
{
|
||||
public static NullMessageFilter Instance { get; } = new();
|
||||
|
||||
public override bool Filter(Message message) => true;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,106 @@
|
|||
using System.Linq;
|
||||
using DiscordChatExporter.Core.Utils.Extensions;
|
||||
using Superpower;
|
||||
using Superpower.Parsers;
|
||||
|
||||
namespace DiscordChatExporter.Core.Exporting.Filtering.Parsing
|
||||
{
|
||||
internal static class FilterGrammar
|
||||
{
|
||||
// Choice(a, b) looks cleaner than a.Or(b)
|
||||
private static TextParser<T> Choice<T>(params TextParser<T>[] parsers) =>
|
||||
parsers.Aggregate((current, next) => current.Or(next));
|
||||
|
||||
private static readonly TextParser<char> EscapedCharacter =
|
||||
Character.EqualTo('\\').IgnoreThen(Character.AnyChar);
|
||||
|
||||
private static readonly TextParser<string> QuotedString =
|
||||
from open in Character.In('"', '\'')
|
||||
from value in Choice(EscapedCharacter, Character.Except(open)).Many().Text()
|
||||
from close in Character.EqualTo(open)
|
||||
select value;
|
||||
|
||||
private static readonly TextParser<char> FreeCharacter =
|
||||
Character.Matching(c =>
|
||||
!char.IsWhiteSpace(c) &&
|
||||
// Avoid all special tokens used by the grammar
|
||||
c is not ('(' or ')' or '"' or '\'' or '-' or '|' or '&'),
|
||||
"any character except whitespace or `(`, `)`, `\"`, `'`, `-`, `|`, `&`"
|
||||
);
|
||||
|
||||
private static readonly TextParser<string> UnquotedString =
|
||||
Choice(EscapedCharacter, FreeCharacter).AtLeastOnce().Text();
|
||||
|
||||
private static readonly TextParser<string> String =
|
||||
Choice(QuotedString, UnquotedString).Named("text string");
|
||||
|
||||
private static readonly TextParser<MessageFilter> ContainsFilter =
|
||||
String.Select(v => (MessageFilter) new ContainsMessageFilter(v));
|
||||
|
||||
private static readonly TextParser<MessageFilter> FromFilter = Span
|
||||
.EqualToIgnoreCase("from:")
|
||||
.IgnoreThen(String)
|
||||
.Select(v => (MessageFilter) new FromMessageFilter(v))
|
||||
.Named("from:<value>");
|
||||
|
||||
private static readonly TextParser<MessageFilter> MentionsFilter = Span
|
||||
.EqualToIgnoreCase("mentions:")
|
||||
.IgnoreThen(String)
|
||||
.Select(v => (MessageFilter) new MentionsMessageFilter(v))
|
||||
.Named("mentions:<value>");
|
||||
|
||||
private static readonly TextParser<MessageFilter> HasFilter = Span
|
||||
.EqualToIgnoreCase("has:")
|
||||
.IgnoreThen(Choice(
|
||||
Span.EqualToIgnoreCase("link").IgnoreThen(Parse.Return(MessageContentMatchKind.Link)),
|
||||
Span.EqualToIgnoreCase("embed").IgnoreThen(Parse.Return(MessageContentMatchKind.Embed)),
|
||||
Span.EqualToIgnoreCase("video").IgnoreThen(Parse.Return(MessageContentMatchKind.Video)),
|
||||
Span.EqualToIgnoreCase("image").IgnoreThen(Parse.Return(MessageContentMatchKind.Image)),
|
||||
Span.EqualToIgnoreCase("sound").IgnoreThen(Parse.Return(MessageContentMatchKind.Sound))
|
||||
))
|
||||
.Select(k => (MessageFilter) new HasMessageFilter(k))
|
||||
.Named("has:<value>");
|
||||
|
||||
private static readonly TextParser<MessageFilter> NegatedFilter = Character
|
||||
.EqualTo('-')
|
||||
.IgnoreThen(Parse.Ref(() => StandaloneFilter))
|
||||
.Select(f => (MessageFilter) new NegatedMessageFilter(f));
|
||||
|
||||
private static readonly TextParser<MessageFilter> GroupedFilter =
|
||||
from open in Character.EqualTo('(')
|
||||
from content in Parse.Ref(() => BinaryExpressionFilter).Token()
|
||||
from close in Character.EqualTo(')')
|
||||
select content;
|
||||
|
||||
private static readonly TextParser<MessageFilter> StandaloneFilter = Choice(
|
||||
GroupedFilter,
|
||||
FromFilter,
|
||||
MentionsFilter,
|
||||
HasFilter,
|
||||
ContainsFilter
|
||||
);
|
||||
|
||||
private static readonly TextParser<MessageFilter> UnaryExpressionFilter = Choice(
|
||||
NegatedFilter,
|
||||
StandaloneFilter
|
||||
);
|
||||
|
||||
private static readonly TextParser<MessageFilter> BinaryExpressionFilter = Parse.Chain(
|
||||
Choice(
|
||||
// Explicit operator
|
||||
Character.In('|', '&').Token().Try(),
|
||||
// Implicit operator (resolves to 'and')
|
||||
Character.WhiteSpace.AtLeastOnce().IgnoreThen(Parse.Return(' '))
|
||||
),
|
||||
UnaryExpressionFilter,
|
||||
(op, left, right) => op switch
|
||||
{
|
||||
'|' => new BinaryExpressionMessageFilter(left, right, BinaryExpressionKind.Or),
|
||||
_ => new BinaryExpressionMessageFilter(left, right, BinaryExpressionKind.And)
|
||||
}
|
||||
);
|
||||
|
||||
public static readonly TextParser<MessageFilter> Filter =
|
||||
BinaryExpressionFilter.Token().AtEnd();
|
||||
}
|
||||
}
|
|
@ -1,68 +0,0 @@
|
|||
using Superpower;
|
||||
using Superpower.Model;
|
||||
using Superpower.Parsers;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
|
||||
namespace DiscordChatExporter.Core.Exporting.Filtering.Parsing
|
||||
{
|
||||
public static class FilterParser
|
||||
{
|
||||
public static TextParser<string> QuotedString { get; } =
|
||||
from open in Character.EqualTo('"')
|
||||
from content in Character.EqualTo('\\').IgnoreThen(Character.AnyChar).Try()
|
||||
.Or(Character.Except('"'))
|
||||
.Many()
|
||||
from close in Character.EqualTo('"')
|
||||
select new string(content);
|
||||
|
||||
public static TextParser<string> UnquotedString { get; } =
|
||||
from content in Character.EqualTo('\\').IgnoreThen(Character.In('"', '/')).Try()
|
||||
.Or(Character.Except(c => char.IsWhiteSpace(c) || "():-|\"".Contains(c), "non-whitespace character except for (, ), :, -, |, and \""))
|
||||
.AtLeastOnce()
|
||||
select new string(content);
|
||||
|
||||
public static TokenListParser<FilterToken, string> AnyString { get; } =
|
||||
Token.EqualTo(FilterToken.QuotedString).Apply(QuotedString)
|
||||
.Or(Token.EqualTo(FilterToken.UnquotedString).Apply(UnquotedString));
|
||||
|
||||
public static TokenListParser<FilterToken, MessageFilter> AnyFilter { get; } =
|
||||
from minus in Token.EqualTo(FilterToken.Minus).Optional()
|
||||
from content in KeyValueFilter.Or(TextFilter).Or(GroupedFilter)
|
||||
select minus.HasValue ? new NegatedMessageFilter(content) : content;
|
||||
|
||||
public static TokenListParser<FilterToken, MessageFilter> TextFilter { get; } =
|
||||
from value in AnyString
|
||||
select MessageFilter.CreateFilter(value);
|
||||
|
||||
public static TokenListParser<FilterToken, MessageFilter> KeyValueFilter { get; } =
|
||||
from key in AnyString.Try()
|
||||
from colon in Token.EqualTo(FilterToken.Colon).Try()
|
||||
from value in AnyString
|
||||
select MessageFilter.CreateFilter(key, value);
|
||||
|
||||
public static TokenListParser<FilterToken, MessageFilter> GroupedFilter { get; } =
|
||||
from open in Token.EqualTo(FilterToken.LParen)
|
||||
from content in BinaryExpression
|
||||
from close in Token.EqualTo(FilterToken.RParen)
|
||||
select content;
|
||||
|
||||
public static TokenListParser<FilterToken, MessageFilter> OrBinaryExpression { get; } =
|
||||
from first in AnyFilter
|
||||
from vbar in Token.EqualTo(FilterToken.VBar)
|
||||
from rest in BinaryExpression
|
||||
select (MessageFilter)new BinaryExpressionMessageFilter(first, rest, BinaryExpressionKind.Or);
|
||||
|
||||
public static TokenListParser<FilterToken, MessageFilter> AndBinaryExpression { get; } =
|
||||
from first in AnyFilter
|
||||
from rest in BinaryExpression
|
||||
select (MessageFilter)new BinaryExpressionMessageFilter(first, rest, BinaryExpressionKind.And);
|
||||
|
||||
public static TokenListParser<FilterToken, MessageFilter> BinaryExpression { get; } = OrBinaryExpression.Try().Or(AndBinaryExpression.Try()).Or(AnyFilter);
|
||||
|
||||
public static TokenListParser<FilterToken, MessageFilter> Instance { get; } = BinaryExpression.AtEnd();
|
||||
}
|
||||
}
|
|
@ -1,18 +0,0 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
namespace DiscordChatExporter.Core.Exporting.Filtering.Parsing
|
||||
{
|
||||
public enum FilterToken
|
||||
{
|
||||
None,
|
||||
LParen,
|
||||
RParen,
|
||||
Colon,
|
||||
Minus,
|
||||
VBar,
|
||||
UnquotedString,
|
||||
QuotedString
|
||||
}
|
||||
}
|
|
@ -1,23 +0,0 @@
|
|||
using Superpower;
|
||||
using Superpower.Parsers;
|
||||
using Superpower.Tokenizers;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
namespace DiscordChatExporter.Core.Exporting.Filtering.Parsing
|
||||
{
|
||||
public static class FilterTokenizer
|
||||
{
|
||||
public static Tokenizer<FilterToken> Instance { get; } = new TokenizerBuilder<FilterToken>()
|
||||
.Ignore(Span.WhiteSpace)
|
||||
.Match(Character.EqualTo('('), FilterToken.LParen)
|
||||
.Match(Character.EqualTo(')'), FilterToken.RParen)
|
||||
.Match(Character.EqualTo(':'), FilterToken.Colon)
|
||||
.Match(Character.EqualTo('-'), FilterToken.Minus)
|
||||
.Match(Character.EqualTo('|'), FilterToken.VBar)
|
||||
.Match(FilterParser.QuotedString, FilterToken.QuotedString)
|
||||
.Match(FilterParser.UnquotedString, FilterToken.UnquotedString)
|
||||
.Build();
|
||||
}
|
||||
}
|
|
@ -1,6 +1,6 @@
|
|||
namespace DiscordChatExporter.Core.Exporting.Partitioning
|
||||
{
|
||||
public class FileSizePartitionLimit : PartitionLimit
|
||||
internal class FileSizePartitionLimit : PartitionLimit
|
||||
{
|
||||
private readonly long _limit;
|
||||
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
namespace DiscordChatExporter.Core.Exporting.Partitioning
|
||||
{
|
||||
public class MessageCountPartitionLimit : PartitionLimit
|
||||
internal class MessageCountPartitionLimit : PartitionLimit
|
||||
{
|
||||
private readonly long _limit;
|
||||
|
||||
|
|
|
@ -1,9 +1,7 @@
|
|||
namespace DiscordChatExporter.Core.Exporting.Partitioning
|
||||
{
|
||||
public class NullPartitionLimit : PartitionLimit
|
||||
internal class NullPartitionLimit : PartitionLimit
|
||||
{
|
||||
public static NullPartitionLimit Instance { get; } = new();
|
||||
|
||||
public override bool IsReached(long messagesWritten, long bytesWritten) => false;
|
||||
}
|
||||
}
|
|
@ -11,6 +11,8 @@ namespace DiscordChatExporter.Core.Exporting.Partitioning
|
|||
|
||||
public partial class PartitionLimit
|
||||
{
|
||||
public static PartitionLimit Null { get; } = new NullPartitionLimit();
|
||||
|
||||
private static long? TryParseFileSizeBytes(string value, IFormatProvider? formatProvider = null)
|
||||
{
|
||||
var match = Regex.Match(value, @"^\s*(\d+[\.,]?\d*)\s*(\w)?b\s*$", RegexOptions.IgnoreCase);
|
||||
|
|
|
@ -6,7 +6,7 @@ using System.Text.RegularExpressions;
|
|||
using DiscordChatExporter.Core.Discord;
|
||||
using DiscordChatExporter.Core.Discord.Data;
|
||||
using DiscordChatExporter.Core.Markdown;
|
||||
using DiscordChatExporter.Core.Markdown.Ast;
|
||||
using DiscordChatExporter.Core.Markdown.Parsing;
|
||||
using DiscordChatExporter.Core.Utils.Extensions;
|
||||
|
||||
namespace DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors
|
||||
|
@ -78,14 +78,14 @@ namespace DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors
|
|||
protected override MarkdownNode VisitMention(MentionNode mention)
|
||||
{
|
||||
var mentionId = Snowflake.TryParse(mention.Id);
|
||||
if (mention.Type == MentionType.Meta)
|
||||
if (mention.Kind == MentionKind.Meta)
|
||||
{
|
||||
_buffer
|
||||
.Append("<span class=\"mention\">")
|
||||
.Append("@").Append(HtmlEncode(mention.Id))
|
||||
.Append("</span>");
|
||||
}
|
||||
else if (mention.Type == MentionType.User)
|
||||
else if (mention.Kind == MentionKind.User)
|
||||
{
|
||||
var member = mentionId?.Pipe(_context.TryGetMember);
|
||||
var fullName = member?.User.FullName ?? "Unknown";
|
||||
|
@ -96,7 +96,7 @@ namespace DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors
|
|||
.Append("@").Append(HtmlEncode(nick))
|
||||
.Append("</span>");
|
||||
}
|
||||
else if (mention.Type == MentionType.Channel)
|
||||
else if (mention.Kind == MentionKind.Channel)
|
||||
{
|
||||
var channel = mentionId?.Pipe(_context.TryGetChannel);
|
||||
var name = channel?.Name ?? "deleted-channel";
|
||||
|
@ -106,7 +106,7 @@ namespace DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors
|
|||
.Append("#").Append(HtmlEncode(name))
|
||||
.Append("</span>");
|
||||
}
|
||||
else if (mention.Type == MentionType.Role)
|
||||
else if (mention.Kind == MentionKind.Role)
|
||||
{
|
||||
var role = mentionId?.Pipe(_context.TryGetRole);
|
||||
var name = role?.Name ?? "deleted-role";
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
using System.Text;
|
||||
using DiscordChatExporter.Core.Discord;
|
||||
using DiscordChatExporter.Core.Markdown;
|
||||
using DiscordChatExporter.Core.Markdown.Ast;
|
||||
using DiscordChatExporter.Core.Markdown.Parsing;
|
||||
using DiscordChatExporter.Core.Utils.Extensions;
|
||||
|
||||
namespace DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors
|
||||
|
@ -26,25 +26,25 @@ namespace DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors
|
|||
protected override MarkdownNode VisitMention(MentionNode mention)
|
||||
{
|
||||
var mentionId = Snowflake.TryParse(mention.Id);
|
||||
if (mention.Type == MentionType.Meta)
|
||||
if (mention.Kind == MentionKind.Meta)
|
||||
{
|
||||
_buffer.Append($"@{mention.Id}");
|
||||
}
|
||||
else if (mention.Type == MentionType.User)
|
||||
else if (mention.Kind == MentionKind.User)
|
||||
{
|
||||
var member = mentionId?.Pipe(_context.TryGetMember);
|
||||
var name = member?.User.Name ?? "Unknown";
|
||||
|
||||
_buffer.Append($"@{name}");
|
||||
}
|
||||
else if (mention.Type == MentionType.Channel)
|
||||
else if (mention.Kind == MentionKind.Channel)
|
||||
{
|
||||
var channel = mentionId?.Pipe(_context.TryGetChannel);
|
||||
var name = channel?.Name ?? "deleted-channel";
|
||||
|
||||
_buffer.Append($"#{name}");
|
||||
}
|
||||
else if (mention.Type == MentionType.Role)
|
||||
else if (mention.Kind == MentionKind.Role)
|
||||
{
|
||||
var role = mentionId?.Pipe(_context.TryGetRole);
|
||||
var name = role?.Name ?? "deleted-role";
|
||||
|
|
|
@ -1,17 +0,0 @@
|
|||
namespace DiscordChatExporter.Core.Markdown.Ast
|
||||
{
|
||||
internal class MentionNode : MarkdownNode
|
||||
{
|
||||
public string Id { get; }
|
||||
|
||||
public MentionType Type { get; }
|
||||
|
||||
public MentionNode(string id, MentionType type)
|
||||
{
|
||||
Id = id;
|
||||
Type = type;
|
||||
}
|
||||
|
||||
public override string ToString() => $"<{Type} mention> {Id}";
|
||||
}
|
||||
}
|
|
@ -1,10 +0,0 @@
|
|||
namespace DiscordChatExporter.Core.Markdown.Ast
|
||||
{
|
||||
internal enum MentionType
|
||||
{
|
||||
Meta,
|
||||
User,
|
||||
Channel,
|
||||
Role
|
||||
}
|
||||
}
|
|
@ -1,6 +1,6 @@
|
|||
using DiscordChatExporter.Core.Utils;
|
||||
|
||||
namespace DiscordChatExporter.Core.Markdown.Ast
|
||||
namespace DiscordChatExporter.Core.Markdown
|
||||
{
|
||||
internal class EmojiNode : MarkdownNode
|
||||
{
|
|
@ -1,6 +1,6 @@
|
|||
using System.Collections.Generic;
|
||||
|
||||
namespace DiscordChatExporter.Core.Markdown.Ast
|
||||
namespace DiscordChatExporter.Core.Markdown
|
||||
{
|
||||
internal class FormattedNode : MarkdownNode
|
||||
{
|
|
@ -1,4 +1,4 @@
|
|||
namespace DiscordChatExporter.Core.Markdown.Ast
|
||||
namespace DiscordChatExporter.Core.Markdown
|
||||
{
|
||||
internal class InlineCodeBlockNode : MarkdownNode
|
||||
{
|
|
@ -1,4 +1,4 @@
|
|||
namespace DiscordChatExporter.Core.Markdown.Ast
|
||||
namespace DiscordChatExporter.Core.Markdown
|
||||
{
|
||||
internal class LinkNode : MarkdownNode
|
||||
{
|
|
@ -1,4 +1,4 @@
|
|||
namespace DiscordChatExporter.Core.Markdown.Ast
|
||||
namespace DiscordChatExporter.Core.Markdown
|
||||
{
|
||||
internal abstract class MarkdownNode
|
||||
{
|
10
DiscordChatExporter.Core/Markdown/MentionKind.cs
Normal file
10
DiscordChatExporter.Core/Markdown/MentionKind.cs
Normal file
|
@ -0,0 +1,10 @@
|
|||
namespace DiscordChatExporter.Core.Markdown
|
||||
{
|
||||
internal enum MentionKind
|
||||
{
|
||||
Meta,
|
||||
User,
|
||||
Channel,
|
||||
Role
|
||||
}
|
||||
}
|
17
DiscordChatExporter.Core/Markdown/MentionNode.cs
Normal file
17
DiscordChatExporter.Core/Markdown/MentionNode.cs
Normal file
|
@ -0,0 +1,17 @@
|
|||
namespace DiscordChatExporter.Core.Markdown
|
||||
{
|
||||
internal class MentionNode : MarkdownNode
|
||||
{
|
||||
public string Id { get; }
|
||||
|
||||
public MentionKind Kind { get; }
|
||||
|
||||
public MentionNode(string id, MentionKind kind)
|
||||
{
|
||||
Id = id;
|
||||
Kind = kind;
|
||||
}
|
||||
|
||||
public override string ToString() => $"<{Kind} mention> {Id}";
|
||||
}
|
||||
}
|
|
@ -1,4 +1,4 @@
|
|||
namespace DiscordChatExporter.Core.Markdown.Ast
|
||||
namespace DiscordChatExporter.Core.Markdown
|
||||
{
|
||||
internal class MultiLineCodeBlockNode : MarkdownNode
|
||||
{
|
|
@ -1,6 +1,6 @@
|
|||
using System.Collections.Generic;
|
||||
|
||||
namespace DiscordChatExporter.Core.Markdown.Matching
|
||||
namespace DiscordChatExporter.Core.Markdown.Parsing
|
||||
{
|
||||
internal class AggregateMatcher<T> : IMatcher<T>
|
||||
{
|
|
@ -1,7 +1,7 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace DiscordChatExporter.Core.Markdown.Matching
|
||||
namespace DiscordChatExporter.Core.Markdown.Parsing
|
||||
{
|
||||
internal interface IMatcher<T>
|
||||
{
|
|
@ -1,11 +1,9 @@
|
|||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using DiscordChatExporter.Core.Markdown.Ast;
|
||||
using DiscordChatExporter.Core.Markdown.Matching;
|
||||
using DiscordChatExporter.Core.Utils;
|
||||
|
||||
namespace DiscordChatExporter.Core.Markdown
|
||||
namespace DiscordChatExporter.Core.Markdown.Parsing
|
||||
{
|
||||
// The following parsing logic is meant to replicate Discord's markdown grammar as close as possible
|
||||
internal static partial class MarkdownParser
|
||||
|
@ -120,31 +118,31 @@ namespace DiscordChatExporter.Core.Markdown
|
|||
// Capture @everyone
|
||||
private static readonly IMatcher<MarkdownNode> EveryoneMentionNodeMatcher = new StringMatcher<MarkdownNode>(
|
||||
"@everyone",
|
||||
_ => new MentionNode("everyone", MentionType.Meta)
|
||||
_ => new MentionNode("everyone", MentionKind.Meta)
|
||||
);
|
||||
|
||||
// Capture @here
|
||||
private static readonly IMatcher<MarkdownNode> HereMentionNodeMatcher = new StringMatcher<MarkdownNode>(
|
||||
"@here",
|
||||
_ => new MentionNode("here", MentionType.Meta)
|
||||
_ => new MentionNode("here", MentionKind.Meta)
|
||||
);
|
||||
|
||||
// Capture <@123456> or <@!123456>
|
||||
private static readonly IMatcher<MarkdownNode> UserMentionNodeMatcher = new RegexMatcher<MarkdownNode>(
|
||||
new Regex("<@!?(\\d+)>", DefaultRegexOptions),
|
||||
(_, m) => new MentionNode(m.Groups[1].Value, MentionType.User)
|
||||
(_, m) => new MentionNode(m.Groups[1].Value, MentionKind.User)
|
||||
);
|
||||
|
||||
// Capture <#123456>
|
||||
private static readonly IMatcher<MarkdownNode> ChannelMentionNodeMatcher = new RegexMatcher<MarkdownNode>(
|
||||
new Regex("<#(\\d+)>", DefaultRegexOptions),
|
||||
(_, m) => new MentionNode(m.Groups[1].Value, MentionType.Channel)
|
||||
(_, m) => new MentionNode(m.Groups[1].Value, MentionKind.Channel)
|
||||
);
|
||||
|
||||
// Capture <@&123456>
|
||||
private static readonly IMatcher<MarkdownNode> RoleMentionNodeMatcher = new RegexMatcher<MarkdownNode>(
|
||||
new Regex("<@&(\\d+)>", DefaultRegexOptions),
|
||||
(_, m) => new MentionNode(m.Groups[1].Value, MentionType.Role)
|
||||
(_, m) => new MentionNode(m.Groups[1].Value, MentionKind.Role)
|
||||
);
|
||||
|
||||
/* Emojis */
|
||||
|
@ -293,12 +291,16 @@ namespace DiscordChatExporter.Core.Markdown
|
|||
|
||||
internal static partial class MarkdownParser
|
||||
{
|
||||
private static IReadOnlyList<MarkdownNode> Parse(StringPart stringPart) => Parse(stringPart, AggregateNodeMatcher);
|
||||
private static IReadOnlyList<MarkdownNode> Parse(StringPart stringPart) =>
|
||||
Parse(stringPart, AggregateNodeMatcher);
|
||||
|
||||
private static IReadOnlyList<MarkdownNode> ParseMinimal(StringPart stringPart) => Parse(stringPart, MinimalAggregateNodeMatcher);
|
||||
private static IReadOnlyList<MarkdownNode> ParseMinimal(StringPart stringPart) =>
|
||||
Parse(stringPart, MinimalAggregateNodeMatcher);
|
||||
|
||||
public static IReadOnlyList<MarkdownNode> Parse(string input) => Parse(new StringPart(input));
|
||||
public static IReadOnlyList<MarkdownNode> Parse(string input) =>
|
||||
Parse(new StringPart(input));
|
||||
|
||||
public static IReadOnlyList<MarkdownNode> ParseMinimal(string input) => ParseMinimal(new StringPart(input));
|
||||
public static IReadOnlyList<MarkdownNode> ParseMinimal(string input) =>
|
||||
ParseMinimal(new StringPart(input));
|
||||
}
|
||||
}
|
|
@ -1,8 +1,7 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using DiscordChatExporter.Core.Markdown.Ast;
|
||||
|
||||
namespace DiscordChatExporter.Core.Markdown
|
||||
namespace DiscordChatExporter.Core.Markdown.Parsing
|
||||
{
|
||||
internal abstract class MarkdownVisitor
|
||||
{
|
|
@ -1,4 +1,4 @@
|
|||
namespace DiscordChatExporter.Core.Markdown.Matching
|
||||
namespace DiscordChatExporter.Core.Markdown.Parsing
|
||||
{
|
||||
internal class ParsedMatch<T>
|
||||
{
|
|
@ -1,7 +1,7 @@
|
|||
using System;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace DiscordChatExporter.Core.Markdown.Matching
|
||||
namespace DiscordChatExporter.Core.Markdown.Parsing
|
||||
{
|
||||
internal class RegexMatcher<T> : IMatcher<T>
|
||||
{
|
|
@ -1,6 +1,6 @@
|
|||
using System;
|
||||
|
||||
namespace DiscordChatExporter.Core.Markdown.Matching
|
||||
namespace DiscordChatExporter.Core.Markdown.Parsing
|
||||
{
|
||||
internal class StringMatcher<T> : IMatcher<T>
|
||||
{
|
|
@ -1,6 +1,6 @@
|
|||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace DiscordChatExporter.Core.Markdown.Matching
|
||||
namespace DiscordChatExporter.Core.Markdown.Parsing
|
||||
{
|
||||
internal readonly struct StringPart
|
||||
{
|
|
@ -1,4 +1,4 @@
|
|||
namespace DiscordChatExporter.Core.Markdown.Ast
|
||||
namespace DiscordChatExporter.Core.Markdown
|
||||
{
|
||||
internal enum TextFormatting
|
||||
{
|
|
@ -1,4 +1,4 @@
|
|||
namespace DiscordChatExporter.Core.Markdown.Ast
|
||||
namespace DiscordChatExporter.Core.Markdown
|
||||
{
|
||||
internal class TextNode : MarkdownNode
|
||||
{
|
|
@ -0,0 +1,24 @@
|
|||
using System;
|
||||
using Superpower;
|
||||
using Superpower.Parsers;
|
||||
|
||||
namespace DiscordChatExporter.Core.Utils.Extensions
|
||||
{
|
||||
public static class SuperpowerExtensions
|
||||
{
|
||||
public static TextParser<string> Text(this TextParser<char[]> parser) =>
|
||||
parser.Select(chars => new string(chars));
|
||||
|
||||
public static TextParser<T> Token<T>(this TextParser<T> parser) =>
|
||||
parser.Between(Character.WhiteSpace.IgnoreMany(), Character.WhiteSpace.IgnoreMany());
|
||||
|
||||
// From: https://twitter.com/nblumhardt/status/1389349059786264578
|
||||
public static TextParser<T> Log<T>(this TextParser<T> parser, string description) => i =>
|
||||
{
|
||||
Console.WriteLine($"Trying {description} ->");
|
||||
var r = parser(i);
|
||||
Console.WriteLine($"Result was {r}");
|
||||
return r;
|
||||
};
|
||||
}
|
||||
}
|
|
@ -21,14 +21,15 @@ namespace DiscordChatExporter.Core.Utils
|
|||
.OrResult<HttpResponseMessage>(m => m.StatusCode == HttpStatusCode.TooManyRequests)
|
||||
.OrResult(m => m.StatusCode == HttpStatusCode.RequestTimeout)
|
||||
.OrResult(m => m.StatusCode >= HttpStatusCode.InternalServerError)
|
||||
.WaitAndRetryAsync(8,
|
||||
.WaitAndRetryAsync(
|
||||
8,
|
||||
(i, result, _) =>
|
||||
{
|
||||
// If rate-limited, use retry-after as a guide
|
||||
if (result.Result?.StatusCode == HttpStatusCode.TooManyRequests)
|
||||
{
|
||||
// Only start respecting retry-after after a few attempts.
|
||||
// The reason is that Discord often sends unreasonable (20+ minutes) retry-after
|
||||
// Only start respecting retry-after after a few attempts, because
|
||||
// Discord often sends unreasonable (20+ minutes) retry-after
|
||||
// on the very first request.
|
||||
if (i > 3)
|
||||
{
|
||||
|
@ -40,7 +41,8 @@ namespace DiscordChatExporter.Core.Utils
|
|||
|
||||
return TimeSpan.FromSeconds(Math.Pow(2, i) + 1);
|
||||
},
|
||||
(_, _, _, _) => Task.CompletedTask);
|
||||
(_, _, _, _) => Task.CompletedTask
|
||||
);
|
||||
|
||||
private static HttpStatusCode? TryGetStatusCodeFromException(HttpRequestException ex) =>
|
||||
// This is extremely frail, but there's no other way
|
||||
|
|
|
@ -52,13 +52,13 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs
|
|||
|
||||
public PartitionLimit PartitionLimit => !string.IsNullOrWhiteSpace(PartitionLimitValue)
|
||||
? PartitionLimit.Parse(PartitionLimitValue)
|
||||
: NullPartitionLimit.Instance;
|
||||
: PartitionLimit.Null;
|
||||
|
||||
public string? MessageFilterValue { get; set; }
|
||||
|
||||
public MessageFilter MessageFilter => !string.IsNullOrWhiteSpace(MessageFilterValue)
|
||||
? MessageFilter.Parse(MessageFilterValue)
|
||||
: NullMessageFilter.Instance;
|
||||
: MessageFilter.Null;
|
||||
|
||||
public bool ShouldDownloadMedia { get; set; }
|
||||
|
||||
|
|
|
@ -131,7 +131,7 @@
|
|||
materialDesign:HintAssist.Hint="Partition limit"
|
||||
materialDesign:HintAssist.IsFloating="True"
|
||||
Text="{Binding PartitionLimitValue}"
|
||||
ToolTip="Split output into partitions, each limited to this number of messages (e.g. 100) or file size (e.g. 10mb)" />
|
||||
ToolTip="Split output into partitions, each limited to this number of messages (e.g. '100') or file size (e.g. '10mb')" />
|
||||
|
||||
<!-- Filtering -->
|
||||
<TextBox
|
||||
|
@ -139,7 +139,7 @@
|
|||
materialDesign:HintAssist.Hint="Message filter"
|
||||
materialDesign:HintAssist.IsFloating="True"
|
||||
Text="{Binding MessageFilterValue}"
|
||||
ToolTip="Only include messages that satisfy this filter (e.g. from:foo#1234)." />
|
||||
ToolTip="Only include messages that satisfy this filter (e.g. 'from:foo#1234' or 'has:image')." />
|
||||
|
||||
<!-- Download media -->
|
||||
<Grid Margin="16,16" ToolTip="Download referenced media content (user avatars, attached files, embedded images, etc)">
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
<Window
|
||||
<Window
|
||||
x:Class="DiscordChatExporter.Gui.Views.RootView"
|
||||
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
|
||||
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue