From 1fadc0755b6f6d6a8b79bf29515253cdfccff173 Mon Sep 17 00:00:00 2001 From: Tyrrrz <1935960+Tyrrrz@users.noreply.github.com> Date: Mon, 21 Apr 2025 00:45:26 +0300 Subject: [PATCH] Refactor after last changes --- .../Discord/DiscordClient.cs | 168 ++++++------------ 1 file changed, 54 insertions(+), 114 deletions(-) diff --git a/DiscordChatExporter.Core/Discord/DiscordClient.cs b/DiscordChatExporter.Core/Discord/DiscordClient.cs index 8dd3da53..67cf18ec 100644 --- a/DiscordChatExporter.Core/Discord/DiscordClient.cs +++ b/DiscordChatExporter.Core/Discord/DiscordClient.cs @@ -148,14 +148,14 @@ public class DiscordClient(string token) _ => throw new DiscordChatExporterException( $""" - Request to '{url}' failed: {response - .StatusCode.ToString() - .ToSpaceSeparatedWords() - .ToLowerInvariant()}. - Response content: {await response.Content.ReadAsStringAsync( - cancellationToken - )} - """, + Request to '{url}' failed: {response + .StatusCode.ToString() + .ToSpaceSeparatedWords() + .ToLowerInvariant()}. + Response content: {await response.Content.ReadAsStringAsync( + cancellationToken + )} + """, true ), }; @@ -311,58 +311,14 @@ public class DiscordClient(string token) // User accounts can only fetch threads using the search endpoint if (await ResolveTokenKindAsync(cancellationToken) == TokenKind.User) { - // Active threads foreach (var channel in channels) { - var currentOffset = 0; - while (true) - { - var url = new UrlBuilder() - .SetPath($"channels/{channel.Id}/threads/search") - .SetQueryParameter("sort_by", "last_message_time") - .SetQueryParameter("sort_order", "desc") - .SetQueryParameter("archived", "false") - .SetQueryParameter("offset", currentOffset.ToString()) - .Build(); - - // Can be null on channels that the user cannot access or channels without threads - var response = await TryGetJsonResponseAsync(url, cancellationToken); - if (response is null) - break; - - var breakOuter = false; - - foreach ( - var threadJson in response.Value.GetProperty("threads").EnumerateArray() - ) - { - var thread = Channel.Parse(threadJson, channel); - - // If the 'after' boundary is specified, we can break early, - // because threads are sorted by last message time. - if (after is not null && !thread.MayHaveMessagesAfter(after.Value)) - { - breakOuter = true; - break; - } - - yield return thread; - currentOffset++; - } - - if (breakOuter) - break; - - if (!response.Value.GetProperty("has_more").GetBoolean()) - break; - } - } - - // Archived threads - if (includeArchived) - { - foreach (var channel in channels) + // Either include both active and archived threads, or only active threads + foreach ( + var isArchived in includeArchived ? new[] { false, true } : new[] { false } + ) { + // Offset is just the index of the last thread in the previous batch var currentOffset = 0; while (true) { @@ -370,7 +326,7 @@ public class DiscordClient(string token) .SetPath($"channels/{channel.Id}/threads/search") .SetQueryParameter("sort_by", "last_message_time") .SetQueryParameter("sort_order", "desc") - .SetQueryParameter("archived", "true") + .SetQueryParameter("archived", isArchived.ToString().ToLowerInvariant()) .SetQueryParameter("offset", currentOffset.ToString()) .Build(); @@ -388,7 +344,7 @@ public class DiscordClient(string token) var thread = Channel.Parse(threadJson, channel); // If the 'after' boundary is specified, we can break early, - // because threads are sorted by last message time. + // because threads are sorted by last message timestamp. if (after is not null && !thread.MayHaveMessagesAfter(after.Value)) { breakOuter = true; @@ -437,64 +393,50 @@ public class DiscordClient(string token) { foreach (var channel in channels) { - // Public archived threads - await foreach ( - var th in GetAllArchivedThreadsAsync(channel, "public", cancellationToken) - ) - yield return th; + foreach (var archiveType in new[] { "public", "private" }) + { + // This endpoint parameter expects an ISO8601 timestamp, not a snowflake + var currentBefore = before + ?.ToDate() + .ToString("O", CultureInfo.InvariantCulture); - // Private archived threads - await foreach ( - var th in GetAllArchivedThreadsAsync(channel, "private", cancellationToken) - ) - yield return th; + while (true) + { + // Threads are sorted by archive timestamp, not by last message timestamp + var url = new UrlBuilder() + .SetPath($"channels/{channel.Id}/threads/archived/{archiveType}") + .SetQueryParameter("before", currentBefore) + .Build(); + + // Can be null on certain channels + var response = await TryGetJsonResponseAsync(url, cancellationToken); + if (response is null) + break; + + foreach ( + var threadJson in response + .Value.GetProperty("threads") + .EnumerateArray() + ) + { + var thread = Channel.Parse(threadJson, channel); + yield return thread; + + currentBefore = threadJson + .GetProperty("thread_metadata") + .GetProperty("archive_timestamp") + .GetString(); + } + + if (!response.Value.GetProperty("has_more").GetBoolean()) + break; + } + } } } } } - private async IAsyncEnumerable GetAllArchivedThreadsAsync( - Channel channel, - string archiveType, - [EnumeratorCancellation] CancellationToken cancellationToken - ) - { - // Base endpoint: "public" or "private" - var endpointBase = $"channels/{channel.Id}/threads/archived/{archiveType}"; - // Cursor parameter: ISO8601 timestamp string - string? beforeTimestamp = null; - bool hasMorePages = true; - - while (hasMorePages && !cancellationToken.IsCancellationRequested) - { - // Build URL with optional before= parameter - var url = beforeTimestamp is null - ? endpointBase - : $"{endpointBase}?before={Uri.EscapeDataString(beforeTimestamp)}"; - - var response = await TryGetJsonResponseAsync(url, cancellationToken); - if (response is null) - yield break; - - // Parse out the threads array - var threadsJson = response.Value.GetProperty("threads").EnumerateArray().ToList(); - foreach (var threadJson in threadsJson) - { - yield return Channel.Parse(threadJson, channel); - } - - // Check pagination flag - hasMorePages = response.Value.GetProperty("has_more").GetBoolean(); - - if (hasMorePages && threadsJson.Count > 0) - { - // Prepare next cursor: the archived timestamp of the last thread - var lastThreadMeta = threadsJson.Last().GetProperty("thread_metadata"); - beforeTimestamp = lastThreadMeta.GetProperty("archive_timestamp").GetString(); - } - } - } - public async IAsyncEnumerable GetGuildRolesAsync( Snowflake guildId, [EnumeratorCancellation] CancellationToken cancellationToken = default @@ -708,9 +650,7 @@ public class DiscordClient(string token) count++; } - // Each batch can contain up to 100 users. - // If we got fewer, then it's definitely the last batch. - if (count < 100) + if (count <= 0) yield break; } }