mirror of
https://github.com/Tyrrrz/DiscordChatExporter.git
synced 2025-05-29 05:55:21 -04:00
Add support for extracting channels from data dump in exportall
Closes #597
This commit is contained in:
parent
83e3289ead
commit
0e1c3e4c76
11 changed files with 99 additions and 32 deletions
|
@ -7,7 +7,7 @@ using DiscordChatExporter.Core.Discord;
|
|||
|
||||
namespace DiscordChatExporter.Cli.Commands.Base;
|
||||
|
||||
public abstract class TokenCommandBase : ICommand
|
||||
public abstract class DiscordCommandBase : ICommand
|
||||
{
|
||||
[CommandOption(
|
||||
"token",
|
|
@ -20,7 +20,7 @@ using Gress;
|
|||
|
||||
namespace DiscordChatExporter.Cli.Commands.Base;
|
||||
|
||||
public abstract class ExportCommandBase : TokenCommandBase
|
||||
public abstract class ExportCommandBase : DiscordCommandBase
|
||||
{
|
||||
private readonly string _outputPath = Directory.GetCurrentDirectory();
|
||||
|
||||
|
@ -268,7 +268,7 @@ public abstract class ExportCommandBase : TokenCommandBase
|
|||
await console.Output.WriteLineAsync("Resolving channel(s)...");
|
||||
|
||||
var channels = new List<Channel>();
|
||||
var guildChannelMap = new Dictionary<Snowflake, IReadOnlyList<Channel>>();
|
||||
var channelsByGuild = new Dictionary<Snowflake, IReadOnlyList<Channel>>();
|
||||
|
||||
foreach (var channelId in channelIds)
|
||||
{
|
||||
|
@ -278,7 +278,7 @@ public abstract class ExportCommandBase : TokenCommandBase
|
|||
if (channel.Kind == ChannelKind.GuildCategory)
|
||||
{
|
||||
var guildChannels =
|
||||
guildChannelMap.GetValueOrDefault(channel.GuildId) ??
|
||||
channelsByGuild.GetValueOrDefault(channel.GuildId) ??
|
||||
await Discord.GetGuildChannelsAsync(channel.GuildId, cancellationToken);
|
||||
|
||||
foreach (var guildChannel in guildChannels)
|
||||
|
@ -288,7 +288,7 @@ public abstract class ExportCommandBase : TokenCommandBase
|
|||
}
|
||||
|
||||
// Cache the guild channels to avoid redundant work
|
||||
guildChannelMap[channel.GuildId] = guildChannels;
|
||||
channelsByGuild[channel.GuildId] = guildChannels;
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
|
@ -1,9 +1,15 @@
|
|||
using System.Collections.Generic;
|
||||
using System.IO.Compression;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
using CliFx.Attributes;
|
||||
using CliFx.Exceptions;
|
||||
using CliFx.Infrastructure;
|
||||
using DiscordChatExporter.Cli.Commands.Base;
|
||||
using DiscordChatExporter.Core.Discord;
|
||||
using DiscordChatExporter.Core.Discord.Data;
|
||||
using DiscordChatExporter.Core.Exceptions;
|
||||
using JsonExtensions.Reading;
|
||||
|
||||
namespace DiscordChatExporter.Cli.Commands;
|
||||
|
||||
|
@ -14,7 +20,19 @@ public class ExportAllCommand : ExportCommandBase
|
|||
"include-dm",
|
||||
Description = "Include direct message channels."
|
||||
)]
|
||||
public bool IncludeDirectMessages { get; init; } = true;
|
||||
public bool IncludeDirectChannels { get; init; } = true;
|
||||
|
||||
[CommandOption(
|
||||
"include-guilds",
|
||||
Description = "Include guild channels."
|
||||
)]
|
||||
public bool IncludeGuildChannels { get; init; } = true;
|
||||
|
||||
[CommandOption(
|
||||
"data-package",
|
||||
Description = "Path to the personal data package (ZIP file) requested from Discord. If provided, only channels referenced in the dump will be exported."
|
||||
)]
|
||||
public string? DataPackageFilePath { get; init; }
|
||||
|
||||
public override async ValueTask ExecuteAsync(IConsole console)
|
||||
{
|
||||
|
@ -23,16 +41,60 @@ public class ExportAllCommand : ExportCommandBase
|
|||
var cancellationToken = console.RegisterCancellationHandler();
|
||||
var channels = new List<Channel>();
|
||||
|
||||
await console.Output.WriteLineAsync("Fetching channels...");
|
||||
await foreach (var guild in Discord.GetUserGuildsAsync(cancellationToken))
|
||||
// Pull from the API
|
||||
if (string.IsNullOrWhiteSpace(DataPackageFilePath))
|
||||
{
|
||||
// Skip DMs if instructed to
|
||||
if (!IncludeDirectMessages && guild.Id == Guild.DirectMessages.Id)
|
||||
continue;
|
||||
await console.Output.WriteLineAsync("Fetching channels...");
|
||||
|
||||
await foreach (var channel in Discord.GetGuildChannelsAsync(guild.Id, cancellationToken))
|
||||
channels.Add(channel);
|
||||
await foreach (var guild in Discord.GetUserGuildsAsync(cancellationToken))
|
||||
{
|
||||
await foreach (var channel in Discord.GetGuildChannelsAsync(guild.Id, cancellationToken))
|
||||
{
|
||||
channels.Add(channel);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Pull from the data package
|
||||
else
|
||||
{
|
||||
await console.Output.WriteLineAsync("Extracting channels...");
|
||||
using var archive = ZipFile.OpenRead(DataPackageFilePath);
|
||||
|
||||
var entry = archive.GetEntry("messages/index.json");
|
||||
if (entry is null)
|
||||
throw new CommandException("Cannot find channel index inside the data package.");
|
||||
|
||||
await using var stream = entry.Open();
|
||||
using var document = await JsonDocument.ParseAsync(stream, default, cancellationToken);
|
||||
|
||||
foreach (var property in document.RootElement.EnumerateObjectOrEmpty())
|
||||
{
|
||||
var channelId = Snowflake.Parse(property.Name);
|
||||
var channelName = property.Value.GetString();
|
||||
|
||||
// Null items refer to deleted channels
|
||||
if (channelName is null)
|
||||
continue;
|
||||
|
||||
await console.Output.WriteLineAsync($"Fetching channel '{channelName}' ({channelId})...");
|
||||
|
||||
try
|
||||
{
|
||||
var channel = await Discord.GetChannelAsync(channelId, cancellationToken);
|
||||
channels.Add(channel);
|
||||
}
|
||||
catch (DiscordChatExporterException)
|
||||
{
|
||||
await console.Output.WriteLineAsync($"Channel '{channelName}' ({channelId}) is inaccessible.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Filter out unwanted channels
|
||||
if (!IncludeDirectChannels)
|
||||
channels.RemoveAll(c => c.Kind.IsDirect());
|
||||
if (!IncludeGuildChannels)
|
||||
channels.RemoveAll(c => c.Kind.IsGuild());
|
||||
|
||||
await base.ExecuteAsync(console, channels);
|
||||
}
|
||||
|
|
|
@ -14,7 +14,7 @@ public class ExportChannelsCommand : ExportCommandBase
|
|||
[CommandOption(
|
||||
"channel",
|
||||
'c',
|
||||
Description = "Channel ID(s). If provided with a category ID, all channels in that category will be exported."
|
||||
Description = "Channel ID(s). If provided with category IDs, all channels inside those categories will be exported."
|
||||
)]
|
||||
public required IReadOnlyList<Snowflake> ChannelIds { get; init; }
|
||||
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using System.Threading.Tasks;
|
||||
using CliFx.Attributes;
|
||||
using CliFx.Infrastructure;
|
||||
using DiscordChatExporter.Cli.Commands.Base;
|
||||
|
@ -18,10 +17,7 @@ public class ExportDirectMessagesCommand : ExportCommandBase
|
|||
var cancellationToken = console.RegisterCancellationHandler();
|
||||
|
||||
await console.Output.WriteLineAsync("Fetching channels...");
|
||||
|
||||
var channels = (await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken))
|
||||
.Where(c => c.Kind != ChannelKind.GuildCategory)
|
||||
.ToArray();
|
||||
var channels = await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken);
|
||||
|
||||
await base.ExecuteAsync(console, channels);
|
||||
}
|
||||
|
|
|
@ -11,7 +11,7 @@ using DiscordChatExporter.Core.Utils.Extensions;
|
|||
namespace DiscordChatExporter.Cli.Commands;
|
||||
|
||||
[Command("channels", Description = "Get the list of channels in a guild.")]
|
||||
public class GetChannelsCommand : TokenCommandBase
|
||||
public class GetChannelsCommand : DiscordCommandBase
|
||||
{
|
||||
[CommandOption(
|
||||
"guild",
|
||||
|
|
|
@ -10,7 +10,7 @@ using DiscordChatExporter.Core.Utils.Extensions;
|
|||
namespace DiscordChatExporter.Cli.Commands;
|
||||
|
||||
[Command("dm", Description = "Get the list of direct message channels.")]
|
||||
public class GetDirectMessageChannelsCommand : TokenCommandBase
|
||||
public class GetDirectChannelsCommand : DiscordCommandBase
|
||||
{
|
||||
public override async ValueTask ExecuteAsync(IConsole console)
|
||||
{
|
|
@ -10,7 +10,7 @@ using DiscordChatExporter.Core.Utils.Extensions;
|
|||
namespace DiscordChatExporter.Cli.Commands;
|
||||
|
||||
[Command("guilds", Description = "Get the list of accessible guilds.")]
|
||||
public class GetGuildsCommand : TokenCommandBase
|
||||
public class GetGuildsCommand : DiscordCommandBase
|
||||
{
|
||||
public override async ValueTask ExecuteAsync(IConsole console)
|
||||
{
|
||||
|
|
|
@ -16,4 +16,13 @@ public enum ChannelKind
|
|||
GuildStageVoice = 13,
|
||||
GuildDirectory = 14,
|
||||
GuildForum = 15
|
||||
}
|
||||
|
||||
public static class ChannelKindExtensions
|
||||
{
|
||||
public static bool IsDirect(this ChannelKind kind) =>
|
||||
kind is ChannelKind.DirectTextChat or ChannelKind.DirectGroupTextChat;
|
||||
|
||||
public static bool IsGuild(this ChannelKind kind) =>
|
||||
!kind.IsDirect();
|
||||
}
|
|
@ -38,16 +38,16 @@ public class DashboardViewModel : PropertyChangedBase
|
|||
|
||||
public string? Token { get; set; }
|
||||
|
||||
private IReadOnlyDictionary<Guild, IReadOnlyList<Channel>>? GuildChannelMap { get; set; }
|
||||
private IReadOnlyDictionary<Guild, IReadOnlyList<Channel>>? ChannelsByGuild { get; set; }
|
||||
|
||||
public IReadOnlyList<Guild>? AvailableGuilds => GuildChannelMap?.Keys.ToArray();
|
||||
public IReadOnlyList<Guild>? AvailableGuilds => ChannelsByGuild?.Keys.ToArray();
|
||||
|
||||
public Guild? SelectedGuild { get; set; }
|
||||
|
||||
public bool IsDirectMessageGuildSelected => SelectedGuild?.Id == Guild.DirectMessages.Id;
|
||||
|
||||
public IReadOnlyList<Channel>? AvailableChannels => SelectedGuild is not null
|
||||
? GuildChannelMap?[SelectedGuild]
|
||||
? ChannelsByGuild?[SelectedGuild]
|
||||
: null;
|
||||
|
||||
public IReadOnlyList<Channel>? SelectedChannels { get; set; }
|
||||
|
@ -103,17 +103,17 @@ public class DashboardViewModel : PropertyChangedBase
|
|||
|
||||
var discord = new DiscordClient(token);
|
||||
|
||||
var guildChannelMap = new Dictionary<Guild, IReadOnlyList<Channel>>();
|
||||
var channelsByGuild = new Dictionary<Guild, IReadOnlyList<Channel>>();
|
||||
await foreach (var guild in discord.GetUserGuildsAsync())
|
||||
{
|
||||
guildChannelMap[guild] = (await discord.GetGuildChannelsAsync(guild.Id))
|
||||
channelsByGuild[guild] = (await discord.GetGuildChannelsAsync(guild.Id))
|
||||
.Where(c => c.Kind != ChannelKind.GuildCategory)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
_discord = discord;
|
||||
GuildChannelMap = guildChannelMap;
|
||||
SelectedGuild = guildChannelMap.Keys.FirstOrDefault();
|
||||
ChannelsByGuild = channelsByGuild;
|
||||
SelectedGuild = channelsByGuild.Keys.FirstOrDefault();
|
||||
}
|
||||
catch (DiscordChatExporterException ex) when (!ex.IsFatal)
|
||||
{
|
||||
|
|
|
@ -19,7 +19,7 @@
|
|||
mc:Ignorable="d">
|
||||
<UserControl.Resources>
|
||||
<!-- Sort DMs by last message -->
|
||||
<CollectionViewSource x:Key="AvailableDirectMessageChannelsViewSource" Source="{Binding AvailableChannels, Mode=OneWay}">
|
||||
<CollectionViewSource x:Key="AvailableDirectChannelsViewSource" Source="{Binding AvailableChannels, Mode=OneWay}">
|
||||
<CollectionViewSource.SortDescriptions>
|
||||
<componentModel:SortDescription Direction="Descending" PropertyName="LastMessageId" />
|
||||
<componentModel:SortDescription Direction="Ascending" PropertyName="Name" />
|
||||
|
@ -300,7 +300,7 @@
|
|||
<Style BasedOn="{StaticResource {x:Type ListBox}}" TargetType="{x:Type ListBox}">
|
||||
<Style.Triggers>
|
||||
<DataTrigger Binding="{Binding IsDirectMessageGuildSelected}" Value="True">
|
||||
<Setter Property="ItemsSource" Value="{Binding Source={StaticResource AvailableDirectMessageChannelsViewSource}}" />
|
||||
<Setter Property="ItemsSource" Value="{Binding Source={StaticResource AvailableDirectChannelsViewSource}}" />
|
||||
</DataTrigger>
|
||||
<DataTrigger Binding="{Binding IsDirectMessageGuildSelected}" Value="False">
|
||||
<Setter Property="ItemsSource" Value="{Binding Source={StaticResource AvailableChannelsViewSource}}" />
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue