mirrors/DiscordChatExporter
1
0
Fork 0
mirror of https://github.com/Tyrrrz/DiscordChatExporter.git synced 2025-06-03 08:08:40 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add partition by file size (#497)

Browse source
This commit is contained in:
Andrew Kolos 2021-04-12 06:50:32 -04:00 • committed by GitHub
parent ad3655396f
commit eb89ea5b40
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 331 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

1
DiscordChatExporter.Core/Discord/Data/Attachment.cs
View file

@ -5,6 +5,7 @@ using System.Text.Json;
using DiscordChatExporter.Core.Discord.Data.Common;
using DiscordChatExporter.Core.Utils.Extensions;
using JsonExtensions.Reading;
using FileSize = DiscordChatExporter.Core.Discord.Data.Common.FileSize;
namespace DiscordChatExporter.Core.Discord.Data
{

2
DiscordChatExporter.Core/Discord/Data/Common/FileSize.cs
View file

@ -62,4 +62,4 @@ namespace DiscordChatExporter.Core.Discord.Data.Common
{
public static FileSize FromBytes(long bytes) => new(bytes);
}
}
}

1
DiscordChatExporter.Core/DiscordChatExporter.Core.csproj
View file

@ -5,6 +5,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="ByteSize" Version="2.0.0" />
<PackageReference Include="JsonExtensions" Version="1.0.1" />
<PackageReference Include="MiniRazor.CodeGen" Version="2.1.2" />
<PackageReference Include="Polly" Version="7.2.1" />

6
DiscordChatExporter.Core/Exporting/ExportRequest.cs
View file

@ -28,7 +28,7 @@ namespace DiscordChatExporter.Core.Exporting
public Snowflake? Before { get; }
public int? PartitionLimit { get; }
public IPartitioner Partitoner { get; }
public bool ShouldDownloadMedia { get; }
@ -43,7 +43,7 @@ namespace DiscordChatExporter.Core.Exporting
ExportFormat format,
Snowflake? after,
Snowflake? before,
int? partitionLimit,
IPartitioner partitioner,
bool shouldDownloadMedia,
bool shouldReuseMedia,
string dateFormat)
@ -54,7 +54,7 @@ namespace DiscordChatExporter.Core.Exporting
Format = format;
After = after;
Before = before;
PartitionLimit = partitionLimit;
Partitoner = partitioner;
ShouldDownloadMedia = shouldDownloadMedia;
ShouldReuseMedia = shouldReuseMedia;
DateFormat = dateFormat;

22
DiscordChatExporter.Core/Exporting/MessageExporter.cs
View file

@ -1,13 +1,18 @@
using System;
using System.IO;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using ByteSizeLib;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exporting;
using DiscordChatExporter.Core.Exporting.Partitioners;
using DiscordChatExporter.Core.Exporting.Writers;
namespace DiscordChatExporter.Core.Exporting
{
internal partial class MessageExporter : IAsyncDisposable
{
private readonly ExportContext _context;
private long _messageCount;
@ -19,11 +24,16 @@ namespace DiscordChatExporter.Core.Exporting
_context = context;
}
private bool IsPartitionLimitReached() =>
_messageCount > 0 &&
_context.Request.PartitionLimit is not null &&
_context.Request.PartitionLimit != 0 &&
_messageCount % _context.Request.PartitionLimit == 0;
private bool IsPartitionLimitReached()
{
if (_writer is null)
{
return false;
}
return _context.Request.Partitoner.IsLimitReached(
new ExportPartitioningContext(_messageCount, _writer.SizeInBytes));
}
private async ValueTask ResetWriterAsync()
{
@ -38,7 +48,7 @@ namespace DiscordChatExporter.Core.Exporting
private async ValueTask<MessageWriter> GetWriterAsync()
{
// Ensure partition limit has not been exceeded
if (IsPartitionLimitReached())
if (_writer != null && IsPartitionLimitReached())
{
await ResetWriterAsync();
_partitionIndex++;

18
DiscordChatExporter.Core/Exporting/Partitioners/ExportPartitioningContext.cs Normal file
View file

@ -0,0 +1,18 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace DiscordChatExporter.Core.Exporting.Partitioners
{
public class ExportPartitioningContext
{
public long MessageCount { get; }
public long SizeInBytes { get; }
public ExportPartitioningContext(long messageCount, long sizeInBytes)
{
MessageCount = messageCount;
SizeInBytes = sizeInBytes;
}
}
}

21
DiscordChatExporter.Core/Exporting/Partitioners/FileSizePartitioner.cs Normal file
View file

@ -0,0 +1,21 @@
using DiscordChatExporter.Core.Exporting.Partitioners;
using System;
using System.Collections.Generic;
using System.Text;
namespace DiscordChatExporter.Core.Exporting
{
public class FileSizePartitioner : IPartitioner
{
private long _bytesPerFile;
public FileSizePartitioner(long bytesPerFile)
{
_bytesPerFile = bytesPerFile;
}
public bool IsLimitReached(ExportPartitioningContext context)
{
return context.SizeInBytes >= _bytesPerFile;
}
}
}

12
DiscordChatExporter.Core/Exporting/Partitioners/IPartitioner.cs Normal file
View file

@ -0,0 +1,12 @@
using DiscordChatExporter.Core.Exporting.Partitioners;
using System;
using System.Collections.Generic;
using System.Text;
namespace DiscordChatExporter.Core.Exporting
{
public interface IPartitioner
{
bool IsLimitReached(ExportPartitioningContext context);
}
}

25
DiscordChatExporter.Core/Exporting/Partitioners/MessageCountPartitioner.cs Normal file
View file

@ -0,0 +1,25 @@
using DiscordChatExporter.Core.Exporting.Partitioners;
using System;
using System.Collections.Generic;
using System.Text;
namespace DiscordChatExporter.Core.Exporting
{
public class MessageCountPartitioner : IPartitioner
{
private int _messagesPerPartition;
public MessageCountPartitioner(int messagesPerPartition)
{
_messagesPerPartition = messagesPerPartition;
}
public bool IsLimitReached(ExportPartitioningContext context)
{
return context.MessageCount > 0 &&
_messagesPerPartition != 0 &&
context.MessageCount % _messagesPerPartition == 0;
}
}
}

16
DiscordChatExporter.Core/Exporting/Partitioners/NullPartitioner.cs Normal file
View file

@ -0,0 +1,16 @@
using DiscordChatExporter.Core.Exporting;
using DiscordChatExporter.Core.Exporting.Partitioners;
using System;
using System.Collections.Generic;
using System.Text;
namespace DiscordChatExporter.Core.Exporting
{
public class NullPartitioner : IPartitioner
{
public bool IsLimitReached(ExportPartitioningContext context)
{
return false;
}
}
}

2
DiscordChatExporter.Core/Exporting/Writers/MessageWriter.cs
View file

@ -17,6 +17,8 @@ namespace DiscordChatExporter.Core.Exporting.Writers
Context = context;
}
public long SizeInBytes => Stream.Length;
public virtual ValueTask WritePreambleAsync() => default;
public abstract ValueTask WriteMessageAsync(Message message);