Rework retry policy

Fixes #832

DiscordChatExporter.Core/Discord/DiscordClient.cs

@@ -3,7 +3,6 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Net;
 using System.Net.Http;
-using System.Net.Http.Headers;
 using System.Runtime.CompilerServices;
 using System.Text.Json;
 using System.Threading;
@@ -52,7 +51,7 @@ public class DiscordClient
         string url,
         CancellationToken cancellationToken = default)
     {
-        return await Http.ResponsePolicy.ExecuteAsync(async innerCancellationToken =>
+        return await Http.ResponseResiliencePolicy.ExecuteAsync(async innerCancellationToken =>
         {
             if (_tokenKind == TokenKind.User)
                 return await GetResponseAsync(url, false, innerCancellationToken);

DiscordChatExporter.Core/Exporting/MediaDownloader.cs

@@ -35,45 +35,42 @@ internal partial class MediaDownloader
         var filePath = Path.Combine(_workingDirPath, fileName);
 
         // Reuse existing files if we're allowed to
-        if (_reuseMedia && File.Exists(filePath))
-            return _pathCache[url] = filePath;
-
-        Directory.CreateDirectory(_workingDirPath);
-
-        // This retries on IOExceptions which is dangerous as we're also working with files
-        await Http.ExceptionPolicy.ExecuteAsync(async () =>
+        if (!_reuseMedia || !File.Exists(filePath))
         {
-            // Download the file
-            using var response = await Http.Client.GetAsync(url, cancellationToken);
-            await using (var output = File.Create(filePath))
-            {
-                await response.Content.CopyToAsync(output, cancellationToken);
-            }
+            Directory.CreateDirectory(_workingDirPath);
 
-            // Try to set the file date according to the last-modified header
-            try
+            await Http.ResiliencePolicy.ExecuteAsync(async () =>
             {
-                var lastModified = response.Content.Headers.TryGetValue("Last-Modified")?.Pipe(s =>
-                    DateTimeOffset.TryParse(s, CultureInfo.InvariantCulture, DateTimeStyles.None, out var date)
-                        ? date
-                        : (DateTimeOffset?) null
-                );
+                // Download the file
+                using var response = await Http.Client.GetAsync(url, cancellationToken);
+                await using (var output = File.Create(filePath))
+                    await response.Content.CopyToAsync(output, cancellationToken);
 
-                if (lastModified is not null)
+                // Try to set the file date according to the last-modified header
+                try
                 {
-                    File.SetCreationTimeUtc(filePath, lastModified.Value.UtcDateTime);
-                    File.SetLastWriteTimeUtc(filePath, lastModified.Value.UtcDateTime);
-                    File.SetLastAccessTimeUtc(filePath, lastModified.Value.UtcDateTime);
+                    var lastModified = response.Content.Headers.TryGetValue("Last-Modified")?.Pipe(s =>
+                        DateTimeOffset.TryParse(s, CultureInfo.InvariantCulture, DateTimeStyles.None, out var date)
+                            ? date
+                            : (DateTimeOffset?) null
+                    );
+
+                    if (lastModified is not null)
+                    {
+                        File.SetCreationTimeUtc(filePath, lastModified.Value.UtcDateTime);
+                        File.SetLastWriteTimeUtc(filePath, lastModified.Value.UtcDateTime);
+                        File.SetLastAccessTimeUtc(filePath, lastModified.Value.UtcDateTime);
+                    }
                 }
-            }
-            catch
-            {
-                // This can apparently fail for some reason.
-                // https://github.com/Tyrrrz/DiscordChatExporter/issues/585
-                // Updating file dates is not a critical task, so we'll just
-                // ignore exceptions thrown here.
-            }
-        });
+                catch
+                {
+                    // This can apparently fail for some reason.
+                    // https://github.com/Tyrrrz/DiscordChatExporter/issues/585
+                    // Updating file dates is not a critical task, so we'll just
+                    // ignore exceptions thrown here.
+                }
+            });
+        }
 
         return _pathCache[url] = filePath;
     }

DiscordChatExporter.Core/Utils/Extensions/ExceptionExtensions.cs

@@ -0,0 +1,44 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Net;
+using System.Net.Http;
+using System.Text.RegularExpressions;
+
+namespace DiscordChatExporter.Core.Utils.Extensions;
+
+public static class ExceptionExtensions
+{
+    private static void PopulateChildren(this Exception exception, ICollection<Exception> children)
+    {
+        if (exception is AggregateException aggregateException)
+        {
+            foreach (var innerException in aggregateException.InnerExceptions)
+            {
+                children.Add(innerException);
+                PopulateChildren(innerException, children);
+            }
+        }
+        else if (exception.InnerException is not null)
+        {
+            children.Add(exception.InnerException);
+            PopulateChildren(exception.InnerException, children);
+        }
+    }
+
+    public static IReadOnlyList<Exception> GetSelfAndChildren(this Exception exception)
+    {
+        var children = new List<Exception> {exception};
+        PopulateChildren(exception, children);
+        return children;
+    }
+
+    public static HttpStatusCode? TryGetStatusCode(this HttpRequestException ex) =>
+        // This is extremely frail, but there's no other way
+        Regex
+            .Match(ex.Message, @": (\d+) \(")
+            .Groups[1]
+            .Value
+            .NullIfWhiteSpace()?
+            .Pipe(s => (HttpStatusCode) int.Parse(s, CultureInfo.InvariantCulture));
+}

DiscordChatExporter.Core/Utils/Http.cs

@@ -1,9 +1,8 @@
 using System;
-using System.Globalization;
-using System.IO;
+using System.Linq;
 using System.Net;
 using System.Net.Http;
-using System.Text.RegularExpressions;
+using System.Net.Sockets;
 using System.Threading.Tasks;
 using DiscordChatExporter.Core.Utils.Extensions;
 using Polly;
@@ -14,13 +13,26 @@ public static class Http
 {
     public static HttpClient Client { get; } = new();
 
-    public static IAsyncPolicy<HttpResponseMessage> ResponsePolicy { get; } =
+    private static bool IsRetryableStatusCode(HttpStatusCode statusCode) => statusCode is
+        HttpStatusCode.TooManyRequests or
+        HttpStatusCode.RequestTimeout or
+        HttpStatusCode.InternalServerError;
+
+    private static bool IsRetryableException(Exception exception) =>
+        exception.GetSelfAndChildren().Any(ex =>
+            ex is TimeoutException or SocketException ||
+            ex is HttpRequestException hrex && IsRetryableStatusCode(hrex.TryGetStatusCode() ?? HttpStatusCode.OK)
+        );
+
+    public static IAsyncPolicy ResiliencePolicy { get; } =
         Policy
-            .Handle<IOException>()
-            .Or<HttpRequestException>()
-            .OrResult<HttpResponseMessage>(m => m.StatusCode == HttpStatusCode.TooManyRequests)
-            .OrResult(m => m.StatusCode == HttpStatusCode.RequestTimeout)
-            .OrResult(m => m.StatusCode >= HttpStatusCode.InternalServerError)
+            .Handle<Exception>(IsRetryableException)
+            .WaitAndRetryAsync(4, i => TimeSpan.FromSeconds(Math.Pow(2, i) + 1));
+
+    public static IAsyncPolicy<HttpResponseMessage> ResponseResiliencePolicy { get; } =
+        Policy
+            .Handle<Exception>(IsRetryableException)
+            .OrResult<HttpResponseMessage>(m => IsRetryableStatusCode(m.StatusCode))
             .WaitAndRetryAsync(
                 8,
                 (i, result, _) =>
@@ -43,24 +55,4 @@ public static class Http
                 },
                 (_, _, _, _) => Task.CompletedTask
             );
-
-    private static HttpStatusCode? TryGetStatusCodeFromException(HttpRequestException ex) =>
-        // This is extremely frail, but there's no other way
-        Regex
-            .Match(ex.Message, @": (\d+) \(")
-            .Groups[1]
-            .Value
-            .NullIfWhiteSpace()?
-            .Pipe(s => (HttpStatusCode) int.Parse(s, CultureInfo.InvariantCulture));
-
-    public static IAsyncPolicy ExceptionPolicy { get; } =
-        Policy
-            .Handle<IOException>() // dangerous
-            .Or<HttpRequestException>(ex =>
-                TryGetStatusCodeFromException(ex) is
-                    HttpStatusCode.TooManyRequests or
-                    HttpStatusCode.RequestTimeout or
-                    HttpStatusCode.InternalServerError
-            )
-            .WaitAndRetryAsync(4, i => TimeSpan.FromSeconds(Math.Pow(2, i) + 1));
 }