Support side-by-side transports in Kestrel (#44657)

8 files changed, 187 insertions, 27 deletions

diff --git a/src/Servers/Connections.Abstractions/src/IConnectionListenerFactorySelector.cs b/src/Servers/Connections.Abstractions/src/IConnectionListenerFactorySelector.cs
new file mode 100644
index 0000000000..fc46c377d9
--- /dev/null
+++ b/src/Servers/Connections.Abstractions/src/IConnectionListenerFactorySelector.cs
@@ -0,0 +1,23 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Net;
+
+namespace Microsoft.AspNetCore.Connections;
+
+/// <summary>
+/// Defines an interface that determines whether the listener factory supports binding to the specified <see cref="EndPoint"/>.
+/// </summary>
+/// <remarks>
+/// This interface should be implemented by <see cref="IConnectionListenerFactory"/> and <see cref="IMultiplexedConnectionListenerFactory"/>
+/// types that want to control want endpoint instances they can bind to.
+/// </remarks>
+public interface IConnectionListenerFactorySelector
+{
+    /// <summary>
+    /// Returns a value that indicates whether the listener factory supports binding to the specified <see cref="EndPoint"/>.
+    /// </summary>
+    /// <param name="endpoint">The <see cref="EndPoint" /> to bind to.</param>
+    /// <returns>A value that indicates whether the listener factory supports binding to the specified <see cref="EndPoint"/>.</returns>
+    bool CanBind(EndPoint endpoint);
+}
diff --git a/src/Servers/Connections.Abstractions/src/PublicAPI/net462/PublicAPI.Unshipped.txt b/src/Servers/Connections.Abstractions/src/PublicAPI/net462/PublicAPI.Unshipped.txt
index 553973838a..88184eb7b6 100644
--- a/src/Servers/Connections.Abstractions/src/PublicAPI/net462/PublicAPI.Unshipped.txt
+++ b/src/Servers/Connections.Abstractions/src/PublicAPI/net462/PublicAPI.Unshipped.txt
@@ -1,3 +1,5 @@
 #nullable enable
 Microsoft.AspNetCore.Connections.Features.IStreamClosedFeature
 Microsoft.AspNetCore.Connections.Features.IStreamClosedFeature.OnClosed(System.Action<object?>! callback, object? state) -> void
+Microsoft.AspNetCore.Connections.IConnectionListenerFactorySelector
+Microsoft.AspNetCore.Connections.IConnectionListenerFactorySelector.CanBind(System.Net.EndPoint! endpoint) -> bool
diff --git a/src/Servers/Connections.Abstractions/src/PublicAPI/net7.0/PublicAPI.Unshipped.txt b/src/Servers/Connections.Abstractions/src/PublicAPI/net7.0/PublicAPI.Unshipped.txt
index 4dd8553d83..aa5fad13eb 100644
--- a/src/Servers/Connections.Abstractions/src/PublicAPI/net7.0/PublicAPI.Unshipped.txt
+++ b/src/Servers/Connections.Abstractions/src/PublicAPI/net7.0/PublicAPI.Unshipped.txt
@@ -1,6 +1,8 @@
 #nullable enable
 Microsoft.AspNetCore.Connections.Features.IStreamClosedFeature
 Microsoft.AspNetCore.Connections.Features.IStreamClosedFeature.OnClosed(System.Action<object?>! callback, object? state) -> void
+Microsoft.AspNetCore.Connections.IConnectionListenerFactorySelector
+Microsoft.AspNetCore.Connections.IConnectionListenerFactorySelector.CanBind(System.Net.EndPoint! endpoint) -> bool
 Microsoft.AspNetCore.Connections.TlsConnectionCallbackContext
 Microsoft.AspNetCore.Connections.TlsConnectionCallbackContext.ClientHelloInfo.get -> System.Net.Security.SslClientHelloInfo
 Microsoft.AspNetCore.Connections.TlsConnectionCallbackContext.ClientHelloInfo.set -> void
diff --git a/src/Servers/Connections.Abstractions/src/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt b/src/Servers/Connections.Abstractions/src/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt
index 553973838a..88184eb7b6 100644
--- a/src/Servers/Connections.Abstractions/src/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt
+++ b/src/Servers/Connections.Abstractions/src/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt
@@ -1,3 +1,5 @@
 #nullable enable
 Microsoft.AspNetCore.Connections.Features.IStreamClosedFeature
 Microsoft.AspNetCore.Connections.Features.IStreamClosedFeature.OnClosed(System.Action<object?>! callback, object? state) -> void
+Microsoft.AspNetCore.Connections.IConnectionListenerFactorySelector
+Microsoft.AspNetCore.Connections.IConnectionListenerFactorySelector.CanBind(System.Net.EndPoint! endpoint) -> bool
diff --git a/src/Servers/Connections.Abstractions/src/PublicAPI/netstandard2.1/PublicAPI.Unshipped.txt b/src/Servers/Connections.Abstractions/src/PublicAPI/netstandard2.1/PublicAPI.Unshipped.txt
index 553973838a..88184eb7b6 100644
--- a/src/Servers/Connections.Abstractions/src/PublicAPI/netstandard2.1/PublicAPI.Unshipped.txt
+++ b/src/Servers/Connections.Abstractions/src/PublicAPI/netstandard2.1/PublicAPI.Unshipped.txt
@@ -1,3 +1,5 @@
 #nullable enable
 Microsoft.AspNetCore.Connections.Features.IStreamClosedFeature
 Microsoft.AspNetCore.Connections.Features.IStreamClosedFeature.OnClosed(System.Action<object?>! callback, object? state) -> void
+Microsoft.AspNetCore.Connections.IConnectionListenerFactorySelector
+Microsoft.AspNetCore.Connections.IConnectionListenerFactorySelector.CanBind(System.Net.EndPoint! endpoint) -> bool
diff --git a/src/Servers/Kestrel/Core/src/Internal/Infrastructure/TransportManager.cs b/src/Servers/Kestrel/Core/src/Internal/Infrastructure/TransportManager.cs
index b1871b6d5c..441883b9dc 100644
--- a/src/Servers/Kestrel/Core/src/Internal/Infrastructure/TransportManager.cs
+++ b/src/Servers/Kestrel/Core/src/Internal/Infrastructure/TransportManager.cs
@@ -18,17 +18,17 @@ internal sealed class TransportManager
 {
     private readonly List<ActiveTransport> _transports = new List<ActiveTransport>();
 
-    private readonly IConnectionListenerFactory? _transportFactory;
-    private readonly IMultiplexedConnectionListenerFactory? _multiplexedTransportFactory;
+    private readonly List<IConnectionListenerFactory> _transportFactories;
+    private readonly List<IMultiplexedConnectionListenerFactory> _multiplexedTransportFactories;
     private readonly ServiceContext _serviceContext;
 
     public TransportManager(
-        IConnectionListenerFactory? transportFactory,
-        IMultiplexedConnectionListenerFactory? multiplexedTransportFactory,
+        List<IConnectionListenerFactory> transportFactories,
+        List<IMultiplexedConnectionListenerFactory> multiplexedTransportFactories,
         ServiceContext serviceContext)
     {
-        _transportFactory = transportFactory;
-        _multiplexedTransportFactory = multiplexedTransportFactory;
+        _transportFactories = transportFactories;
+        _multiplexedTransportFactories = multiplexedTransportFactories;
         _serviceContext = serviceContext;
     }
 
@@ -37,19 +37,28 @@ internal sealed class TransportManager
 
     public async Task<EndPoint> BindAsync(EndPoint endPoint, ConnectionDelegate connectionDelegate, EndpointConfig? endpointConfig, CancellationToken cancellationToken)
     {
-        if (_transportFactory is null)
+        if (_transportFactories.Count == 0)
         {
             throw new InvalidOperationException($"Cannot bind with {nameof(ConnectionDelegate)} no {nameof(IConnectionListenerFactory)} is registered.");
         }
 
-        var transport = await _transportFactory.BindAsync(endPoint, cancellationToken).ConfigureAwait(false);
-        StartAcceptLoop(new GenericConnectionListener(transport), c => connectionDelegate(c), endpointConfig);
-        return transport.EndPoint;
+        foreach (var transportFactory in _transportFactories)
+        {
+            var selector = transportFactory as IConnectionListenerFactorySelector;
+            if (CanBindFactory(endPoint, selector))
+            {
+                var transport = await transportFactory.BindAsync(endPoint, cancellationToken).ConfigureAwait(false);
+                StartAcceptLoop(new GenericConnectionListener(transport), c => connectionDelegate(c), endpointConfig);
+                return transport.EndPoint;
+            }
+        }
+
+        throw new InvalidOperationException($"No registered {nameof(IConnectionListenerFactory)} supports endpoint {endPoint.GetType().Name}: {endPoint}");
     }
 
     public async Task<EndPoint> BindAsync(EndPoint endPoint, MultiplexedConnectionDelegate multiplexedConnectionDelegate, ListenOptions listenOptions, CancellationToken cancellationToken)
     {
-        if (_multiplexedTransportFactory is null)
+        if (_multiplexedTransportFactories.Count == 0)
         {
             throw new InvalidOperationException($"Cannot bind with {nameof(MultiplexedConnectionDelegate)} no {nameof(IMultiplexedConnectionListenerFactory)} is registered.");
         }
@@ -87,9 +96,25 @@ internal sealed class TransportManager
             });
         }
 
-        var transport = await _multiplexedTransportFactory.BindAsync(endPoint, features, cancellationToken).ConfigureAwait(false);
-        StartAcceptLoop(new GenericMultiplexedConnectionListener(transport), c => multiplexedConnectionDelegate(c), listenOptions.EndpointConfig);
-        return transport.EndPoint;
+        foreach (var multiplexedTransportFactory in _multiplexedTransportFactories)
+        {
+            var selector = multiplexedTransportFactory as IConnectionListenerFactorySelector;
+            if (CanBindFactory(endPoint, selector))
+            {
+                var transport = await multiplexedTransportFactory.BindAsync(endPoint, features, cancellationToken).ConfigureAwait(false);
+                StartAcceptLoop(new GenericMultiplexedConnectionListener(transport), c => multiplexedConnectionDelegate(c), listenOptions.EndpointConfig);
+                return transport.EndPoint;
+            }
+        }
+
+        throw new InvalidOperationException($"No registered {nameof(IMultiplexedConnectionListenerFactory)} supports endpoint {endPoint.GetType().Name}: {endPoint}");
+    }
+
+    private static bool CanBindFactory(EndPoint endPoint, IConnectionListenerFactorySelector? selector)
+    {
+        // By default, the last registered factory binds to the endpoint.
+        // A factory can implement IConnectionListenerFactorySelector to decide whether it can bind to the endpoint.
+        return selector?.CanBind(endPoint) ?? true;
     }
 
     /// <summary>
diff --git a/src/Servers/Kestrel/Core/src/Internal/KestrelServerImpl.cs b/src/Servers/Kestrel/Core/src/Internal/KestrelServerImpl.cs
index 1063849f60..ea9d14ee13 100644
--- a/src/Servers/Kestrel/Core/src/Internal/KestrelServerImpl.cs
+++ b/src/Servers/Kestrel/Core/src/Internal/KestrelServerImpl.cs
@@ -22,8 +22,8 @@ internal sealed class KestrelServerImpl : IServer
 {
     private readonly ServerAddressesFeature _serverAddresses;
     private readonly TransportManager _transportManager;
-    private readonly IConnectionListenerFactory? _transportFactory;
-    private readonly IMultiplexedConnectionListenerFactory? _multiplexedTransportFactory;
+    private readonly List<IConnectionListenerFactory> _transportFactories;
+    private readonly List<IMultiplexedConnectionListenerFactory> _multiplexedTransportFactories;
 
     private readonly SemaphoreSlim _bindSemaphore = new SemaphoreSlim(initialCount: 1);
     private bool _hasStarted;
@@ -37,7 +37,7 @@ internal sealed class KestrelServerImpl : IServer
         IOptions<KestrelServerOptions> options,
         IEnumerable<IConnectionListenerFactory> transportFactories,
         ILoggerFactory loggerFactory)
-        : this(transportFactories, null, CreateServiceContext(options, loggerFactory, null))
+        : this(transportFactories, Array.Empty<IMultiplexedConnectionListenerFactory>(), CreateServiceContext(options, loggerFactory, null))
     {
     }
 
@@ -62,22 +62,22 @@ internal sealed class KestrelServerImpl : IServer
 
     // For testing
     internal KestrelServerImpl(IConnectionListenerFactory transportFactory, ServiceContext serviceContext)
-        : this(new[] { transportFactory }, null, serviceContext)
+        : this(new[] { transportFactory }, Array.Empty<IMultiplexedConnectionListenerFactory>(), serviceContext)
     {
     }
 
     // For testing
     internal KestrelServerImpl(
         IEnumerable<IConnectionListenerFactory> transportFactories,
-        IEnumerable<IMultiplexedConnectionListenerFactory>? multiplexedFactories,
+        IEnumerable<IMultiplexedConnectionListenerFactory> multiplexedFactories,
         ServiceContext serviceContext)
     {
         ArgumentNullException.ThrowIfNull(transportFactories);
 
-        _transportFactory = transportFactories.LastOrDefault();
-        _multiplexedTransportFactory = multiplexedFactories?.LastOrDefault();
+        _transportFactories = transportFactories.Reverse().ToList();
+        _multiplexedTransportFactories = multiplexedFactories.Reverse().ToList();
 
-        if (_transportFactory == null && _multiplexedTransportFactory == null)
+        if (_transportFactories.Count == 0 && _multiplexedTransportFactories.Count == 0)
         {
             throw new InvalidOperationException(CoreStrings.TransportNotFound);
         }
@@ -88,7 +88,7 @@ internal sealed class KestrelServerImpl : IServer
         _serverAddresses = new ServerAddressesFeature();
         Features.Set<IServerAddressesFeature>(_serverAddresses);
 
-        _transportManager = new TransportManager(_transportFactory, _multiplexedTransportFactory, ServiceContext);
+        _transportManager = new TransportManager(_transportFactories, _multiplexedTransportFactories, ServiceContext);
 
         HttpCharacters.Initialize();
     }
@@ -177,14 +177,14 @@ internal sealed class KestrelServerImpl : IServer
                 }
 
                 // Quic isn't registered if it's not supported, throw if we can't fall back to 1 or 2
-                if (hasHttp3 && _multiplexedTransportFactory is null && !(hasHttp1 || hasHttp2))
+                if (hasHttp3 && _multiplexedTransportFactories.Count == 0 && !(hasHttp1 || hasHttp2))
                 {
                     throw new InvalidOperationException("This platform doesn't support QUIC or HTTP/3.");
                 }
 
                 // Disable adding alt-svc header if endpoint has configured not to or there is no
                 // multiplexed transport factory, which happens if QUIC isn't supported.
-                var addAltSvcHeader = !options.DisableAltSvcHeader && _multiplexedTransportFactory != null;
+                var addAltSvcHeader = !options.DisableAltSvcHeader && _multiplexedTransportFactories.Count > 0;
 
                 var configuredEndpoint = options.EndPoint;
 
@@ -193,7 +193,7 @@ internal sealed class KestrelServerImpl : IServer
                     || options.Protocols == HttpProtocols.None) // TODO a test fails because it doesn't throw an exception in the right place
                                                                 // when there is no HttpProtocols in KestrelServer, can we remove/change the test?
                 {
-                    if (_transportFactory is null)
+                    if (_transportFactories.Count == 0)
                     {
                         throw new InvalidOperationException($"Cannot start HTTP/1.x or HTTP/2 server if no {nameof(IConnectionListenerFactory)} is registered.");
                     }
@@ -207,7 +207,7 @@ internal sealed class KestrelServerImpl : IServer
                     options.EndPoint = await _transportManager.BindAsync(configuredEndpoint, connectionDelegate, options.EndpointConfig, onBindCancellationToken).ConfigureAwait(false);
                 }
 
-                if (hasHttp3 && _multiplexedTransportFactory is not null)
+                if (hasHttp3 && _multiplexedTransportFactories.Count > 0)
                 {
                     // Check if a previous transport has changed the endpoint. If it has then the endpoint is dynamic and we can't guarantee it will work for other transports.
                     // For more details, see https://github.com/dotnet/aspnetcore/issues/42982
diff --git a/src/Servers/Kestrel/Core/test/KestrelServerTests.cs b/src/Servers/Kestrel/Core/test/KestrelServerTests.cs
index 8e7c282d2e..37ecd99169 100644
--- a/src/Servers/Kestrel/Core/test/KestrelServerTests.cs
+++ b/src/Servers/Kestrel/Core/test/KestrelServerTests.cs
@@ -244,6 +244,90 @@ public class KestrelServerTests
     }
 
     [Fact]
+    public async Task StartWithNoValidTransportFactoryThrows()
+    {
+        var serverOptions = CreateServerOptions();
+        serverOptions.Listen(new IPEndPoint(IPAddress.Loopback, 0));
+
+        var server = new KestrelServerImpl(
+                Options.Create<KestrelServerOptions>(serverOptions),
+                new List<IConnectionListenerFactory> { new NonBindableTransportFactory() },
+                new LoggerFactory(new[] { new KestrelTestLoggerProvider() }));
+
+        var exception = await Assert.ThrowsAsync<InvalidOperationException>(
+            async () => await server.StartAsync(new DummyApplication(context => Task.CompletedTask), CancellationToken.None));
+
+        Assert.Equal("No registered IConnectionListenerFactory supports endpoint IPEndPoint: 127.0.0.1:0", exception.Message);
+    }
+
+    [Fact]
+    public async Task StartWithMultipleTransportFactories_UseSupported()
+    {
+        var endpoint = new IPEndPoint(IPAddress.Loopback, 0);
+        var serverOptions = CreateServerOptions();
+        serverOptions.Listen(endpoint);
+
+        var transportFactory = new MockTransportFactory();
+
+        var server = new KestrelServerImpl(
+                Options.Create<KestrelServerOptions>(serverOptions),
+                new List<IConnectionListenerFactory> { transportFactory, new NonBindableTransportFactory() },
+                new LoggerFactory(new[] { new KestrelTestLoggerProvider() }));
+
+        await server.StartAsync(new DummyApplication(context => Task.CompletedTask), CancellationToken.None);
+
+        Assert.Collection(transportFactory.BoundEndPoints,
+            ep => Assert.Equal(endpoint, ep.OriginalEndPoint));
+    }
+
+    [Fact]
+    public async Task StartWithNoValidTransportFactoryThrows_Http3()
+    {
+        var serverOptions = CreateServerOptions();
+        serverOptions.Listen(new IPEndPoint(IPAddress.Loopback, 0), c =>
+        {
+            c.Protocols = HttpProtocols.Http3;
+            c.UseHttps(TestResources.GetTestCertificate());
+        });
+
+        var server = new KestrelServerImpl(
+                Options.Create<KestrelServerOptions>(serverOptions),
+                new List<IConnectionListenerFactory>(),
+                new List<IMultiplexedConnectionListenerFactory> { new NonBindableMultiplexedTransportFactory() },
+                new LoggerFactory(new[] { new KestrelTestLoggerProvider() }));
+
+        var exception = await Assert.ThrowsAsync<InvalidOperationException>(
+            async () => await server.StartAsync(new DummyApplication(context => Task.CompletedTask), CancellationToken.None));
+
+        Assert.Equal("No registered IMultiplexedConnectionListenerFactory supports endpoint IPEndPoint: 127.0.0.1:0", exception.Message);
+    }
+
+    [Fact]
+    public async Task StartWithMultipleTransportFactories_Http3_UseSupported()
+    {
+        var endpoint = new IPEndPoint(IPAddress.Loopback, 0);
+        var serverOptions = CreateServerOptions();
+        serverOptions.Listen(endpoint, c =>
+        {
+            c.Protocols = HttpProtocols.Http3;
+            c.UseHttps(TestResources.GetTestCertificate());
+        });
+
+        var transportFactory = new MockMultiplexedTransportFactory();
+
+        var server = new KestrelServerImpl(
+                Options.Create<KestrelServerOptions>(serverOptions),
+                new List<IConnectionListenerFactory>(),
+                new List<IMultiplexedConnectionListenerFactory> { transportFactory, new NonBindableMultiplexedTransportFactory() },
+                new LoggerFactory(new[] { new KestrelTestLoggerProvider() }));
+
+        await server.StartAsync(new DummyApplication(context => Task.CompletedTask), CancellationToken.None);
+
+        Assert.Collection(transportFactory.BoundEndPoints,
+            ep => Assert.Equal(endpoint, ep.OriginalEndPoint));
+    }
+
+    [Fact]
     public async Task ListenWithCustomEndpoint_DoesNotThrow()
     {
         var options = new KestrelServerOptions();
@@ -850,6 +934,26 @@ public class KestrelServerTests
         }
     }
 
+    private class NonBindableTransportFactory : IConnectionListenerFactory, IConnectionListenerFactorySelector
+    {
+        public ValueTask<IConnectionListener> BindAsync(EndPoint endpoint, CancellationToken cancellationToken = default)
+        {
+            throw new InvalidOperationException();
+        }
+
+        public bool CanBind(EndPoint endpoint) => false;
+    }
+
+    private class NonBindableMultiplexedTransportFactory : IMultiplexedConnectionListenerFactory, IConnectionListenerFactorySelector
+    {
+        public ValueTask<IMultiplexedConnectionListener> BindAsync(EndPoint endpoint, IFeatureCollection features = null, CancellationToken cancellationToken = default)
+        {
+            throw new InvalidOperationException();
+        }
+
+        public bool CanBind(EndPoint endpoint) => false;
+    }
+
     private class MockMultiplexedTransportFactory : IMultiplexedConnectionListenerFactory
     {
         public List<BindDetail> BoundEndPoints { get; } = new List<BindDetail>();