From 67f5aefae8fa2f154677e5001f1f7bed8e490521 Mon Sep 17 00:00:00 2001
From: Hendrik Leppkes <h.leppkes@gmail.com>
Date: Tue, 22 Aug 2017 14:39:02 +0200
Subject: d3d11: return all surfaces to the allocator when breaking the
 renderer connection

---
 decoder/LAVVideo/DecodeManager.cpp      | 10 ++++++++++
 decoder/LAVVideo/DecodeManager.h        |  1 +
 decoder/LAVVideo/LAVVideo.cpp           |  4 ++++
 decoder/LAVVideo/decoders/DecBase.h     |  1 +
 decoder/LAVVideo/decoders/ILAVDecoder.h |  5 +++++
 decoder/LAVVideo/decoders/d3d11va.cpp   | 15 +++++++++++++++
 decoder/LAVVideo/decoders/d3d11va.h     |  1 +
 7 files changed, 37 insertions(+)

diff --git a/decoder/LAVVideo/DecodeManager.cpp b/decoder/LAVVideo/DecodeManager.cpp
index 4ea51b0b..1178742e 100644
--- a/decoder/LAVVideo/DecodeManager.cpp
+++ b/decoder/LAVVideo/DecodeManager.cpp
@@ -237,3 +237,13 @@ STDMETHODIMP CDecodeManager::PostConnect(IPin *pPin)
   }
   return hr;
 }
+
+STDMETHODIMP CDecodeManager::BreakConnect()
+{
+  CAutoLock decoderLock(this);
+
+  if (!m_pDecoder)
+    return E_UNEXPECTED;
+
+  return m_pDecoder->BreakConnect();
+}
diff --git a/decoder/LAVVideo/DecodeManager.h b/decoder/LAVVideo/DecodeManager.h
index 81162bcc..06aeb257 100644
--- a/decoder/LAVVideo/DecodeManager.h
+++ b/decoder/LAVVideo/DecodeManager.h
@@ -44,6 +44,7 @@ public:
   // Allocator/memory management
   STDMETHODIMP InitAllocator(IMemAllocator **ppAlloc);
   STDMETHODIMP PostConnect(IPin *pPin);
+  STDMETHODIMP BreakConnect();
 
   // HWAccel Query
   STDMETHODIMP_(BOOL) IsHWDecoderActive() { return m_bHWDecoder; }
diff --git a/decoder/LAVVideo/LAVVideo.cpp b/decoder/LAVVideo/LAVVideo.cpp
index dbea52cc..d7f84c15 100644
--- a/decoder/LAVVideo/LAVVideo.cpp
+++ b/decoder/LAVVideo/LAVVideo.cpp
@@ -907,6 +907,10 @@ HRESULT CLAVVideo::BreakConnect(PIN_DIRECTION dir)
     if (m_pFilterGraph)
       avfilter_graph_free(&m_pFilterGraph);
   }
+  else if (dir == PINDIR_OUTPUT)
+  {
+    m_Decoder.BreakConnect();
+  }
   return __super::BreakConnect(dir);
 }
 
diff --git a/decoder/LAVVideo/decoders/DecBase.h b/decoder/LAVVideo/decoders/DecBase.h
index fb535b21..2ef16125 100644
--- a/decoder/LAVVideo/decoders/DecBase.h
+++ b/decoder/LAVVideo/decoders/DecBase.h
@@ -38,6 +38,7 @@ public:
   STDMETHODIMP_(BOOL) IsInterlaced(BOOL bAllowGuess) { return TRUE; }
   STDMETHODIMP InitAllocator(IMemAllocator **ppAlloc) { return E_NOTIMPL; }
   STDMETHODIMP PostConnect(IPin *pPin) { return S_FALSE; }
+  STDMETHODIMP BreakConnect() { return S_FALSE; }
   STDMETHODIMP_(long) GetBufferCount(long *pMaxBuffers = nullptr) { return 2; }
 
   STDMETHODIMP HasThreadSafeBuffers() { return S_FALSE; }
diff --git a/decoder/LAVVideo/decoders/ILAVDecoder.h b/decoder/LAVVideo/decoders/ILAVDecoder.h
index 9e6bcca0..17f36c67 100644
--- a/decoder/LAVVideo/decoders/ILAVDecoder.h
+++ b/decoder/LAVVideo/decoders/ILAVDecoder.h
@@ -385,6 +385,11 @@ interface ILAVDecoder
    */
   STDMETHOD(PostConnect)(IPin *pPin) PURE;
 
+  /**
+   * Notify the decoder the output connection was broken
+   */
+  STDMETHOD(BreakConnect)() PURE;
+
   /**
    * Get the number of sample buffers optimal for this decoder
    */
diff --git a/decoder/LAVVideo/decoders/d3d11va.cpp b/decoder/LAVVideo/decoders/d3d11va.cpp
index 27f939b0..f0c08251 100644
--- a/decoder/LAVVideo/decoders/d3d11va.cpp
+++ b/decoder/LAVVideo/decoders/d3d11va.cpp
@@ -426,6 +426,21 @@ fail:
   return E_FAIL;
 }
 
+STDMETHODIMP CDecD3D11::BreakConnect()
+{
+  if (m_bReadBackFallback)
+    return S_FALSE;
+
+  // release any resources held by the core
+  m_pCallback->ReleaseAllDXVAResources();
+
+  // flush all buffers out of the decoder to ensure the allocator can be properly de-allocated
+  if (m_pAVCtx && avcodec_is_open(m_pAVCtx))
+    avcodec_flush_buffers(m_pAVCtx);
+
+  return S_OK;
+}
+
 STDMETHODIMP CDecD3D11::InitDecoder(AVCodecID codec, const CMediaType *pmt)
 {
   HRESULT hr = S_OK;
diff --git a/decoder/LAVVideo/decoders/d3d11va.h b/decoder/LAVVideo/decoders/d3d11va.h
index 82b67ef9..75e4914c 100644
--- a/decoder/LAVVideo/decoders/d3d11va.h
+++ b/decoder/LAVVideo/decoders/d3d11va.h
@@ -51,6 +51,7 @@ public:
 
   STDMETHODIMP InitAllocator(IMemAllocator **ppAlloc);
   STDMETHODIMP PostConnect(IPin *pPin);
+  STDMETHODIMP BreakConnect();
   STDMETHODIMP_(long) GetBufferCount(long *pMaxBuffers = nullptr);
   STDMETHODIMP_(const WCHAR*) GetDecoderName() { return m_bReadBackFallback ? (m_bDirect ? L"d3d11 cb direct" : L"d3d11 cb") : L"d3d11 native"; }
   STDMETHODIMP HasThreadSafeBuffers() { return S_FALSE; }
-- 
cgit v1.2.3