Ignore screen size changes which don't change anything (#203)

* Only reallocate shared memory if the size changes
Co-authored-by: Nexarian <cmp@pitstick.net>

2 files changed, 35 insertions, 27 deletions

diff --git a/module/rdpClientCon.c b/module/rdpClientCon.c
index f35743c..cd1d2f9 100644
--- a/module/rdpClientCon.c
+++ b/module/rdpClientCon.c
@@ -675,6 +675,37 @@ rdpClientConProcessMsgVersion(rdpPtr dev, rdpClientCon *clientCon,
     return 0;
 }
 
+/**************************************************************************//**
+ * Allocate shared memory
+ *
+ * This memory is shared with the xup driver in xrdp which avoids a lot
+ * of unnecessary copying
+ *
+ * @param clientCon Client connection
+ * @param bytes Size of area to attach
+ */
+static void
+rdpClientConAllocateSharedMemory(rdpClientCon *clientCon, int bytes)
+{
+    if (clientCon->shmemptr != NULL && clientCon->shmem_bytes == bytes)
+    {
+        LLOGLN(0, ("rdpClientConAllocateSharedMemory: reusing shmemid %d",
+               clientCon->shmemid));
+        return;
+    }
+    
+    if (clientCon->shmemptr != 0)
+    {
+        shmdt(clientCon->shmemptr);
+    }
+    clientCon->shmemid = shmget(IPC_PRIVATE, bytes, IPC_CREAT | 0777);
+    clientCon->shmemptr = shmat(clientCon->shmemid, 0, 0);
+    clientCon->shmem_bytes = bytes;
+    shmctl(clientCon->shmemid, IPC_RMID, NULL);
+    LLOGLN(0, ("rdpClientConAllocateSharedMemory: shmemid %d shmemptr %p bytes %d",
+           clientCon->shmemid, clientCon->shmemptr,
+           clientCon->shmem_bytes));
+}
 /******************************************************************************/
 /*
     this from miScreenInit
@@ -727,17 +758,9 @@ rdpClientConProcessScreenSizeMsg(rdpPtr dev, rdpClientCon *clientCon,
 
     clientCon->cap_stride_bytes = clientCon->rdp_width * clientCon->rdp_Bpp;
 
-    if (clientCon->shmemptr != 0)
-    {
-        shmdt(clientCon->shmemptr);
-    }
     bytes = clientCon->rdp_width * clientCon->rdp_height *
             clientCon->rdp_Bpp;
-    clientCon->shmemid = shmget(IPC_PRIVATE, bytes, IPC_CREAT | 0777);
-    clientCon->shmemptr = shmat(clientCon->shmemid, 0, 0);
-    shmctl(clientCon->shmemid, IPC_RMID, NULL);
-    LLOGLN(0, ("rdpClientConProcessScreenSizeMsg: shmemid %d shmemptr %p",
-           clientCon->shmemid, clientCon->shmemptr));
+    rdpClientConAllocateSharedMemory(clientCon, bytes);
     clientCon->shmem_lineBytes = clientCon->rdp_Bpp * clientCon->rdp_width;
 
     if (clientCon->shmRegion != 0)
@@ -868,17 +891,9 @@ rdpClientConProcessMsgClientInfo(rdpPtr dev, rdpClientCon *clientCon)
         clientCon->cap_height = RDPALIGN(clientCon->rdp_height, 64);
         LLOGLN(0, ("  cap_width %d cap_height %d",
                clientCon->cap_width, clientCon->cap_height));
-        if (clientCon->shmemptr != 0)
-        {
-            shmdt(clientCon->shmemptr);
-        }
         bytes = clientCon->cap_width * clientCon->cap_height *
                 clientCon->rdp_Bpp;
-        clientCon->shmemid = shmget(IPC_PRIVATE, bytes, IPC_CREAT | 0777);
-        clientCon->shmemptr = shmat(clientCon->shmemid, 0, 0);
-        shmctl(clientCon->shmemid, IPC_RMID, NULL);
-        LLOGLN(0, ("rdpClientConProcessMsgClientInfo: shmemid %d shmemptr %p "
-               "bytes %d", clientCon->shmemid, clientCon->shmemptr, bytes));
+        rdpClientConAllocateSharedMemory(clientCon, bytes);
         clientCon->shmem_lineBytes = clientCon->rdp_Bpp * clientCon->cap_width;
         clientCon->cap_stride_bytes = clientCon->cap_width * 4;
         shmemstatus = SHM_RFX_ACTIVE;
@@ -890,16 +905,8 @@ rdpClientConProcessMsgClientInfo(rdpPtr dev, rdpClientCon *clientCon)
         clientCon->cap_height = clientCon->rdp_height;
         LLOGLN(0, ("  cap_width %d cap_height %d",
                clientCon->cap_width, clientCon->cap_height));
-        if (clientCon->shmemptr != 0)
-        {
-            shmdt(clientCon->shmemptr);
-        }
         bytes = clientCon->cap_width * clientCon->cap_height * 2;
-        clientCon->shmemid = shmget(IPC_PRIVATE, bytes, IPC_CREAT | 0777);
-        clientCon->shmemptr = shmat(clientCon->shmemid, 0, 0);
-        shmctl(clientCon->shmemid, IPC_RMID, NULL);
-        LLOGLN(0, ("rdpClientConProcessMsgClientInfo: shmemid %d shmemptr %p "
-               "bytes %d", clientCon->shmemid, clientCon->shmemptr, bytes));
+        rdpClientConAllocateSharedMemory(clientCon, bytes);
         clientCon->shmem_lineBytes = clientCon->rdp_Bpp * clientCon->cap_width;
         clientCon->cap_stride_bytes = clientCon->cap_width * 4;
         shmemstatus = SHM_H264_ACTIVE;
diff --git a/module/rdpClientCon.h b/module/rdpClientCon.h
index 42b39c6..da845a7 100644
--- a/module/rdpClientCon.h
+++ b/module/rdpClientCon.h
@@ -107,6 +107,7 @@ struct _rdpClientCon
 
     uint8_t *shmemptr;
     int shmemid;
+    int shmem_bytes;
     int shmem_lineBytes;
     RegionPtr shmRegion;
     int rect_id;