Add function prologue and epilogue in debug mode (#49)

If we get a trap in the rfxcodec_encode_dwt_shift_amd64_sseXx routine,
the back trace will be garbage because the debugger does not know where
on the stack the frame of the function begins.

This patch adds a standard prologue and epilogue to the function, so the
debugger will find the start of the frame from the base pointer. It is
only added if the library is compiled with debug information enabled.

3 files changed, 52 insertions, 12 deletions

diff --git a/configure.ac b/configure.ac
index 11b9ed4..3cbde12 100644
--- a/configure.ac
+++ b/configure.ac
@@ -43,6 +43,18 @@ fi
 AM_CONDITIONAL(WITH_SIMD_AMD64, [test x$simd_arch = xx86_64])
 AM_CONDITIONAL(WITH_SIMD_X86, [test x$simd_arch = xi386])
 
+# parent project will propagate these options to us when building
+AC_ARG_ENABLE(devel_all, AS_HELP_STRING([--enable-devel-all]),
+              [devel_all=$enableval], [devel_all=no])
+AC_ARG_ENABLE(devel_debug, AS_HELP_STRING([--enable-devel-debug],
+              [Build library with support for getting better backtraces [default=no]]),
+              [devel_debug=$enableval], [devel_debug=$devel_all])
+AM_CONDITIONAL(DEVEL_DEBUG, [test x$devel_debug = xyes ])
+
+# pass debug option to the assembler if specified
+AM_COND_IF([DEVEL_DEBUG],
+  [AX_APPEND_COMPILE_FLAGS([-DDEBUG], [NAFLAGS])])
+
 AC_CONFIG_FILES([Makefile
                  include/Makefile
                  src/Makefile
diff --git a/src/amd64/rfxcodec_encode_dwt_shift_amd64_sse2.asm b/src/amd64/rfxcodec_encode_dwt_shift_amd64_sse2.asm
index cef3902..b3efda0 100644
--- a/src/amd64/rfxcodec_encode_dwt_shift_amd64_sse2.asm
+++ b/src/amd64/rfxcodec_encode_dwt_shift_amd64_sse2.asm
@@ -1325,8 +1325,8 @@ set_quants_lo:
     movdqa xmm10, [rdx]
     ret
 
-;The first six integer or pointer arguments are passed in registers
-;RDI, RSI, RDX, RCX, R8, and R9
+; Per System V AMD64 ABI, the first six integer or pointer arguments are
+; passed in registers RDI, RSI, RDX, RCX, R8, and R9
 
 ;int
 ;rfxcodec_encode_dwt_shift_amd64_sse2(const char *qtable,
@@ -1336,12 +1336,19 @@ set_quants_lo:
 
 ;******************************************************************************
 PROC rfxcodec_encode_dwt_shift_amd64_sse2
+    ; prologue. this will make the function appear to the debugger as
+    ; having a stack frame, so that a backtrace can be obtained
+%ifdef DEBUG
+    push rbp
+    mov rbp, rsp
+%endif
+
     ; save registers
     push rbx
-    push rdx
-    push rcx
-    push rsi
-    push rdi
+    push rdx  ; rsp+24: out_buffer
+    push rcx  ; rsp+16: work_buffer
+    push rsi  ; rsp+ 8: in_buffer
+    push rdi  ; rsp+ 0: qtable
     pxor xmm0, xmm0
 
     ; verical DWT to work buffer, level 1
@@ -1483,5 +1490,12 @@ PROC rfxcodec_encode_dwt_shift_amd64_sse2
     pop rcx
     pop rdx
     pop rbx
+
+    ; epilogue
+%ifdef DEBUG
+    mov rsp, rbp
+    pop rbp
+%endif
+
     ret
 END_OF_FILE
diff --git a/src/amd64/rfxcodec_encode_dwt_shift_amd64_sse41.asm b/src/amd64/rfxcodec_encode_dwt_shift_amd64_sse41.asm
index da176e7..2629059 100644
--- a/src/amd64/rfxcodec_encode_dwt_shift_amd64_sse41.asm
+++ b/src/amd64/rfxcodec_encode_dwt_shift_amd64_sse41.asm
@@ -1193,8 +1193,8 @@ set_quants_lo:
     movdqa xmm10, [rdx]
     ret
 
-;The first six integer or pointer arguments are passed in registers
-;RDI, RSI, RDX, RCX, R8, and R9
+; Per System V AMD64 ABI, the first six integer or pointer arguments are
+; passed in registers RDI, RSI, RDX, RCX, R8, and R9
 
 ;int
 ;rfxcodec_encode_dwt_shift_amd64_sse41(const char *qtable,
@@ -1204,12 +1204,19 @@ set_quants_lo:
 
 ;******************************************************************************
 PROC rfxcodec_encode_dwt_shift_amd64_sse41
+    ; prologue. this will make the function appear to the debugger as
+    ; having a stack frame, so that a backtrace can be obtained
+%ifdef DEBUG
+    push rbp
+    mov rbp, rsp
+%endif
+
     ; save registers
     push rbx
-    push rdx
-    push rcx
-    push rsi
-    push rdi
+    push rdx  ; rsp+24: out_buffer
+    push rcx  ; rsp+16: work_buffer
+    push rsi  ; rsp+ 8: in_buffer
+    push rdi  ; rsp+ 0: qtable
     pxor xmm0, xmm0
 
     ; verical DWT to work buffer, level 1
@@ -1351,5 +1358,12 @@ PROC rfxcodec_encode_dwt_shift_amd64_sse41
     pop rcx
     pop rdx
     pop rbx
+
+    ; epilogue
+%ifdef DEBUG
+    mov rsp, rbp
+    pop rbp
+%endif
+
     ret
 END_OF_FILE