build: Explicitly set 32-bit/64-bit object formats for nasm/yasm

Consistently use object format names with "32" suffix and set object format
to "win64" on Windows x86_64, which fixes assembling with nasm.

(Cherry-picked from libav commit 808ef43597b1e3d6e69a5b9abe2237c8ddb97b44)

Signed-off-by: James Almer <jamrial@gmail.com>

1 files changed, 9 insertions, 14 deletions

diff --git a/configure b/configure
index 2f32f99944..e7691404b6 100755
--- a/configure
+++ b/configure
@@ -3254,7 +3254,6 @@ doxygen_default="doxygen"
 install="install"
 ln_s_default="ln -s -f"
 nm_default="nm -g"
-objformat="elf"
 pkg_config_default=pkg-config
 ranlib_default="ranlib"
 strip_default="strip"
@@ -4797,6 +4796,8 @@ esac
 enable $subarch
 enabled spic && enable_weak pic
 
+enabled x86_64 && objformat=elf64 || objformat="elf32"
+
 # OS specific
 case $target_os in
     aix)
@@ -4867,7 +4868,7 @@ case $target_os in
         SLIBNAME_WITH_VERSION='$(SLIBPREF)$(FULLNAME).$(LIBVERSION)$(SLIBSUF)'
         SLIBNAME_WITH_MAJOR='$(SLIBPREF)$(FULLNAME).$(LIBMAJOR)$(SLIBSUF)'
         objformat="macho"
-        enabled x86_64 && objformat="macho64"
+        enabled x86_64 && objformat="macho64" || objformat="macho32"
         enabled_any pic shared x86_64 ||
             { check_cflags -mdynamic-no-pic && add_asflags -mdynamic-no-pic; }
         check_header dispatch/dispatch.h &&
@@ -4911,7 +4912,7 @@ case $target_os in
         SLIB_INSTALL_EXTRA_SHLIB='$(SLIBNAME:$(SLIBSUF)=.lib)'
         SLIB_INSTALL_EXTRA_LIB='lib$(SLIBNAME:$(SLIBSUF)=.dll.a) $(SLIBNAME_WITH_MAJOR:$(SLIBSUF)=.def)'
         SHFLAGS='-shared -Wl,--output-def,$$(@:$(SLIBSUF)=.orig.def) -Wl,--out-implib,$(SUBDIR)lib$(SLIBNAME:$(SLIBSUF)=.dll.a) -Wl,--enable-runtime-pseudo-reloc -Wl,--disable-auto-image-base'
-        objformat="win32"
+        enabled x86_64 && objformat="win64" || objformat="win32"
         ranlib=:
         enable dos_paths
         check_ldflags -Wl,--nxcompat,--dynamicbase
@@ -4951,7 +4952,7 @@ case $target_os in
         SLIB_INSTALL_EXTRA_SHLIB='$(SLIBNAME:$(SLIBSUF)=.lib)'
         SLIB_INSTALL_EXTRA_LIB='$(SLIBNAME_WITH_MAJOR:$(SLIBSUF)=.def)'
         SHFLAGS='-dll -def:$$(@:$(SLIBSUF)=.def) -implib:$(SUBDIR)$(SLIBNAME:$(SLIBSUF)=.lib)'
-        objformat="win32"
+        enabled x86_64 && objformat="win64" || objformat="win32"
         ranlib=:
         enable dos_paths
         ;;
@@ -4966,7 +4967,7 @@ case $target_os in
         SLIB_INSTALL_LINKS=
         SLIB_INSTALL_EXTRA_LIB='lib$(FULLNAME).dll.a'
         SHFLAGS='-shared -Wl,--out-implib,$(SUBDIR)lib$(FULLNAME).dll.a'
-        objformat="win32"
+        enabled x86_64 && objformat="win64" || objformat="win32"
         enable dos_paths
         enabled shared && ! enabled small && check_cmd $windres --version && enable gnu_windres
         add_cppflags -D_POSIX_C_SOURCE=200112 -D_XOPEN_SOURCE=600
@@ -5377,7 +5378,8 @@ EOF
 EOF
 
     # llvm's integrated assembler supports .object_arch from llvm 3.5
-    [ "$objformat" = elf ] && check_as <<EOF && enable as_object_arch
+    [ "$objformat" = elf32 ] || [ "$objformat" = elf64 ] &&
+        check_as <<EOF && enable as_object_arch
 .object_arch armv4
 EOF
 
@@ -5490,22 +5492,15 @@ EOF
 
     if ! disabled_any asm mmx x86asm; then
         if check_cmd $x86asmexe --version; then
-            enabled x86_64 && x86asm_extra="-m amd64"
             x86asm_debug="-g dwarf2"
             X86ASMDEP='$(DEPX86ASM) $(X86ASMFLAGS) -M $(X86ASM_O) $< > $(@:.o=.d)'
         elif check_cmd nasm -v; then
             x86asmexe=nasm
             x86asm_debug="-g -F dwarf"
             X86ASM_DEPFLAGS='-MD $(@:.o=.d)'
-            if enabled x86_64; then
-                case "$objformat" in
-                    elf)   objformat=elf64 ;;
-                    win32) objformat=win64 ;;
-                esac
-            fi
         fi
 
-        X86ASMFLAGS="-f $objformat $x86asm_extra"
+        X86ASMFLAGS="-f $objformat"
         enabled pic               && append X86ASMFLAGS "-DPIC"
         test -n "$extern_prefix"  && append X86ASMFLAGS "-DPREFIX"
         case "$objformat" in