REFAC(overlay): improve find_odlsym(), fix crash when DT_HASH is not present

dlsym() is changed so that it returns NULL when the original dlsym() (odlsym()) is not available.

find_odlsym() is refactored:

- A segmentation fault was reported many years ago on Gentoo's bug tracker: https://bugs.gentoo.org/527504
Turns out it happens when DT_HASH is not available. The function didn't check whether the required tables existed.
This commit fixes the crash. Please note that support for DT_GNU_HASH is not added, it will be in a future commit.

- The function currently supports Linux and FreeBSD. Only the way the library is opened and some types change.
Because of that, this commit attempts to reduce the amount of duplicate code as much as possible.

- Correct types are now used, e.g. "uint32_t" instead of "int" for the number of chains.

- Other minor changes.

1 files changed, 67 insertions, 77 deletions

diff --git a/overlay_gl/init_unix.c b/overlay_gl/init_unix.c
index 349611c3c..e7655b359 100644
--- a/overlay_gl/init_unix.c
+++ b/overlay_gl/init_unix.c
@@ -6,6 +6,16 @@
 // This file is included by overlay.c for
 // Unix/X11/GLX-specific overlay initialization.
 
+#include <inttypes.h>
+
+#if defined(__linux__)
+// ELF32_ST_TYPE and ELF64_ST_TYPE are the same.
+#	define ELF_ST_TYPE ELF32_ST_TYPE
+
+typedef ElfW(Dyn) Elf_Dyn;
+typedef ElfW(Sym) Elf_Sym;
+#endif
+
 static void initializeLibrary();
 
 #define RESOLVE(x) \
@@ -132,7 +142,7 @@ __attribute__((visibility("default"))) void *dlsym(void *handle, const char *nam
 
 	void *symbol;
 
-	ods("Request for symbol %s (%p:%p)", name, handle, odlsym);
+	ods("Request for symbol; name: %s, handle: %p, odlsym: %p", name, handle, odlsym);
 
 	if (strcmp(name, "glXSwapBuffers") == 0) {
 		OGRAB(glXSwapBuffers);
@@ -142,9 +152,12 @@ __attribute__((visibility("default"))) void *dlsym(void *handle, const char *nam
 		OGRAB(glXGetProcAddressARB);
 	} else if (strcmp(name, "dlsym") == 0) {
 		return (void *) dlsym;
-	} else {
+	} else if (odlsym) {
 		symbol = odlsym(handle, name);
+	} else {
+		return NULL;
 	}
+
 	return symbol;
 }
 
@@ -152,108 +165,85 @@ static int find_odlsym() {
 #if defined(__linux__)
 	void *dl = dlopen("libdl.so.2", RTLD_LAZY);
 	if (!dl) {
-		ods("Failed to open libdl.so.2");
-	} else {
-		int i               = 0;
-		struct link_map *lm = (struct link_map *) dl;
-		int nchains         = 0;
-		ElfW(Sym) *symtab   = NULL;
-		const char *strtab  = NULL;
-#	if defined(__GLIBC__)
-		const ElfW(Addr) base = 0;
-#	else
-		const ElfW(Addr) base = lm->l_addr;
-#	endif
-
-		ElfW(Dyn) *dyn = lm->l_ld;
-
-		while (dyn->d_tag) {
-			switch (dyn->d_tag) {
-				case DT_HASH:
-					nchains = *(int *) (base + dyn->d_un.d_ptr + 4);
-					break;
-				case DT_STRTAB:
-					strtab = (const char *) (base + dyn->d_un.d_ptr);
-					break;
-				case DT_SYMTAB:
-					symtab = (ElfW(Sym) *) (base + dyn->d_un.d_ptr);
-					break;
-			}
-			dyn++;
-		}
-		ods("Iterating dlsym table %p %p %d", symtab, strtab, nchains);
-		for (i = 0; i < nchains; i++) {
-			// ELF32_ST_TYPE and ELF64_ST_TYPE are the same
-			if (ELF32_ST_TYPE(symtab[i].st_info) != STT_FUNC) {
-				continue;
-			}
-			if (strcmp(strtab + symtab[i].st_name, "dlsym") == 0) {
-				odlsym = (void *) lm->l_addr + symtab[i].st_value;
-			}
-		}
-		if (odlsym == NULL) {
-			goto err;
-		}
-		ods("Original dlsym at %p", odlsym);
+		ods("Failed to open libdl.so.2!");
+		return -1;
 	}
 
-	return 0;
+	struct link_map *lm = dl;
 #elif defined(__FreeBSD__)
-	int i               = 0;
 	struct link_map *lm = NULL;
-	int nchains         = 0;
-	Elf_Sym *symtab     = NULL;
-	const char *strtab  = NULL;
-
 	if (dlinfo(RTLD_SELF, RTLD_DI_LINKMAP, &lm) == -1) {
 		ods("Unable to acquire link_map: %s", dlerror());
-		goto err;
+		return -1;
 	}
 
-	while (lm != NULL) {
-		if (!strcmp(lm->l_name, "/libexec/ld-elf.so.1")) {
+	while (lm) {
+		if (strcmp(lm->l_name, "/libexec/ld-elf.so.1") == 0) {
 			break;
 		}
+
 		lm = lm->l_next;
 	}
-	if (lm == NULL) {
-		goto err;
-	}
-
-	Elf_Dyn *dyn = (Elf_Dyn *) lm->l_ld;
 
-	while (dyn->d_tag) {
+	if (!lm) {
+		ods("Failed to find ld-elf.so.1!");
+		return -1;
+	}
+#endif
+	uintptr_t hashTable  = 0;
+	const char *strTable = NULL;
+	Elf_Sym *symTable    = NULL;
+#if defined(__GLIBC__)
+	const uintptr_t base = 0;
+#else
+	const uintptr_t base = (uintptr_t) lm->l_addr;
+#endif
+	for (const Elf_Dyn *dyn = lm->l_ld; dyn; ++dyn) {
 		switch (dyn->d_tag) {
 			case DT_HASH:
-				nchains = *(int *) ((uintptr_t) lm->l_addr + (uintptr_t) dyn->d_un.d_ptr + 4);
+				hashTable = base + dyn->d_un.d_ptr;
 				break;
 			case DT_STRTAB:
-				strtab = (const char *) ((uintptr_t) lm->l_addr + (uintptr_t) dyn->d_un.d_ptr);
+				strTable = (const char *) (base + dyn->d_un.d_ptr);
 				break;
 			case DT_SYMTAB:
-				symtab = (Elf_Sym *) ((uintptr_t) lm->l_addr + (uintptr_t) dyn->d_un.d_ptr);
+				symTable = (Elf_Sym *) (base + dyn->d_un.d_ptr);
 				break;
 		}
-		dyn++;
-	}
-	ods("Iterating dsym table %p %p %d", symtab, strtab, nchains);
-	for (i = 0; i < nchains; i++) {
-		if (ELF_ST_TYPE(symtab[i].st_info) != STT_FUNC) {
-			continue;
+
+		if (hashTable && strTable && symTable) {
+			break;
 		}
-		if (strcmp(strtab + symtab[i].st_name, "dlsym") == 0) {
-			odlsym = (void *) lm->l_addr + symtab[i].st_value;
+	}
+
+	ods("hashTable: 0x%" PRIxPTR ", strTable: %p, symTable: %p", hashTable, strTable, symTable);
+
+	if (hashTable && strTable && symTable) {
+		// Hash table pseudo-struct:
+		// uint32_t nBucket;
+		// uint32_t nChain;
+		// uint32_t bucket[nBucket];
+		// uint32_t chain[nChain];
+		const uint32_t nChain = ((uint32_t *) hashTable)[1];
+
+		for (uint32_t i = 0; i < nChain; ++i) {
+			if (ELF_ST_TYPE(symTable[i].st_info) != STT_FUNC) {
+				continue;
+			}
+
+			if (strcmp(strTable + symTable[i].st_name, "dlsym") == 0) {
+				odlsym = (void *) lm->l_addr + symTable[i].st_value;
+				break;
+			}
 		}
 	}
-	if (odlsym == NULL) {
-		goto err;
+
+	if (!odlsym) {
+		return -1;
 	}
-	ods("Original dlsym at %p", odlsym);
 
+	ods("Original dlsym at %p", odlsym);
 	return 0;
-#endif
-err:
-	return -1;
 }
 
 __attribute__((constructor)) static void initializeLibrary() {