...G...PS. [ZBX-15027] fixed restart failures due to orphaned or zombie processes remaining when zabbix daemon is terminated during startup

9 files changed, 85 insertions, 72 deletions

diff --git a/ChangeLog b/ChangeLog
index 44f547b5d7d..d32cdbf022e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -12,6 +12,7 @@ New features:
 ...G...PS. [ZBXNEXT-4836] added license information and OpenSSL linking exception to README file, show crypto library version when started with '-V' (Andris)
 
 Bug fixes:
+...G...PS. [ZBX-15027] fixed startup failures due to orphaned or zombie processes remaining when zabbix daemon is terminated during startup (vso)
 
 --------------------------------------------------------------------------------
 Changes for 4.0.1
@@ -609,6 +610,7 @@ New features:
 .......PS. [ZBXNEXT-4782] improve out of memory error message by adding statistics and backtrace; improve something impossible has just happened error message by adding backtrace (vso)
 
 Bug fixes:
+...G...PS. [ZBX-15027] fixed startup failures due to orphaned or zombie processes remaining when zabbix daemon is terminated during startup (vso)
 A......... [ZBX-14396] fixed memory usage during template full clone (gcalenko, Sasha)
 ...G...... [ZBX-14779] extended support of system.stat[ent], system.stat[cpu,pc], system.stat[cpu,ec] on IBM AIX to LPAR type 'dedicated' (Andris)
 ........S. [ZBX-14851] fixed the host visible name in the event details/messages from server when using long utf8 text (MVekslers)
@@ -1990,6 +1992,7 @@ New features:
 ...G...PS. [ZBXNEXT-4836] added license information and OpenSSL linking exception to README file, show crypto library version when started with '-V' (Andris)
 
 Bug fixes:
+...G...PS. [ZBX-15027] fixed startup failures due to orphaned or zombie processes remaining when zabbix daemon is terminated during startup (vso)
 
 --------------------------------------------------------------------------------
 Changes for 3.0.23
diff --git a/include/threads.h b/include/threads.h
index 4f0e09c7be2..89f2bf53529 100644
--- a/include/threads.h
+++ b/include/threads.h
@@ -48,7 +48,7 @@
 #else	/* not _WINDOWS */
 
 	int	zbx_fork(void);
-	int	zbx_child_fork(void);
+	void	zbx_child_fork(pid_t *pid);
 
 	#define ZBX_THREAD_ERROR	-1
 
@@ -84,8 +84,8 @@ typedef struct
 }
 zbx_thread_args_t;
 
-ZBX_THREAD_HANDLE	zbx_thread_start(ZBX_THREAD_ENTRY_POINTER(handler), zbx_thread_args_t *thread_args);
-int			zbx_thread_wait(ZBX_THREAD_HANDLE thread);
+void	zbx_thread_start(ZBX_THREAD_ENTRY_POINTER(handler), zbx_thread_args_t *thread_args, ZBX_THREAD_HANDLE *thread);
+int	zbx_thread_wait(ZBX_THREAD_HANDLE thread);
 void			zbx_threads_wait(ZBX_THREAD_HANDLE *threads, int threads_num);
 /* zbx_thread_exit(status) -- declared as define !!! */
 long int		zbx_get_thread_id(void);
diff --git a/src/libs/zbxsys/threads.c b/src/libs/zbxsys/threads.c
index 4a78d6d5e11..b32240dff19 100644
--- a/src/libs/zbxsys/threads.c
+++ b/src/libs/zbxsys/threads.c
@@ -53,9 +53,8 @@ int	zbx_fork(void)
  * Comments: use this function only for forks from the main process           *
  *                                                                            *
  ******************************************************************************/
-int	zbx_child_fork(void)
+void	zbx_child_fork(pid_t *pid)
 {
-	pid_t		pid;
 	sigset_t	mask, orig_mask;
 
 	/* block SIGTERM, SIGINT and SIGCHLD during fork to avoid deadlock (we've seen one in __unregister_atfork()) */
@@ -65,15 +64,14 @@ int	zbx_child_fork(void)
 	sigaddset(&mask, SIGCHLD);
 	sigprocmask(SIG_BLOCK, &mask, &orig_mask);
 
-	pid = zbx_fork();
+	/* set process id instead of returning, this is to avoid race condition when signal arrives before return */
+	*pid = zbx_fork();
 
 	sigprocmask(SIG_SETMASK, &orig_mask, NULL);
 
 	/* ignore SIGCHLD to avoid problems with exiting scripts in zbx_execute() and other cases */
-	if (0 == pid)
+	if (0 == *pid)
 		signal(SIGCHLD, SIG_DFL);
-
-	return pid;
 }
 #else
 int	zbx_win_exception_filter(unsigned int code, struct _EXCEPTION_POINTERS *ep);
@@ -104,46 +102,46 @@ void CALLBACK	ZBXEndThread(ULONG_PTR dwParam)
  *                                                                            *
  * Purpose: Start the handled function as "thread"                            *
  *                                                                            *
- * Parameters: "thread" handle                                                *
- *                                                                            *
- * Return value: returns a handle to the newly created "thread",              *
- *               ZBX_THREAD_ERROR on an error                                 *
+ * Parameters: handler     - [IN] new thread starts execution from this       *
+ *                                handler function                            *
+ *             thread_args - [IN] arguments for thread function               *
+ *             thread      - [OUT] handle to a newly created thread           *
  *                                                                            *
  * Author: Eugene Grigorjev                                                   *
  *                                                                            *
  * Comments: The zbx_thread_exit must be called from the handler!             *
  *                                                                            *
  ******************************************************************************/
-ZBX_THREAD_HANDLE	zbx_thread_start(ZBX_THREAD_ENTRY_POINTER(handler), zbx_thread_args_t *thread_args)
+void	zbx_thread_start(ZBX_THREAD_ENTRY_POINTER(handler), zbx_thread_args_t *thread_args, ZBX_THREAD_HANDLE *thread)
 {
-	ZBX_THREAD_HANDLE	thread = ZBX_THREAD_HANDLE_NULL;
 #ifdef _WINDOWS
 	unsigned		thrdaddr;
 
 	thread_args->entry = handler;
 	/* NOTE: _beginthreadex returns 0 on failure, rather than 1 */
-	if (0 == (thread = (ZBX_THREAD_HANDLE)_beginthreadex(NULL, 0, zbx_win_thread_entry, thread_args, 0, &thrdaddr)))
+	if (0 == (*thread = (ZBX_THREAD_HANDLE)_beginthreadex(NULL, 0, zbx_win_thread_entry, thread_args, 0, &thrdaddr)))
 	{
 		zabbix_log(LOG_LEVEL_CRIT, "failed to create a thread: %s", strerror_from_system(GetLastError()));
-		thread = (ZBX_THREAD_HANDLE)ZBX_THREAD_ERROR;
+		*thread = (ZBX_THREAD_HANDLE)ZBX_THREAD_ERROR;
 	}
 #else
-	if (0 == (thread = zbx_child_fork()))	/* child process */
+	zbx_child_fork(thread);
+
+	if (0 == *thread)	/* child process */
 	{
 		(*handler)(thread_args);
 
 		/* The zbx_thread_exit must be called from the handler. */
 		/* And in normal case the program will never reach this point. */
-		zbx_thread_exit(EXIT_SUCCESS);
+		THIS_SHOULD_NEVER_HAPPEN;
 		/* program will never reach this point */
 	}
-	else if (-1 == thread)
+	else if (-1 == *thread)
 	{
 		zbx_error("failed to fork: %s", zbx_strerror(errno));
-		thread = (ZBX_THREAD_HANDLE)ZBX_THREAD_ERROR;
+		*thread = (ZBX_THREAD_HANDLE)ZBX_THREAD_ERROR;
 	}
 #endif
-	return thread;
 }
 
 /******************************************************************************
diff --git a/src/libs/zbxsysinfo/sysinfo.c b/src/libs/zbxsysinfo/sysinfo.c
index aea1c16e5dd..957d9ef661c 100644
--- a/src/libs/zbxsysinfo/sysinfo.c
+++ b/src/libs/zbxsysinfo/sysinfo.c
@@ -1410,7 +1410,9 @@ int	zbx_execute_threaded_metric(zbx_metric_func_t metric_func, AGENT_REQUEST *re
 
 	args.args = (void *)&metric_args;
 
-	if (ZBX_THREAD_ERROR == (thread = zbx_thread_start(agent_metric_thread, &args)))
+	zbx_thread_start(agent_metric_thread, &args, &thread);
+
+	if (ZBX_THREAD_ERROR == thread)
 	{
 		SET_MSG_RESULT(result, zbx_dsprintf(NULL, "Cannot start data thread: %s",
 				strerror_from_system(GetLastError())));
diff --git a/src/zabbix_agent/zabbix_agentd.c b/src/zabbix_agent/zabbix_agentd.c
index 627d55456b7..8016a6ee38e 100644
--- a/src/zabbix_agent/zabbix_agentd.c
+++ b/src/zabbix_agent/zabbix_agentd.c
@@ -1039,15 +1039,15 @@ int	MAIN_ZABBIX_ENTRY(int flags)
 		switch (thread_args->process_type)
 		{
 			case ZBX_PROCESS_TYPE_COLLECTOR:
-				threads[i] = zbx_thread_start(collector_thread, thread_args);
+				zbx_thread_start(collector_thread, thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_LISTENER:
 				thread_args->args = &listen_sock;
-				threads[i] = zbx_thread_start(listener_thread, thread_args);
+				zbx_thread_start(listener_thread, thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_ACTIVE_CHECKS:
 				thread_args->args = &CONFIG_ACTIVE_ARGS[j++];
-				threads[i] = zbx_thread_start(active_checks_thread, thread_args);
+				zbx_thread_start(active_checks_thread, thread_args, &threads[i]);
 				break;
 		}
 #ifndef _WINDOWS
diff --git a/src/zabbix_proxy/proxy.c b/src/zabbix_proxy/proxy.c
index 72e522a1a62..b706fbf8ed4 100644
--- a/src/zabbix_proxy/proxy.c
+++ b/src/zabbix_proxy/proxy.c
@@ -1072,67 +1072,67 @@ int	MAIN_ZABBIX_ENTRY(int flags)
 		switch (thread_args.process_type)
 		{
 			case ZBX_PROCESS_TYPE_CONFSYNCER:
-				threads[i] = zbx_thread_start(proxyconfig_thread, &thread_args);
+				zbx_thread_start(proxyconfig_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_HEARTBEAT:
-				threads[i] = zbx_thread_start(heart_thread, &thread_args);
+				zbx_thread_start(heart_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_DATASENDER:
-				threads[i] = zbx_thread_start(datasender_thread, &thread_args);
+				zbx_thread_start(datasender_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_POLLER:
 				poller_type = ZBX_PROCESS_TYPE_POLLER;
 				thread_args.args = &poller_type;
-				threads[i] = zbx_thread_start(poller_thread, &thread_args);
+				zbx_thread_start(poller_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_UNREACHABLE:
 				poller_type = ZBX_PROCESS_TYPE_UNREACHABLE;
 				thread_args.args = &poller_type;
-				threads[i] = zbx_thread_start(poller_thread, &thread_args);
+				zbx_thread_start(poller_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_TRAPPER:
 				thread_args.args = &listen_sock;
-				threads[i] = zbx_thread_start(trapper_thread, &thread_args);
+				zbx_thread_start(trapper_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_PINGER:
-				threads[i] = zbx_thread_start(pinger_thread, &thread_args);
+				zbx_thread_start(pinger_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_HOUSEKEEPER:
-				threads[i] = zbx_thread_start(housekeeper_thread, &thread_args);
+				zbx_thread_start(housekeeper_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_HTTPPOLLER:
-				threads[i] = zbx_thread_start(httppoller_thread, &thread_args);
+				zbx_thread_start(httppoller_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_DISCOVERER:
-				threads[i] = zbx_thread_start(discoverer_thread, &thread_args);
+				zbx_thread_start(discoverer_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_HISTSYNCER:
-				threads[i] = zbx_thread_start(dbsyncer_thread, &thread_args);
+				zbx_thread_start(dbsyncer_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_JAVAPOLLER:
 				poller_type = ZBX_PROCESS_TYPE_JAVAPOLLER;
 				thread_args.args = &poller_type;
-				threads[i] = zbx_thread_start(poller_thread, &thread_args);
+				zbx_thread_start(poller_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_SNMPTRAPPER:
-				threads[i] = zbx_thread_start(snmptrapper_thread, &thread_args);
+				zbx_thread_start(snmptrapper_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_SELFMON:
-				threads[i] = zbx_thread_start(selfmon_thread, &thread_args);
+				zbx_thread_start(selfmon_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_VMWARE:
-				threads[i] = zbx_thread_start(vmware_thread, &thread_args);
+				zbx_thread_start(vmware_thread, &thread_args, &threads[i]);
 				break;
 #ifdef HAVE_OPENIPMI
 			case ZBX_PROCESS_TYPE_IPMIMANAGER:
-				threads[i] = zbx_thread_start(ipmi_manager_thread, &thread_args);
+				zbx_thread_start(ipmi_manager_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_IPMIPOLLER:
-				threads[i] = zbx_thread_start(ipmi_poller_thread, &thread_args);
+				zbx_thread_start(ipmi_poller_thread, &thread_args, &threads[i]);
 				break;
 #endif
 			case ZBX_PROCESS_TYPE_TASKMANAGER:
-				threads[i] = zbx_thread_start(taskmanager_thread, &thread_args);
+				zbx_thread_start(taskmanager_thread, &thread_args, &threads[i]);
 				break;
 		}
 	}
diff --git a/src/zabbix_sender/zabbix_sender.c b/src/zabbix_sender/zabbix_sender.c
index 7c7ba35a346..1b3fb5c11fa 100644
--- a/src/zabbix_sender/zabbix_sender.c
+++ b/src/zabbix_sender/zabbix_sender.c
@@ -1010,6 +1010,7 @@ int	main(int argc, char **argv)
 	const char		*p;
 	zbx_thread_args_t	thread_args;
 	ZBX_THREAD_SENDVAL_ARGS sendval_args;
+	ZBX_THREAD_HANDLE	thread;
 
 	progname = get_program_name(argv[0]);
 
@@ -1230,8 +1231,8 @@ int	main(int argc, char **argv)
 				zbx_json_close(&sendval_args.json);
 
 				last_send = zbx_time();
-
-				ret = update_exit_status(ret, zbx_thread_wait(zbx_thread_start(send_value, &thread_args)));
+				zbx_thread_start(send_value, &thread_args, &thread);
+				ret = update_exit_status(ret, zbx_thread_wait(thread));
 
 				buffer_count = 0;
 				zbx_json_clean(&sendval_args.json);
@@ -1244,7 +1245,8 @@ int	main(int argc, char **argv)
 		if (FAIL != ret && 0 != buffer_count)
 		{
 			zbx_json_close(&sendval_args.json);
-			ret = update_exit_status(ret, zbx_thread_wait(zbx_thread_start(send_value, &thread_args)));
+			zbx_thread_start(send_value, &thread_args, &thread);
+			ret = update_exit_status(ret, zbx_thread_wait(thread));
 		}
 
 		if (in != stdin)
@@ -1285,8 +1287,8 @@ int	main(int argc, char **argv)
 			zbx_json_close(&sendval_args.json);
 
 			succeed_count++;
-
-			ret = update_exit_status(ret, zbx_thread_wait(zbx_thread_start(send_value, &thread_args)));
+			zbx_thread_start(send_value, &thread_args, &thread);
+			ret = update_exit_status(ret, zbx_thread_wait(thread));
 		}
 		while (0); /* try block simulation */
 	}
diff --git a/src/zabbix_server/poller/checks_snmp.c b/src/zabbix_server/poller/checks_snmp.c
index 40e5a86a5ab..916373ca1f5 100644
--- a/src/zabbix_server/poller/checks_snmp.c
+++ b/src/zabbix_server/poller/checks_snmp.c
@@ -2151,7 +2151,15 @@ out:
 
 void	zbx_init_snmp(void)
 {
+	sigset_t	mask, orig_mask;
+
+	sigemptyset(&mask);
+	sigaddset(&mask, SIGTERM);
+	sigprocmask(SIG_BLOCK, &mask, &orig_mask);
+
 	init_snmp(progname);
+
+	sigprocmask(SIG_SETMASK, &orig_mask, NULL);
 }
 
 #endif	/* HAVE_NETSNMP */
diff --git a/src/zabbix_server/server.c b/src/zabbix_server/server.c
index 7e9ab9cf0a1..1b060c0deb6 100644
--- a/src/zabbix_server/server.c
+++ b/src/zabbix_server/server.c
@@ -1101,82 +1101,82 @@ int	MAIN_ZABBIX_ENTRY(int flags)
 		switch (thread_args.process_type)
 		{
 			case ZBX_PROCESS_TYPE_CONFSYNCER:
-				threads[i] = zbx_thread_start(dbconfig_thread, &thread_args);
+				zbx_thread_start(dbconfig_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_POLLER:
 				poller_type = ZBX_PROCESS_TYPE_POLLER;
 				thread_args.args = &poller_type;
-				threads[i] = zbx_thread_start(poller_thread, &thread_args);
+				zbx_thread_start(poller_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_UNREACHABLE:
 				poller_type = ZBX_PROCESS_TYPE_UNREACHABLE;
 				thread_args.args = &poller_type;
-				threads[i] = zbx_thread_start(poller_thread, &thread_args);
+				zbx_thread_start(poller_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_TRAPPER:
 				thread_args.args = &listen_sock;
-				threads[i] = zbx_thread_start(trapper_thread, &thread_args);
+				zbx_thread_start(trapper_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_PINGER:
-				threads[i] = zbx_thread_start(pinger_thread, &thread_args);
+				zbx_thread_start(pinger_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_ALERTER:
-				threads[i] = zbx_thread_start(alerter_thread, &thread_args);
+				zbx_thread_start(alerter_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_HOUSEKEEPER:
-				threads[i] = zbx_thread_start(housekeeper_thread, &thread_args);
+				zbx_thread_start(housekeeper_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_TIMER:
-				threads[i] = zbx_thread_start(timer_thread, &thread_args);
+				zbx_thread_start(timer_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_HTTPPOLLER:
-				threads[i] = zbx_thread_start(httppoller_thread, &thread_args);
+				zbx_thread_start(httppoller_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_DISCOVERER:
-				threads[i] = zbx_thread_start(discoverer_thread, &thread_args);
+				zbx_thread_start(discoverer_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_HISTSYNCER:
-				threads[i] = zbx_thread_start(dbsyncer_thread, &thread_args);
+				zbx_thread_start(dbsyncer_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_ESCALATOR:
-				threads[i] = zbx_thread_start(escalator_thread, &thread_args);
+				zbx_thread_start(escalator_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_JAVAPOLLER:
 				poller_type = ZBX_PROCESS_TYPE_JAVAPOLLER;
 				thread_args.args = &poller_type;
-				threads[i] = zbx_thread_start(poller_thread, &thread_args);
+				zbx_thread_start(poller_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_SNMPTRAPPER:
-				threads[i] = zbx_thread_start(snmptrapper_thread, &thread_args);
+				zbx_thread_start(snmptrapper_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_PROXYPOLLER:
-				threads[i] = zbx_thread_start(proxypoller_thread, &thread_args);
+				zbx_thread_start(proxypoller_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_SELFMON:
-				threads[i] = zbx_thread_start(selfmon_thread, &thread_args);
+				zbx_thread_start(selfmon_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_VMWARE:
-				threads[i] = zbx_thread_start(vmware_thread, &thread_args);
+				zbx_thread_start(vmware_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_TASKMANAGER:
-				threads[i] = zbx_thread_start(taskmanager_thread, &thread_args);
+				zbx_thread_start(taskmanager_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_PREPROCMAN:
-				threads[i] = zbx_thread_start(preprocessing_manager_thread, &thread_args);
+				zbx_thread_start(preprocessing_manager_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_PREPROCESSOR:
-				threads[i] = zbx_thread_start(preprocessing_worker_thread, &thread_args);
+				zbx_thread_start(preprocessing_worker_thread, &thread_args, &threads[i]);
 				break;
 #ifdef HAVE_OPENIPMI
 			case ZBX_PROCESS_TYPE_IPMIMANAGER:
-				threads[i] = zbx_thread_start(ipmi_manager_thread, &thread_args);
+				zbx_thread_start(ipmi_manager_thread, &thread_args, &threads[i]);
 				break;
 			case ZBX_PROCESS_TYPE_IPMIPOLLER:
-				threads[i] = zbx_thread_start(ipmi_poller_thread, &thread_args);
+				zbx_thread_start(ipmi_poller_thread, &thread_args, &threads[i]);
 				break;
 #endif
 			case ZBX_PROCESS_TYPE_ALERTMANAGER:
-				threads[i] = zbx_thread_start(alert_manager_thread, &thread_args);
+				zbx_thread_start(alert_manager_thread, &thread_args, &threads[i]);
 				break;
 		}
 	}