2006-02-26 Dan Williams <dcbw@redhat.com>

* src/Makefile.am
		- make and install nm-crash-logger

	* src/nm-logging.[ch]
	  	- New files; consolidate logging and crash handling

	* src/nm-crash-logger.c
	  src/gdb-cmd
		- Standalong crashlogger for NM, grab a backtrace
			using GDB

	* src/NetworkManager.[ch]
		- Remove signal handling and put it into nm-logging.c


git-svn-id: http://svn-archive.gnome.org/svn/NetworkManager/trunk@1499 4912f4e0-d625-0410-9fb7-b9a5a253dbdc

ChangeLog

@@ -1,3 +1,19 @@
+2006-02-26  Dan Williams  <dcbw@redhat.com>
+
+	* src/Makefile.am
+		- make and install nm-crash-logger
+
+	* src/nm-logging.[ch]
+	  	- New files; consolidate logging and crash handling
+
+	* src/nm-crash-logger.c
+	  src/gdb-cmd
+		- Standalong crashlogger for NM, grab a backtrace
+			using GDB
+
+	* src/NetworkManager.[ch]
+		- Remove signal handling and put it into nm-logging.c
+
 2006-02-26  Dan Williams  <dcbw@redhat.com>
 
 	* configure.in

src/Makefile.am

@@ -45,6 +45,8 @@ NetworkManager_SOURCES =				\
 		NetworkManagerUtils.h		\
 		NetworkManagerSystem.c		\
 		NetworkManagerSystem.h		\
+		nm-logging.c				\
+		nm-logging.h				\
 		nm-netlink-monitor.c		\
 		nm-netlink-monitor.h		\
 		nm-activation-request.c		\
@@ -82,6 +84,8 @@ NetworkManager_CPPFLAGS = 							\
 	-DDBUS_API_SUBJECT_TO_CHANGE						\
 	-DG_DISABLE_DEPRECATED							\
 	-DBINDIR=\"$(bindir)\"							\
+	-DSBINDIR=\"$(sbindir)\"							\
+	-DLIBEXECDIR=\"$(libexecdir)\"					\
 	-DDATADIR=\"$(datadir)\"							\
 	-DSYSCONFDIR=\"$(sysconfdir)\"					\
 	-DLOCALSTATEDIR=\"$(localstatedir)\"				\
@@ -102,9 +106,25 @@ NetworkManager_LDADD = 							\
 			./backends/libnmbackend.la			\
 			$(top_builddir)/libnm-util/libnm-util.la
 
+NetworkManager_LDFLAGS = -rdynamic
 
+libexec_PROGRAMS = nm-crash-logger
+nm_crash_logger_SOURCES = nm-crash-logger.c
+nm_crash_logger_CPPFLAGS = \
+	$(GTHREAD_CFLAGS)								\
+	-DG_DISABLE_DEPRECATED							\
+	-DBINDIR=\"$(bindir)\"							\
+	-DSBINDIR=\"$(sbindir)\"							\
+	-DDATADIR=\"$(datadir)\"							\
+	-DSYSCONFDIR=\"$(sysconfdir)\"					\
+	-DLOCALSTATEDIR=\"$(localstatedir)\"
+nm_crash_logger_LDADD = $(GTHREAD_LIBS)
+
+
+NetworkManagerdir = $(datadir)/NetworkManager
+NetworkManager_DATA = gdb-cmd
 
 dbusservicedir = $(DBUS_SYS_DIR)
 dbusservice_DATA = NetworkManager.conf
 
-EXTRA_DIST = $(dbusservice_DATA)
+EXTRA_DIST = $(dbusservice_DATA) $(NetworkManager_DATA)

src/NetworkManager.c

@@ -49,6 +49,7 @@
 #include "nm-dbus-vpn.h"
 #include "nm-netlink-monitor.h"
 #include "nm-dhcp-manager.h"
+#include "nm-logging.h"
 
 #define NM_WIRELESS_LINK_STATE_POLL_INTERVAL (5 * 1000)
 
@@ -58,7 +59,6 @@
 static NMData		*nm_data = NULL;
 
 static gboolean sigterm_pipe_handler (GIOChannel *src, GIOCondition condition, gpointer data);
-static void sigterm_handler (int signum);
 static void nm_data_free (NMData *data);
 
 /*
@@ -376,8 +376,6 @@ void nm_schedule_state_change_signal_broadcast (NMData *data)
  */
 static NMData *nm_data_new (gboolean enable_test_devices)
 {
-	struct sigaction	action;
-	sigset_t			block_mask;
 	NMData *			data;
 	GSource *			iosource;
 	
@@ -386,29 +384,21 @@ static NMData *nm_data_new (gboolean enable_test_devices)
 	data->main_context = g_main_context_new ();
 	data->main_loop = g_main_loop_new (data->main_context, FALSE);
 
+	/* Allow clean shutdowns by having the thread which receives the signal
+	 * notify the main thread to quit, rather than having the receiving
+	 * thread try to quit the glib main loop.
+	 */
 	if (pipe (data->sigterm_pipe) < 0)
 	{
 		nm_error ("Couldn't create pipe: %s", g_strerror (errno));
 		return NULL;
 	}
-
 	data->sigterm_iochannel = g_io_channel_unix_new (data->sigterm_pipe[0]);
 	iosource = g_io_create_watch (data->sigterm_iochannel, G_IO_IN | G_IO_ERR);
 	g_source_set_callback (iosource, (GSourceFunc) sigterm_pipe_handler, data, NULL);
 	g_source_attach (iosource, data->main_context);
 	g_source_unref (iosource);
 
-	action.sa_sigaction = NULL;
-	action.sa_handler = sigterm_handler;
-	sigemptyset (&block_mask);
-	action.sa_mask = block_mask;
-	action.sa_flags = 0;
-	if (sigaction (SIGINT, &action, NULL) || sigaction (SIGTERM, &action, NULL))
-	{
-		nm_error ("Failed to install signal handlers: %s", g_strerror (errno));
-		return NULL;
-	}
-
 	/* Initialize the device list mutex to protect additions/deletions to it. */
 	data->dev_list_mutex = g_mutex_new ();
 	data->dialup_list_mutex = g_mutex_new ();
@@ -488,21 +478,12 @@ static void nm_data_free (NMData *data)
 
 	nm_hal_deinit (data);
 
-	closelog ();
-
 	memset (data, 0, sizeof (NMData));
 }
 
-static void sigterm_handler (int signum)
+int nm_get_sigterm_pipe (void)
 {
-	int ignore;
+	return nm_data->sigterm_pipe[1];
-
-	/* FIXME: This is probably not a great thing to have in a signal handler,
-	 * but you only live once.
-	 */
-	nm_info ("Caught %s", signum == SIGINT ? "SIGINT" : "SIGTERM");
-
-	ignore = write (nm_data->sigterm_pipe[1], "X", 1);
 }
 
 static gboolean sigterm_pipe_handler (GIOChannel *src, GIOCondition condition, gpointer user_data)
@@ -628,58 +609,6 @@ nm_monitor_wired_link_state (NMData *data)
 	data->netlink_monitor = monitor;
 }
 
-static void
-nm_info_handler (const gchar		*log_domain,
-			  GLogLevelFlags	 log_level,
-			  const gchar		*message,
-			  gpointer		 ignored)
-{
-	int syslog_priority;	
-
-	switch (log_level)
-	{
-		case G_LOG_LEVEL_ERROR:
-			syslog_priority = LOG_CRIT;
-		break;
-
-		case G_LOG_LEVEL_CRITICAL:
-			syslog_priority = LOG_ERR;
-		break;
-
-		case G_LOG_LEVEL_WARNING:
-			syslog_priority = LOG_WARNING;
-		break;
-
-		case G_LOG_LEVEL_MESSAGE:
-			syslog_priority = LOG_NOTICE;
-
-		case G_LOG_LEVEL_DEBUG:
-			syslog_priority = LOG_DEBUG;
-		break;
-
-		case G_LOG_LEVEL_INFO:
-		default:
-			syslog_priority = LOG_INFO;
-		break;
-	}
-
-	syslog (syslog_priority, "%s", message);
-}
-
-static void
-nm_set_up_log_handlers (gboolean become_daemon)
-{
-	if (become_daemon)
-		openlog (G_LOG_DOMAIN, LOG_CONS, LOG_DAEMON);
-	else
-		openlog (G_LOG_DOMAIN, LOG_CONS | LOG_PERROR, LOG_USER);
-
-	g_log_set_handler (G_LOG_DOMAIN, 
-				    G_LOG_LEVEL_MASK,
-				    nm_info_handler,
-				    NULL);
-}
-
 
 static LibHalContext *nm_get_hal_ctx (NMData *data)
 {
@@ -827,7 +756,7 @@ int main( int argc, char *argv[] )
 		g_thread_init (NULL);
 	dbus_g_thread_init ();
 
-	nm_set_up_log_handlers (become_daemon);
+	nm_logging_setup (become_daemon);
 	nm_info ("starting...");
 
 	nm_system_init();
@@ -895,6 +824,7 @@ int main( int argc, char *argv[] )
 
 	nm_print_open_socks ();
 	nm_data_free (nm_data);
+	nm_logging_shutdown ();
 
 	exit (0);
 }

src/NetworkManagerMain.h

@@ -98,14 +98,16 @@ NMDevice *	nm_get_active_device					(NMData *data);
 NMDevice *	nm_create_device_and_add_to_list			(NMData *data, const char *udi, const char *iface,
 														gboolean test_device, NMDeviceType test_device_type);
 
-void				nm_add_initial_devices					(NMData *data);
+void			nm_add_initial_devices					(NMData *data);
 
-void				nm_remove_device						(NMData *data, NMDevice *dev);
+void			nm_remove_device						(NMData *data, NMDevice *dev);
 
-void				nm_schedule_state_change_signal_broadcast	(NMData *data);
+void			nm_schedule_state_change_signal_broadcast	(NMData *data);
 
-void				nm_hal_init							(NMData *data);
+void			nm_hal_init							(NMData *data);
 
-void				nm_hal_deinit							(NMData *data);
+void			nm_hal_deinit							(NMData *data);
+
+int			nm_get_sigterm_pipe						(void);
 
 #endif

src/gdb-cmd

@@ -0,0 +1,3 @@
+bt
+thread apply all bt full
+q

src/nm-crash-logger.c

@@ -0,0 +1,87 @@
+/* NetworkManager -- Network link manager
+ *
+ * Dan Williams <dcbw@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * (C) Copyright 2006 Red Hat, Inc.
+ */
+
+
+#include <glib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sys/wait.h>
+#include <stdlib.h>
+#include <syslog.h>
+
+int main (int argc, char ** argv)
+{
+	GPid		gdb_pid;
+	int		out;
+	char		nm_pid[16];
+	char		line[256];
+	int		gdb_stat;
+	int		bytes_read;
+	gboolean	done = FALSE;
+	char *	args[] = { BINDIR "/gdb",
+                          "--batch", 
+                          "--quiet",
+                          "--command=" DATADIR "/NetworkManager/gdb-cmd",
+                          SBINDIR "/NetworkManager",
+                          NULL, NULL };
+
+	snprintf (nm_pid, sizeof (nm_pid), "%d", getppid ());
+	args[5] = &nm_pid[0];
+	if (!g_spawn_async_with_pipes (NULL, args, NULL, G_SPAWN_DO_NOT_REAP_CHILD, NULL, NULL,
+				       &gdb_pid, NULL, &out, NULL, NULL))
+	{
+		exit (1);
+	}
+
+	openlog ("NetworkManager", LOG_CONS | LOG_PERROR, LOG_DAEMON);
+	syslog (LOG_CRIT, "******************* START **********************************");
+	while (!done)
+	{
+		bytes_read = read (out, line, sizeof (line) - 1);
+		if (bytes_read > 0)
+		{
+			char *end = &line[0];
+			char *start = &line[0];
+
+			/* Can't just funnel the output to syslog, have to do a separate
+			 * syslog () for each line in the output.
+			 */
+			line[bytes_read] = '\0';
+			while (*end != '\0')
+			{
+				if (*end == '\n')
+				{
+					*end = '\0';
+					syslog (LOG_CRIT, "%s", start);
+					start = end + 1;
+				}
+				end++;
+			}
+		}
+		else if ((bytes_read <= 0) || ((errno != EINTR) && (errno != EAGAIN)))
+			done = TRUE;
+	}
+	syslog (LOG_CRIT, "******************* END **********************************");
+	close (out);
+	waitpid (gdb_pid, &gdb_stat, 0);
+	exit (0);
+}

src/nm-logging.c

@@ -0,0 +1,250 @@
+/* NetworkManager -- Network link manager
+ *
+ * Dan Williams <dcbw@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * (C) Copyright 2006 Red Hat, Inc.
+ */
+
+#include <syslog.h>
+#include <execinfo.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <signal.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sys/wait.h>
+#include <sys/stat.h>
+
+#include "nm-logging.h"
+#include "nm-utils.h"
+#include "NetworkManagerMain.h"
+
+static void
+fallback_get_backtrace (void)
+{
+	void *	frames[64];
+	size_t	size;
+	char **	strings;
+	size_t	i;
+
+	size = backtrace (frames, G_N_ELEMENTS (frames));
+	if ((strings = backtrace_symbols (frames, size)))
+	{
+		syslog (LOG_CRIT, "******************* START **********************************");
+		for (i = 0; i < size; i++)
+			syslog (LOG_CRIT, "Frame %d: %s", i, strings[i]);
+		free (strings);
+		syslog (LOG_CRIT, "******************* END **********************************");
+	}
+	else
+	{
+		nm_warning ("NetworkManager crashed, but symbols "
+					"couldn't be retrieved.");
+	}
+}
+
+
+static gboolean
+crashlogger_get_backtrace (void)
+{
+	gboolean success = FALSE;
+	int pid;	
+
+	pid = fork();
+	if (pid > 0)
+	{
+		/* Wait for the child to finish */
+		int estatus;
+		if (waitpid (pid, &estatus, 0) != -1)
+		{
+			/* Only succeed if the crashlogger succeeded */
+			if (WIFEXITED (estatus) && (WEXITSTATUS (estatus) == 0))
+				success = TRUE;
+		}
+	}
+	else if (pid == 0)
+	{
+		/* Child process */
+		execl (LIBEXECDIR"/nm-crash-logger",
+				LIBEXECDIR"/nm-crash-logger", NULL);
+	}
+
+	return success;
+}
+
+
+static void
+nm_logging_backtrace (void)
+{
+	struct stat s;
+	gboolean fallback = TRUE;
+	
+	/* Try to use gdb via nm-crash-logger if it exists, since
+	 * we get much better information out of it.  Otherwise
+	 * fall back to execinfo.
+	 */
+	if (stat (LIBEXECDIR"/nm-crash-logger", &s) == 0)
+		fallback = crashlogger_get_backtrace () ? FALSE : TRUE;
+
+	if (fallback)
+		fallback_get_backtrace ();
+}
+
+
+static void
+nm_log_handler (const gchar *		log_domain,
+			  GLogLevelFlags	log_level,
+			  const gchar *	message,
+			  gpointer		ignored)
+{
+	int syslog_priority;	
+
+	switch (log_level)
+	{
+		case G_LOG_LEVEL_ERROR:
+			syslog_priority = LOG_CRIT;
+			break;
+
+		case G_LOG_LEVEL_CRITICAL:
+			syslog_priority = LOG_ERR;
+			break;
+
+		case G_LOG_LEVEL_WARNING:
+			syslog_priority = LOG_WARNING;
+			break;
+
+		case G_LOG_LEVEL_MESSAGE:
+			syslog_priority = LOG_NOTICE;
+			break;
+
+		case G_LOG_LEVEL_DEBUG:
+			syslog_priority = LOG_DEBUG;
+			break;
+
+		case G_LOG_LEVEL_INFO:
+		default:
+			syslog_priority = LOG_INFO;
+			break;
+	}
+
+	syslog (syslog_priority, "%s", message);
+}
+
+
+static void
+nm_signal_handler (int signo)
+{
+	static int in_fatal = 0;
+	int ignore;
+
+	/* avoid loops */
+	if (in_fatal > 0)
+		return;
+	++in_fatal;
+
+	switch (signo)
+	{
+		case SIGSEGV:
+		case SIGBUS:
+		case SIGILL:
+		case SIGABRT:
+			nm_warning ("Caught signal %d.  Generating backtrace...", signo);
+			nm_logging_backtrace ();
+			exit (1);
+			break;
+
+		case SIGFPE:
+		case SIGPIPE:
+			/* let the fatal signals interrupt us */
+			--in_fatal;
+
+			nm_warning ("Caught signal %d, shutting down abnormally.  Generating backtrace...", signo);
+			nm_logging_backtrace ();
+			ignore = write (nm_get_sigterm_pipe (), "X", 1);
+			break;
+
+		case SIGINT:
+		case SIGTERM:
+			/* let the fatal signals interrupt us */
+			--in_fatal;
+
+			nm_warning ("Caught signal %d, shutting down normally.", signo);
+			ignore = write (nm_get_sigterm_pipe (), "X", 1);
+			break;
+
+		case SIGHUP:
+			--in_fatal;
+			/* FIXME:
+			 * Reread config stuff like system config files, VPN service files, etc
+			 */
+			break;
+
+		case SIGUSR1:
+			--in_fatal;
+			/* FIXME:
+			 * Play with log levels or something
+			 */
+			break;
+
+		default:
+			signal (signo, nm_signal_handler);
+			break;
+	}
+}
+
+static void
+setup_signals (void)
+{
+	struct sigaction action;
+	sigset_t mask;
+
+	sigemptyset (&mask);
+	action.sa_handler = nm_signal_handler;
+	action.sa_mask = mask;
+	action.sa_flags = 0;
+	sigaction (SIGTERM,  &action, NULL);
+	sigaction (SIGINT,  &action, NULL);
+	sigaction (SIGILL,  &action, NULL);
+	sigaction (SIGBUS,  &action, NULL);
+	sigaction (SIGFPE,  &action, NULL);
+	sigaction (SIGHUP,  &action, NULL);
+	sigaction (SIGSEGV, &action, NULL);
+	sigaction (SIGABRT, &action, NULL);
+	sigaction (SIGUSR1,  &action, NULL);
+}
+
+void
+nm_logging_setup (gboolean become_daemon)
+{
+	if (become_daemon)
+		openlog (G_LOG_DOMAIN, LOG_CONS, LOG_DAEMON);
+	else
+		openlog (G_LOG_DOMAIN, LOG_CONS | LOG_PERROR, LOG_USER);
+
+	g_log_set_handler (G_LOG_DOMAIN, 
+				    G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL | G_LOG_FLAG_RECURSION,
+				    nm_log_handler,
+				    NULL);
+
+	setup_signals ();
+}
+
+void
+nm_logging_shutdown (void)
+{
+	closelog ();
+}

src/nm-logging.h

@@ -0,0 +1,33 @@
+/* NetworkManager -- Network link manager
+ *
+ * Dan Williams <dcbw@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * (C) Copyright 2006 Red Hat, Inc.
+ */
+
+#ifndef NM_LOGGING_H
+#define NM_LOGGING_H
+
+#include <glib.h>
+
+void
+nm_logging_setup (gboolean become_daemon);
+
+void
+nm_logging_shutdown (void);
+
+#endif