The system is a dual processor / quad core Xeon E5335 running a 64 bit
debian lenny distribution; I've enabled both debug and rtinst configure,
and installed an unstripped binary; I'm attaching the full backtrace here.
the system should act as a client to a remote syslog server; I'm enclosing
the conf files too.
In the previous attempts (before enabling rtinst and debug), I got lines
like "rsyslogd: double free or corruption (!prev): 0x0000000000674c50"
in the failover log (corresponding to the _nodebug backtrace); this last
time, the line I've got is
"msg.c:273: msgDestruct: Assertion `(unsigned) ((obj_t*)
(pThis))->iObjCooCKiE == (unsigned) 0xBADEFEE' failed."
and the double free detector didn't fire.
I'm really at a loss now: I've been lured into experimenting rsyslog from
the high level architectural description being this clean and seemingly
powerful, but now I fear I still cannot rely on this logger in production.
Eager to help in debugging, sincerely yours
lorenzo
+-------------------------+----------------------------------------------+
| Lorenzo M. Catucci | Centro di Calcolo e Documentazione |
| catucci at ccd.uniroma2.it | Universit? degli Studi di Roma "Tor Vergata" |
| | Via O. Raimondo 18 ** I-00173 ROMA ** ITALY |
| Tel. +39 06 7259 2255 | Fax. +39 06 7259 2125 |
+-------------------------+----------------------------------------------+
-------------- next part --------------
GNU gdb 6.8-debian
Copyright (C) 2008 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu"...
Reading symbols from /usr/lib/libz.so.1...done.
Loaded symbols for /usr/lib/libz.so.1
Reading symbols from /lib/libpthread.so.0...Reading symbols from /usr/lib/debug/lib/libpthread-2.7.so...done.
done.
Loaded symbols for /lib/libpthread.so.0
Reading symbols from /lib/libdl.so.2...Reading symbols from /usr/lib/debug/lib/libdl-2.7.so...done.
done.
Loaded symbols for /lib/libdl.so.2
Reading symbols from /lib/librt.so.1...Reading symbols from /usr/lib/debug/lib/librt-2.7.so...done.
done.
Loaded symbols for /lib/librt.so.1
Reading symbols from /lib/libc.so.6...Reading symbols from /usr/lib/debug/lib/libc-2.7.so...done.
done.
Loaded symbols for /lib/libc.so.6
Reading symbols from /lib/ld-linux-x86-64.so.2...Reading symbols from /usr/lib/debug/lib/ld-2.7.so...done.
done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /usr/lib/rsyslog/lmnet.so...done.
Loaded symbols for /usr/lib/rsyslog/lmnet.so
Reading symbols from /lib/libnss_files.so.2...Reading symbols from /usr/lib/debug/lib/libnss_files-2.7.so...done.
done.
Loaded symbols for /lib/libnss_files.so.2
Reading symbols from /usr/lib/rsyslog/lmtcpclt.so...done.
Loaded symbols for /usr/lib/rsyslog/lmtcpclt.so
Reading symbols from /usr/lib/rsyslog/imuxsock.so...done.
Loaded symbols for /usr/lib/rsyslog/imuxsock.so
Reading symbols from /usr/lib/rsyslog/imklog.so...done.
Loaded symbols for /usr/lib/rsyslog/imklog.so
Reading symbols from /lib/libnss_compat.so.2...Reading symbols from /usr/lib/debug/lib/libnss_compat-2.7.so...done.
done.
Loaded symbols for /lib/libnss_compat.so.2
Reading symbols from /lib/libnsl.so.1...Reading symbols from /usr/lib/debug/lib/libnsl-2.7.so...done.
done.
Loaded symbols for /lib/libnsl.so.1
Reading symbols from /lib/libnss_nis.so.2...Reading symbols from /usr/lib/debug/lib/libnss_nis-2.7.so...done.
done.
Loaded symbols for /lib/libnss_nis.so.2
Core was generated by `/usr/sbin/rsyslogd -c3'.
Program terminated with signal 6, Aborted.
[New process 16283]
[New process 16285]
[New process 16284]
[New process 16282]
[New process 16281]
#0 0x00002b44aa374ef5 in raise () from /lib/libc.so.6
(gdb)
Thread 5 (process 16281):
#0 0x00002b44aa40bf12 in select () from /lib/libc.so.6
No symbol table info available.
#1 0x000000000040efb9 in realMain (argc=<value optimized out>,
argv=<value optimized out>) at syslogd.c:2563
dbgCALLStaCK_POP_POINT = 0
iRet = <value optimized out>
i = <value optimized out>
p = <value optimized out>
num_fds = <value optimized out>
ch = <value optimized out>
hent = <value optimized out>
sigAct = {__sigaction_handler = {sa_handler = 0x1,
sa_sigaction = 0x1}, sa_mask = {__val = {0 <repeats 16 times>}},
sa_flags = 0, sa_restorer = 0}
bIsFirstOption = <value optimized out>
bEOptionWasGiven = <value optimized out>
bImUxSockLoaded = <value optimized out>
arg = <value optimized out>
legacyConfLine = '\0' <repeats 79 times>
pdbgFuncDB = (dbgFuncDB_t *) 0x66e320
__func__ = "realMain"
#2 0x00002b44aa3611a6 in __libc_start_main () from /lib/libc.so.6
No symbol table info available.
#3 0x00000000004096b9 in _start ()
No locals.
Thread 4 (process 16282):
#0 0x00002b44a9d23fad in pthread_cond_timedwait@@GLIBC_2.3.2 ()
from /lib/libpthread.so.0
No symbol table info available.
#1 0x0000000000416333 in dbgCondTimedWait (cond=0x686fe0,
pmut=0xffffffffffffff92, abstime=0x407fffd0, pFuncDB=0x6a6bc0, ln=403,
iStackPtr=<value optimized out>) at debug.c:555
ret = <value optimized out>
#2 0x0000000000421f59 in wtiWorker (pThis=0x688160) at wti.c:403
__cancel_buf = {__cancel_jmp_buf = {{__cancel_jmp_buf = {1082130748,
1727653261974306363, 47573906921472, 47573906882304, 0, 4096,
1727512550297697851, 1727653793608105531}, __mask_was_saved = 0}},
__pad = {0x408000d0, 0x0, 0x68a740, 0x19f}}
not_first_call = <value optimized out>
dbgCALLStaCK_POP_POINT = 1
iCancelStateSave = 0
t = {tv_sec = 1221174340, tv_nsec = 609910833}
pWtp = (wtp_t *) 0x687b50
bInactivityTOOccured = 0
pdbgFuncDB = (dbgFuncDB_t *) 0x6a6bc0
__func__ = "wtiWorker"
__PRETTY_FUNCTION__ = "wtiWorker"
#3 0x00000000004209c2 in wtpWorker (arg=0x688160) at wtp.c:417
__cancel_buf = {__cancel_jmp_buf = {{__cancel_jmp_buf = {1082130512,
1727653261974306363, 47573906921472, 47573906882304, 0, 4096,
1727511424978517563, 1727653794260319803}, __mask_was_saved = 0}},
__pad = {0x408001e0, 0x0, 0x40800950, 0x40800950}}
not_first_call = <value optimized out>
dbgCALLStaCK_POP_POINT = 0
iCancelStateSave = 0
pWti = (wti_t *) 0x688160
pThis = (wtp_t *) 0x687b50
sigSet = {__val = {18446744067267100671,
18446744073709551615 <repeats 15 times>}}
pdbgFuncDB = (dbgFuncDB_t *) 0x68a740
__func__ = "wtpWorker"
__PRETTY_FUNCTION__ = "wtpWorker"
#4 0x00002b44a9d1ffc7 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#5 0x00002b44aa4127cd in clone () from /lib/libc.so.6
No symbol table info available.
#6 0x0000000000000000 in ?? ()
No symbol table info available.
Thread 3 (process 16284):
#0 0x00002b44aa40bf12 in select () from /lib/libc.so.6
No symbol table info available.
#1 0x00002b44aacb4773 in runInput (pThrd=<value optimized out>)
at imuxsock.c:235
maxfds = 0
nfds = <value optimized out>
i = <value optimized out>
fd = <value optimized out>
readfds = {fds_bits = {1, 0 <repeats 15 times>}}
pdbgFuncDB = (dbgFuncDB_t *) 0x6aa6f0
__func__ = "runInput"
#2 0x000000000041be2d in thrdStarter (arg=0x6a8a40) at threads.c:139
dbgCALLStaCK_POP_POINT = 0
iRet = <value optimized out>
sigSet = {__val = {18446744067267100671,
18446744073709551615 <repeats 15 times>}}
pdbgFuncDB = (dbgFuncDB_t *) 0x6aa620
__func__ = "thrdStarter"
__PRETTY_FUNCTION__ = "thrdStarter"
#3 0x00002b44a9d1ffc7 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#4 0x00002b44aa4127cd in clone () from /lib/libc.so.6
No symbol table info available.
#5 0x0000000000000000 in ?? ()
No symbol table info available.
Thread 2 (process 16285):
#0 0x00002b44a9d267eb in read () from /lib/libpthread.so.0
No symbol table info available.
#1 0x00002b44aaeb91c7 in klogLogKMsg () at linux.c:454
dbgCALLStaCK_POP_POINT = 2
pdbgFuncDB = (dbgFuncDB_t *) 0x6af730
__func__ = "klogLogKMsg"
#2 0x00002b44aaeb84a4 in runInput (pThrd=0x6abae0) at imklog.c:208
dbgCALLStaCK_POP_POINT = 1
iRet = RS_RET_OK
pdbgFuncDB = (dbgFuncDB_t *) 0x6af660
__func__ = "runInput"
#3 0x000000000041be2d in thrdStarter (arg=0x6abae0) at threads.c:139
dbgCALLStaCK_POP_POINT = 0
iRet = <value optimized out>
sigSet = {__val = {18446744067267100671,
18446744073709551615 <repeats 15 times>}}
pdbgFuncDB = (dbgFuncDB_t *) 0x6aa620
__func__ = "thrdStarter"
__PRETTY_FUNCTION__ = "thrdStarter"
#4 0x00002b44a9d1ffc7 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#5 0x00002b44aa4127cd in clone () from /lib/libc.so.6
No symbol table info available.
#6 0x0000000000000000 in ?? ()
No symbol table info available.
Thread 1 (process 16283):
#0 0x00002b44aa374ef5 in raise () from /lib/libc.so.6
No symbol table info available.
#1 0x00002b44aa376413 in abort () from /lib/libc.so.6
No symbol table info available.
#2 0x0000000000415a5f in sigsegvHdlr (signum=6) at debug.c:741
signame = <value optimized out>
sigAct = {__sigaction_handler = {sa_handler = 0, sa_sigaction = 0},
sa_mask = {__val = {0 <repeats 16 times>}}, sa_flags = 0, sa_restorer = 0}
#3 <signal handler called>
No symbol table info available.
#4 0x00002b44aa374ef5 in raise () from /lib/libc.so.6
No symbol table info available.
#5 0x00002b44aa376413 in abort () from /lib/libc.so.6
No symbol table info available.
#6 0x00002b44aa36ddc9 in __assert_fail () from /lib/libc.so.6
No symbol table info available.
#7 0x000000000042e550 in msgDestruct (ppThis=0x692a20) at msg.c:273
dbgCALLStaCK_POP_POINT = 8
iCancelStateSave = <value optimized out>
pThis = (msg_t *) 0x2aaaac0008c0
pdbgFuncDB = (dbgFuncDB_t *) 0x6addf0
__func__ = "msgDestruct"
__PRETTY_FUNCTION__ = "msgDestruct"
#8 0x000000000043d346 in actionCallAction (pAction=0x6929c0,
pMsg=0x2aaaac000ac0) at action.c:661
__cancel_buf = {__cancel_jmp_buf = {{__cancel_jmp_buf = {6891968,
1727510324980350523, 6, 4240656, 1090522784, 1090522696,
1727510325009710651, 1727653804676349499}, __mask_was_saved = 0}},
__pad = {0x41000f60, 0x0, 0x692820, 0x40b510}}
__cancel_arg = (void *) 0x692a70
not_first_call = <value optimized out>
dbgCALLStaCK_POP_POINT = 7
iRet = <value optimized out>
iCancelStateSave = 0
pdbgFuncDB = (dbgFuncDB_t *) 0x6ad2b0
__func__ = "actionCallAction"
__PRETTY_FUNCTION__ = "actionCallAction"
#9 0x000000000040b567 in processMsgDoActions (pData=0x6929c0,
pParam=0x41000ea0) at syslogd.c:1094
dbgCALLStaCK_POP_POINT = 6
iRet = <value optimized out>
iRetMod = <value optimized out>
pdbgFuncDB = (dbgFuncDB_t *) 0x6ad180
__func__ = "processMsgDoActions"
__PRETTY_FUNCTION__ = "processMsgDoActions"
#10 0x000000000043c0ce in llExecFunc (pThis=0x692820,
pFunc=0x40b510 <processMsgDoActions>, pParam=0x41000ea0)
at linkedlist.c:390
dbgCALLStaCK_POP_POINT = 5
iRet = RS_RET_OK
iRetLL = RS_RET_OK
pData = (void *) 0x6929c0
llCookie = (linkedListCookie_t) 0x6920c0
llCookiePrev = (linkedListCookie_t) 0x0
pdbgFuncDB = (dbgFuncDB_t *) 0x6ad050
__func__ = "llExecFunc"
__PRETTY_FUNCTION__ = "llExecFunc"
#11 0x000000000040b0c9 in msgConsumer (notNeeded=<value optimized out>,
pUsr=0x2aaaac000ac0) at syslogd.c:1137
dbgCALLStaCK_POP_POINT = 3
pMsg = (msg_t *) 0x2aaaac000ac0
pdbgFuncDB = (dbgFuncDB_t *) 0x6accb0
__func__ = "msgConsumer"
__PRETTY_FUNCTION__ = "msgConsumer"
#12 0x000000000042521f in queueConsumerReg (pThis=0x692db0, pWti=0x6a6900,
iCancelStateSave=<value optimized out>) at queue.c:1570
dbgCALLStaCK_POP_POINT = 2
iRet = RS_RET_OK
pdbgFuncDB = (dbgFuncDB_t *) 0x6ac8d0
__func__ = "queueConsumerReg"
__PRETTY_FUNCTION__ = "queueConsumerReg"
#13 0x0000000000421eb0 in wtiWorker (pThis=0x6a6900) at wti.c:413
__cancel_buf = {__cancel_jmp_buf = {{__cancel_jmp_buf = {1090523452,
1727653261974306363, 47573906921472, 47573906882304, 0, 4096,
1727510324967767611, 1727653793608105531}, __mask_was_saved = 0}},
__pad = {0x410010d0, 0x0, 0x68a740, 0x19f}}
not_first_call = <value optimized out>
dbgCALLStaCK_POP_POINT = 1
iCancelStateSave = 0
t = {tv_sec = 1221174340, tv_nsec = 478220993}
pWtp = (wtp_t *) 0x680d90
bInactivityTOOccured = 0
pdbgFuncDB = (dbgFuncDB_t *) 0x6a6bc0
__func__ = "wtiWorker"
__PRETTY_FUNCTION__ = "wtiWorker"
#14 0x00000000004209c2 in wtpWorker (arg=0x6a6900) at wtp.c:417
__cancel_buf = {__cancel_jmp_buf = {{__cancel_jmp_buf = {1090523216,
1727653261974306363, 47573906921472, 47573906882304, 0, 4096,
1727510324930018875, 1727653794260319803}, __mask_was_saved = 0}},
__pad = {0x410011e0, 0x0, 0x0, 0x0}}
not_first_call = <value optimized out>
dbgCALLStaCK_POP_POINT = 0
iCancelStateSave = 0
pWti = (wti_t *) 0x6a6900
pThis = (wtp_t *) 0x680d90
sigSet = {__val = {18446744067267100671,
18446744073709551615 <repeats 15 times>}}
pdbgFuncDB = (dbgFuncDB_t *) 0x68a740
__func__ = "wtpWorker"
__PRETTY_FUNCTION__ = "wtpWorker"
#15 0x00002b44a9d1ffc7 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#16 0x00002b44aa4127cd in clone () from /lib/libc.so.6
No symbol table info available.
#17 0x0000000000000000 in ?? ()
No symbol table info available.
(gdb)
-------------- next part --------------
# /etc/rsyslog.conf Configuration file for rsyslog v3.
#
# For more information see
# /usr/share/doc/rsyslog-doc/html/rsyslog_conf.html
#################
#### MODULES ####
#################
$ModLoad imuxsock # provides support for local system logging
$ModLoad imklog # provides kernel logging support (previously done by rklogd)
#$ModLoad immark # provides --MARK-- message capability
# provides UDP syslog reception
#$ModLoad imudp
#$UDPServerRun 514
# provides TCP syslog reception
#$ModLoad imtcp
#$InputTCPServerRun 514
###########################
#### GLOBAL DIRECTIVES ####
###########################
#
# Use default timestamp format.
# To enable high precision timestamps, comment out the following line.
#
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
#
# Set the default permissions for all log files.
#
$FileOwner root
$FileGroup adm
$FileCreateMode 0640
#
# Include all config files in /etc/rsyslog.d/
#
$IncludeConfig /etc/rsyslog.d/*.conf
###############
#### RULES ####
###############
#
# First some standard log files. Log by facility.
#
auth,authpriv.* /var/log/auth.log
*.*;auth,authpriv.none -/var/log/syslog
#cron.* /var/log/cron.log
daemon.* -/var/log/daemon.log
kern.* -/var/log/kern.log
lpr.* -/var/log/lpr.log
#mail.* -/var/log/mail.log
user.* -/var/log/user.log
#
# Logging for the mail system. Split it up so that
# it is easy to write scripts to parse these files.
#
mail.info -/var/log/mail.info
mail.warn -/var/log/mail.warn
mail.err /var/log/mail.err
#
# Logging for INN news system.
#
news.crit /var/log/news/news.crit
news.err /var/log/news/news.err
news.notice -/var/log/news/news.notice
#
# Some "catch-all" log files.
#
*.=debug;\
auth,authpriv.none;\
news.none;mail.none -/var/log/debug
*.=info;*.=notice;*.=warn;\
auth,authpriv.none;\
cron,daemon.none;\
mail,news.none -/var/log/messages
#
# Emergencies are sent to everybody logged in.
#
*.emerg *
#
# I like to have messages displayed on the console, but only on a virtual
# console I usually leave idle.
#
#daemon,mail.*;\
# news.=crit;news.=err;news.=notice;\
# *.=debug;*.=info;\
# *.=notice;*.=warn /dev/tty8
# The named pipe /dev/xconsole is for the `xconsole' utility. To use it,
# you must invoke `xconsole' with the `-file' option:
#
# $ xconsole -file /dev/xconsole [...]
#
# NOTE: adjust the list below, or you'll go crazy if you have a reasonably
# busy site..
#
daemon.*;mail.*;\
news.err;\
*.=debug;*.=info;\
*.=notice;*.=warn |/dev/xconsole
-------------- next part --------------
#$ModLoad omrelp
$WorkDirectory /var/log/rsyslog
$ActionQueueType LinkedList # use asynchronous processing
$ActionQueueFileName srvrfwd # set file name, also enables disk mode
$ActionResumeRetryCount -1 # infinite retries on insert failure
$ActionQueueSaveOnShutdown on # save in-memory data if rsyslog shuts down
mail.* @@<remote_addr>:514
$ActionExecOnlyIfPreviousIsSuspended on
& /var/log/rsyslog/failover.log
$ActionExecOnlyIfPreviousIsSuspended off
-------------- next part --------------
GNU gdb 6.8-debian
Copyright (C) 2008 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu"...
Reading symbols from /usr/lib/libz.so.1...done.
Loaded symbols for /usr/lib/libz.so.1
Reading symbols from /lib/libpthread.so.0...Reading symbols from /usr/lib/debug/lib/libpthread-2.7.so...done.
done.
Loaded symbols for /lib/libpthread.so.0
Reading symbols from /lib/libdl.so.2...Reading symbols from /usr/lib/debug/lib/libdl-2.7.so...done.
done.
Loaded symbols for /lib/libdl.so.2
Reading symbols from /lib/librt.so.1...Reading symbols from /usr/lib/debug/lib/librt-2.7.so...done.
done.
Loaded symbols for /lib/librt.so.1
Reading symbols from /lib/libc.so.6...Reading symbols from /usr/lib/debug/lib/libc-2.7.so...done.
done.
Loaded symbols for /lib/libc.so.6
Reading symbols from /lib/ld-linux-x86-64.so.2...Reading symbols from /usr/lib/debug/lib/ld-2.7.so...done.
done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /usr/lib/rsyslog/lmnet.so...done.
Loaded symbols for /usr/lib/rsyslog/lmnet.so
Reading symbols from /lib/libnss_files.so.2...Reading symbols from /usr/lib/debug/lib/libnss_files-2.7.so...done.
done.
Loaded symbols for /lib/libnss_files.so.2
Reading symbols from /usr/lib/rsyslog/lmtcpclt.so...done.
Loaded symbols for /usr/lib/rsyslog/lmtcpclt.so
Reading symbols from /usr/lib/rsyslog/imuxsock.so...done.
Loaded symbols for /usr/lib/rsyslog/imuxsock.so
Reading symbols from /usr/lib/rsyslog/imklog.so...done.
Loaded symbols for /usr/lib/rsyslog/imklog.so
Reading symbols from /lib/libnss_compat.so.2...Reading symbols from /usr/lib/debug/lib/libnss_compat-2.7.so...done.
done.
Loaded symbols for /lib/libnss_compat.so.2
Reading symbols from /lib/libnsl.so.1...Reading symbols from /usr/lib/debug/lib/libnsl-2.7.so...done.
done.
Loaded symbols for /lib/libnsl.so.1
Reading symbols from /lib/libnss_nis.so.2...Reading symbols from /usr/lib/debug/lib/libnss_nis-2.7.so...done.
done.
Loaded symbols for /lib/libnss_nis.so.2
Reading symbols from /lib/libgcc_s.so.1...done.
Loaded symbols for /lib/libgcc_s.so.1
Core was generated by `/usr/sbin/rsyslogd -c3'.
Program terminated with signal 6, Aborted.
[New process 23847]
[New process 23849]
[New process 23848]
[New process 23846]
[New process 23845]
#0 0x00002b479d1c3ef5 in raise () from /lib/libc.so.6
(gdb)
Thread 5 (process 23845):
#0 0x00002b479d25af12 in select () from /lib/libc.so.6
No symbol table info available.
#1 0x000000000040e0c9 in realMain (argc=<value optimized out>,
argv=<value optimized out>) at syslogd.c:2563
iRet = <value optimized out>
i = <value optimized out>
p = <value optimized out>
num_fds = <value optimized out>
ch = <value optimized out>
hent = <value optimized out>
sigAct = {__sigaction_handler = {sa_handler = 0x1,
sa_sigaction = 0x1}, sa_mask = {__val = {0 <repeats 16 times>}},
sa_flags = 0, sa_restorer = 0}
bIsFirstOption = <value optimized out>
bEOptionWasGiven = <value optimized out>
bImUxSockLoaded = <value optimized out>
arg = <value optimized out>
legacyConfLine = '\0' <repeats 79 times>
#2 0x00002b479d1b01a6 in __libc_start_main () from /lib/libc.so.6
No symbol table info available.
#3 0x0000000000409609 in _start ()
No locals.
Thread 4 (process 23846):
#0 0x00002b479cb72fad in pthread_cond_timedwait@@GLIBC_2.3.2 ()
from /lib/libpthread.so.0
No symbol table info available.
#1 0x0000000000418e00 in wtiWorker (pThis=0x64f4a0) at wti.c:403
__cancel_buf = {__cancel_jmp_buf = {{__cancel_jmp_buf = {1082130748,
9093895273491594810, 47586571945984, 47586571906816, 0, 4096,
9094035857682333242, 9093894727638318650}, __mask_was_saved = 0}},
__pad = {0x408000d0, 0x0, 0x40800950, 0x40800950}}
not_first_call = <value optimized out>
iCancelStateSave = 0
t = {tv_sec = 1221135378, tv_nsec = 723151645}
pWtp = (wtp_t *) 0x64f360
bInactivityTOOccured = 0
#2 0x0000000000418032 in wtpWorker (arg=<value optimized out>) at wtp.c:417
__cancel_buf = {__cancel_jmp_buf = {{__cancel_jmp_buf = {1082130512,
9093895273491594810, 47586571945984, 47586571906816, 0, 4096,
9094034911474613818, 9093894729333866042}, __mask_was_saved = 0}},
__pad = {0x408001e0, 0x0, 0x40800950, 0x40800950}}
not_first_call = <value optimized out>
iCancelStateSave = 0
pWti = (wti_t *) 0x64f4a0
pThis = (wtp_t *) 0x64f360
sigSet = {__val = {18446744067267100671,
18446744073709551615 <repeats 15 times>}}
#3 0x00002b479cb6efc7 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#4 0x00002b479d2617cd in clone () from /lib/libc.so.6
No symbol table info available.
#5 0x0000000000000000 in ?? ()
No symbol table info available.
Thread 3 (process 23848):
#0 0x00002b479d25af12 in select () from /lib/libc.so.6
No symbol table info available.
#1 0x00002b479db01413 in runInput (pThrd=<value optimized out>)
at imuxsock.c:235
maxfds = 0
nfds = <value optimized out>
i = <value optimized out>
fd = <value optimized out>
readfds = {fds_bits = {1, 0 <repeats 15 times>}}
#2 0x0000000000416420 in thrdStarter (arg=0x66bd70) at threads.c:139
iRet = <value optimized out>
sigSet = {__val = {18446744067267100671,
18446744073709551615 <repeats 15 times>}}
#3 0x00002b479cb6efc7 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#4 0x00002b479d2617cd in clone () from /lib/libc.so.6
No symbol table info available.
#5 0x0000000000000000 in ?? ()
No symbol table info available.
Thread 2 (process 23849):
#0 0x00002b479cb757eb in read () from /lib/libpthread.so.0
No symbol table info available.
#1 0x00002b479dd05c5f in klogLogKMsg () at linux.c:454
No locals.
#2 0x00002b479dd0517c in runInput (pThrd=0x66c1f0) at imklog.c:208
iRet = <value optimized out>
#3 0x0000000000416420 in thrdStarter (arg=0x66c1f0) at threads.c:139
iRet = <value optimized out>
sigSet = {__val = {18446744067267100671,
18446744073709551615 <repeats 15 times>}}
#4 0x00002b479cb6efc7 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#5 0x00002b479d2617cd in clone () from /lib/libc.so.6
No symbol table info available.
#6 0x0000000000000000 in ?? ()
No symbol table info available.
Thread 1 (process 23847):
#0 0x00002b479d1c3ef5 in raise () from /lib/libc.so.6
No symbol table info available.
#1 0x00002b479d1c5413 in abort () from /lib/libc.so.6
No symbol table info available.
#2 0x0000000000411abf in sigsegvHdlr (signum=6) at debug.c:741
signame = <value optimized out>
sigAct = {__sigaction_handler = {sa_handler = 0, sa_sigaction = 0},
sa_mask = {__val = {0 <repeats 16 times>}}, sa_flags = 0, sa_restorer = 0}
#3 <signal handler called>
No symbol table info available.
#4 0x00002b479d1c3ef5 in raise () from /lib/libc.so.6
No symbol table info available.
#5 0x00002b479d1c5413 in abort () from /lib/libc.so.6
No symbol table info available.
#6 0x00002b479d2003e8 in __libc_message () from /lib/libc.so.6
No symbol table info available.
#7 0x00002b479d205968 in malloc_printerr () from /lib/libc.so.6
No symbol table info available.
#8 0x00002b479d207a76 in free () from /lib/libc.so.6
No symbol table info available.
#9 0x0000000000420f2d in msgDestruct (ppThis=0x6532a0) at msg.c:284
iCancelStateSave = 0
pThis = (msg_t *) 0x2aaab401e480
#10 0x000000000042a797 in actionCallAction (pAction=0x653240,
pMsg=0x2aaab0020620) at action.c:661
__cancel_buf = {__cancel_jmp_buf = {{__cancel_jmp_buf = {1090522832,
9093895273491594810, 6631600, 1090522832, 4238944, 1090522856,
9094038209632009786, 9093894735514959418}, __mask_was_saved = 0}},
__pad = {0x41000f70, 0x0, 0xbf11, 0x0}}
__cancel_arg = (void *) 0x6532f0
not_first_call = <value optimized out>
iRet = <value optimized out>
iCancelStateSave = 0
#11 0x000000000040ae79 in processMsgDoActions (pData=0x5d25, pParam=0x41000ed0)
at syslogd.c:1094
iRet = <value optimized out>
iRetMod = <value optimized out>
#12 0x0000000000429b97 in llExecFunc (pThis=0x6530b0,
pFunc=0x40ae60 <processMsgDoActions>, pParam=0x41000ed0)
at linkedlist.c:390
iRet = RS_RET_OK
llCookie = (linkedListCookie_t) 0x652c60
llCookiePrev = (linkedListCookie_t) 0x0
#13 0x000000000040aab0 in msgConsumer (notNeeded=<value optimized out>,
pUsr=<value optimized out>) at syslogd.c:1137
pMsg = (msg_t *) 0x2aaab0020620
#14 0x000000000041aa31 in queueConsumerReg (pThis=0x656980, pWti=0x64bcf0,
iCancelStateSave=<value optimized out>) at queue.c:1570
iRet = RS_RET_OK
#15 0x0000000000418e6f in wtiWorker (pThis=0x64bcf0) at wti.c:413
__cancel_buf = {__cancel_jmp_buf = {{__cancel_jmp_buf = {1090523452,
9093895273491594810, 47586571945984, 47586571906816, 0, 4096,
9094038209713798714, 9093894727638318650}, __mask_was_saved = 0}},
__pad = {0x410010d0, 0x0, 0x0, 0x0}}
not_first_call = <value optimized out>
iCancelStateSave = 0
t = {tv_sec = 1221135378, tv_nsec = 595602238}
pWtp = (wtp_t *) 0x64bbb0
bInactivityTOOccured = 0
#16 0x0000000000418032 in wtpWorker (arg=<value optimized out>) at wtp.c:417
__cancel_buf = {__cancel_jmp_buf = {{__cancel_jmp_buf = {1090523216,
9093895273491594810, 47586571945984, 47586571906816, 0, 4096,
9094038210546368058, 9093894729333866042}, __mask_was_saved = 0}},
__pad = {0x410011e0, 0x0, 0x0, 0x0}}
not_first_call = <value optimized out>
iCancelStateSave = 0
pWti = (wti_t *) 0x64bcf0
pThis = (wtp_t *) 0x64bbb0
sigSet = {__val = {18446744067267100671,
18446744073709551615 <repeats 15 times>}}
#17 0x00002b479cb6efc7 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#18 0x00002b479d2617cd in clone () from /lib/libc.so.6
No symbol table info available.
#19 0x0000000000000000 in ?? ()
No symbol table info available.
(gdb) quit
debian lenny distribution; I've enabled both debug and rtinst configure,
and installed an unstripped binary; I'm attaching the full backtrace here.
the system should act as a client to a remote syslog server; I'm enclosing
the conf files too.
In the previous attempts (before enabling rtinst and debug), I got lines
like "rsyslogd: double free or corruption (!prev): 0x0000000000674c50"
in the failover log (corresponding to the _nodebug backtrace); this last
time, the line I've got is
"msg.c:273: msgDestruct: Assertion `(unsigned) ((obj_t*)
(pThis))->iObjCooCKiE == (unsigned) 0xBADEFEE' failed."
and the double free detector didn't fire.
I'm really at a loss now: I've been lured into experimenting rsyslog from
the high level architectural description being this clean and seemingly
powerful, but now I fear I still cannot rely on this logger in production.
Eager to help in debugging, sincerely yours
lorenzo
+-------------------------+----------------------------------------------+
| Lorenzo M. Catucci | Centro di Calcolo e Documentazione |
| catucci at ccd.uniroma2.it | Universit? degli Studi di Roma "Tor Vergata" |
| | Via O. Raimondo 18 ** I-00173 ROMA ** ITALY |
| Tel. +39 06 7259 2255 | Fax. +39 06 7259 2125 |
+-------------------------+----------------------------------------------+
-------------- next part --------------
GNU gdb 6.8-debian
Copyright (C) 2008 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu"...
Reading symbols from /usr/lib/libz.so.1...done.
Loaded symbols for /usr/lib/libz.so.1
Reading symbols from /lib/libpthread.so.0...Reading symbols from /usr/lib/debug/lib/libpthread-2.7.so...done.
done.
Loaded symbols for /lib/libpthread.so.0
Reading symbols from /lib/libdl.so.2...Reading symbols from /usr/lib/debug/lib/libdl-2.7.so...done.
done.
Loaded symbols for /lib/libdl.so.2
Reading symbols from /lib/librt.so.1...Reading symbols from /usr/lib/debug/lib/librt-2.7.so...done.
done.
Loaded symbols for /lib/librt.so.1
Reading symbols from /lib/libc.so.6...Reading symbols from /usr/lib/debug/lib/libc-2.7.so...done.
done.
Loaded symbols for /lib/libc.so.6
Reading symbols from /lib/ld-linux-x86-64.so.2...Reading symbols from /usr/lib/debug/lib/ld-2.7.so...done.
done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /usr/lib/rsyslog/lmnet.so...done.
Loaded symbols for /usr/lib/rsyslog/lmnet.so
Reading symbols from /lib/libnss_files.so.2...Reading symbols from /usr/lib/debug/lib/libnss_files-2.7.so...done.
done.
Loaded symbols for /lib/libnss_files.so.2
Reading symbols from /usr/lib/rsyslog/lmtcpclt.so...done.
Loaded symbols for /usr/lib/rsyslog/lmtcpclt.so
Reading symbols from /usr/lib/rsyslog/imuxsock.so...done.
Loaded symbols for /usr/lib/rsyslog/imuxsock.so
Reading symbols from /usr/lib/rsyslog/imklog.so...done.
Loaded symbols for /usr/lib/rsyslog/imklog.so
Reading symbols from /lib/libnss_compat.so.2...Reading symbols from /usr/lib/debug/lib/libnss_compat-2.7.so...done.
done.
Loaded symbols for /lib/libnss_compat.so.2
Reading symbols from /lib/libnsl.so.1...Reading symbols from /usr/lib/debug/lib/libnsl-2.7.so...done.
done.
Loaded symbols for /lib/libnsl.so.1
Reading symbols from /lib/libnss_nis.so.2...Reading symbols from /usr/lib/debug/lib/libnss_nis-2.7.so...done.
done.
Loaded symbols for /lib/libnss_nis.so.2
Core was generated by `/usr/sbin/rsyslogd -c3'.
Program terminated with signal 6, Aborted.
[New process 16283]
[New process 16285]
[New process 16284]
[New process 16282]
[New process 16281]
#0 0x00002b44aa374ef5 in raise () from /lib/libc.so.6
(gdb)
Thread 5 (process 16281):
#0 0x00002b44aa40bf12 in select () from /lib/libc.so.6
No symbol table info available.
#1 0x000000000040efb9 in realMain (argc=<value optimized out>,
argv=<value optimized out>) at syslogd.c:2563
dbgCALLStaCK_POP_POINT = 0
iRet = <value optimized out>
i = <value optimized out>
p = <value optimized out>
num_fds = <value optimized out>
ch = <value optimized out>
hent = <value optimized out>
sigAct = {__sigaction_handler = {sa_handler = 0x1,
sa_sigaction = 0x1}, sa_mask = {__val = {0 <repeats 16 times>}},
sa_flags = 0, sa_restorer = 0}
bIsFirstOption = <value optimized out>
bEOptionWasGiven = <value optimized out>
bImUxSockLoaded = <value optimized out>
arg = <value optimized out>
legacyConfLine = '\0' <repeats 79 times>
pdbgFuncDB = (dbgFuncDB_t *) 0x66e320
__func__ = "realMain"
#2 0x00002b44aa3611a6 in __libc_start_main () from /lib/libc.so.6
No symbol table info available.
#3 0x00000000004096b9 in _start ()
No locals.
Thread 4 (process 16282):
#0 0x00002b44a9d23fad in pthread_cond_timedwait@@GLIBC_2.3.2 ()
from /lib/libpthread.so.0
No symbol table info available.
#1 0x0000000000416333 in dbgCondTimedWait (cond=0x686fe0,
pmut=0xffffffffffffff92, abstime=0x407fffd0, pFuncDB=0x6a6bc0, ln=403,
iStackPtr=<value optimized out>) at debug.c:555
ret = <value optimized out>
#2 0x0000000000421f59 in wtiWorker (pThis=0x688160) at wti.c:403
__cancel_buf = {__cancel_jmp_buf = {{__cancel_jmp_buf = {1082130748,
1727653261974306363, 47573906921472, 47573906882304, 0, 4096,
1727512550297697851, 1727653793608105531}, __mask_was_saved = 0}},
__pad = {0x408000d0, 0x0, 0x68a740, 0x19f}}
not_first_call = <value optimized out>
dbgCALLStaCK_POP_POINT = 1
iCancelStateSave = 0
t = {tv_sec = 1221174340, tv_nsec = 609910833}
pWtp = (wtp_t *) 0x687b50
bInactivityTOOccured = 0
pdbgFuncDB = (dbgFuncDB_t *) 0x6a6bc0
__func__ = "wtiWorker"
__PRETTY_FUNCTION__ = "wtiWorker"
#3 0x00000000004209c2 in wtpWorker (arg=0x688160) at wtp.c:417
__cancel_buf = {__cancel_jmp_buf = {{__cancel_jmp_buf = {1082130512,
1727653261974306363, 47573906921472, 47573906882304, 0, 4096,
1727511424978517563, 1727653794260319803}, __mask_was_saved = 0}},
__pad = {0x408001e0, 0x0, 0x40800950, 0x40800950}}
not_first_call = <value optimized out>
dbgCALLStaCK_POP_POINT = 0
iCancelStateSave = 0
pWti = (wti_t *) 0x688160
pThis = (wtp_t *) 0x687b50
sigSet = {__val = {18446744067267100671,
18446744073709551615 <repeats 15 times>}}
pdbgFuncDB = (dbgFuncDB_t *) 0x68a740
__func__ = "wtpWorker"
__PRETTY_FUNCTION__ = "wtpWorker"
#4 0x00002b44a9d1ffc7 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#5 0x00002b44aa4127cd in clone () from /lib/libc.so.6
No symbol table info available.
#6 0x0000000000000000 in ?? ()
No symbol table info available.
Thread 3 (process 16284):
#0 0x00002b44aa40bf12 in select () from /lib/libc.so.6
No symbol table info available.
#1 0x00002b44aacb4773 in runInput (pThrd=<value optimized out>)
at imuxsock.c:235
maxfds = 0
nfds = <value optimized out>
i = <value optimized out>
fd = <value optimized out>
readfds = {fds_bits = {1, 0 <repeats 15 times>}}
pdbgFuncDB = (dbgFuncDB_t *) 0x6aa6f0
__func__ = "runInput"
#2 0x000000000041be2d in thrdStarter (arg=0x6a8a40) at threads.c:139
dbgCALLStaCK_POP_POINT = 0
iRet = <value optimized out>
sigSet = {__val = {18446744067267100671,
18446744073709551615 <repeats 15 times>}}
pdbgFuncDB = (dbgFuncDB_t *) 0x6aa620
__func__ = "thrdStarter"
__PRETTY_FUNCTION__ = "thrdStarter"
#3 0x00002b44a9d1ffc7 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#4 0x00002b44aa4127cd in clone () from /lib/libc.so.6
No symbol table info available.
#5 0x0000000000000000 in ?? ()
No symbol table info available.
Thread 2 (process 16285):
#0 0x00002b44a9d267eb in read () from /lib/libpthread.so.0
No symbol table info available.
#1 0x00002b44aaeb91c7 in klogLogKMsg () at linux.c:454
dbgCALLStaCK_POP_POINT = 2
pdbgFuncDB = (dbgFuncDB_t *) 0x6af730
__func__ = "klogLogKMsg"
#2 0x00002b44aaeb84a4 in runInput (pThrd=0x6abae0) at imklog.c:208
dbgCALLStaCK_POP_POINT = 1
iRet = RS_RET_OK
pdbgFuncDB = (dbgFuncDB_t *) 0x6af660
__func__ = "runInput"
#3 0x000000000041be2d in thrdStarter (arg=0x6abae0) at threads.c:139
dbgCALLStaCK_POP_POINT = 0
iRet = <value optimized out>
sigSet = {__val = {18446744067267100671,
18446744073709551615 <repeats 15 times>}}
pdbgFuncDB = (dbgFuncDB_t *) 0x6aa620
__func__ = "thrdStarter"
__PRETTY_FUNCTION__ = "thrdStarter"
#4 0x00002b44a9d1ffc7 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#5 0x00002b44aa4127cd in clone () from /lib/libc.so.6
No symbol table info available.
#6 0x0000000000000000 in ?? ()
No symbol table info available.
Thread 1 (process 16283):
#0 0x00002b44aa374ef5 in raise () from /lib/libc.so.6
No symbol table info available.
#1 0x00002b44aa376413 in abort () from /lib/libc.so.6
No symbol table info available.
#2 0x0000000000415a5f in sigsegvHdlr (signum=6) at debug.c:741
signame = <value optimized out>
sigAct = {__sigaction_handler = {sa_handler = 0, sa_sigaction = 0},
sa_mask = {__val = {0 <repeats 16 times>}}, sa_flags = 0, sa_restorer = 0}
#3 <signal handler called>
No symbol table info available.
#4 0x00002b44aa374ef5 in raise () from /lib/libc.so.6
No symbol table info available.
#5 0x00002b44aa376413 in abort () from /lib/libc.so.6
No symbol table info available.
#6 0x00002b44aa36ddc9 in __assert_fail () from /lib/libc.so.6
No symbol table info available.
#7 0x000000000042e550 in msgDestruct (ppThis=0x692a20) at msg.c:273
dbgCALLStaCK_POP_POINT = 8
iCancelStateSave = <value optimized out>
pThis = (msg_t *) 0x2aaaac0008c0
pdbgFuncDB = (dbgFuncDB_t *) 0x6addf0
__func__ = "msgDestruct"
__PRETTY_FUNCTION__ = "msgDestruct"
#8 0x000000000043d346 in actionCallAction (pAction=0x6929c0,
pMsg=0x2aaaac000ac0) at action.c:661
__cancel_buf = {__cancel_jmp_buf = {{__cancel_jmp_buf = {6891968,
1727510324980350523, 6, 4240656, 1090522784, 1090522696,
1727510325009710651, 1727653804676349499}, __mask_was_saved = 0}},
__pad = {0x41000f60, 0x0, 0x692820, 0x40b510}}
__cancel_arg = (void *) 0x692a70
not_first_call = <value optimized out>
dbgCALLStaCK_POP_POINT = 7
iRet = <value optimized out>
iCancelStateSave = 0
pdbgFuncDB = (dbgFuncDB_t *) 0x6ad2b0
__func__ = "actionCallAction"
__PRETTY_FUNCTION__ = "actionCallAction"
#9 0x000000000040b567 in processMsgDoActions (pData=0x6929c0,
pParam=0x41000ea0) at syslogd.c:1094
dbgCALLStaCK_POP_POINT = 6
iRet = <value optimized out>
iRetMod = <value optimized out>
pdbgFuncDB = (dbgFuncDB_t *) 0x6ad180
__func__ = "processMsgDoActions"
__PRETTY_FUNCTION__ = "processMsgDoActions"
#10 0x000000000043c0ce in llExecFunc (pThis=0x692820,
pFunc=0x40b510 <processMsgDoActions>, pParam=0x41000ea0)
at linkedlist.c:390
dbgCALLStaCK_POP_POINT = 5
iRet = RS_RET_OK
iRetLL = RS_RET_OK
pData = (void *) 0x6929c0
llCookie = (linkedListCookie_t) 0x6920c0
llCookiePrev = (linkedListCookie_t) 0x0
pdbgFuncDB = (dbgFuncDB_t *) 0x6ad050
__func__ = "llExecFunc"
__PRETTY_FUNCTION__ = "llExecFunc"
#11 0x000000000040b0c9 in msgConsumer (notNeeded=<value optimized out>,
pUsr=0x2aaaac000ac0) at syslogd.c:1137
dbgCALLStaCK_POP_POINT = 3
pMsg = (msg_t *) 0x2aaaac000ac0
pdbgFuncDB = (dbgFuncDB_t *) 0x6accb0
__func__ = "msgConsumer"
__PRETTY_FUNCTION__ = "msgConsumer"
#12 0x000000000042521f in queueConsumerReg (pThis=0x692db0, pWti=0x6a6900,
iCancelStateSave=<value optimized out>) at queue.c:1570
dbgCALLStaCK_POP_POINT = 2
iRet = RS_RET_OK
pdbgFuncDB = (dbgFuncDB_t *) 0x6ac8d0
__func__ = "queueConsumerReg"
__PRETTY_FUNCTION__ = "queueConsumerReg"
#13 0x0000000000421eb0 in wtiWorker (pThis=0x6a6900) at wti.c:413
__cancel_buf = {__cancel_jmp_buf = {{__cancel_jmp_buf = {1090523452,
1727653261974306363, 47573906921472, 47573906882304, 0, 4096,
1727510324967767611, 1727653793608105531}, __mask_was_saved = 0}},
__pad = {0x410010d0, 0x0, 0x68a740, 0x19f}}
not_first_call = <value optimized out>
dbgCALLStaCK_POP_POINT = 1
iCancelStateSave = 0
t = {tv_sec = 1221174340, tv_nsec = 478220993}
pWtp = (wtp_t *) 0x680d90
bInactivityTOOccured = 0
pdbgFuncDB = (dbgFuncDB_t *) 0x6a6bc0
__func__ = "wtiWorker"
__PRETTY_FUNCTION__ = "wtiWorker"
#14 0x00000000004209c2 in wtpWorker (arg=0x6a6900) at wtp.c:417
__cancel_buf = {__cancel_jmp_buf = {{__cancel_jmp_buf = {1090523216,
1727653261974306363, 47573906921472, 47573906882304, 0, 4096,
1727510324930018875, 1727653794260319803}, __mask_was_saved = 0}},
__pad = {0x410011e0, 0x0, 0x0, 0x0}}
not_first_call = <value optimized out>
dbgCALLStaCK_POP_POINT = 0
iCancelStateSave = 0
pWti = (wti_t *) 0x6a6900
pThis = (wtp_t *) 0x680d90
sigSet = {__val = {18446744067267100671,
18446744073709551615 <repeats 15 times>}}
pdbgFuncDB = (dbgFuncDB_t *) 0x68a740
__func__ = "wtpWorker"
__PRETTY_FUNCTION__ = "wtpWorker"
#15 0x00002b44a9d1ffc7 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#16 0x00002b44aa4127cd in clone () from /lib/libc.so.6
No symbol table info available.
#17 0x0000000000000000 in ?? ()
No symbol table info available.
(gdb)
-------------- next part --------------
# /etc/rsyslog.conf Configuration file for rsyslog v3.
#
# For more information see
# /usr/share/doc/rsyslog-doc/html/rsyslog_conf.html
#################
#### MODULES ####
#################
$ModLoad imuxsock # provides support for local system logging
$ModLoad imklog # provides kernel logging support (previously done by rklogd)
#$ModLoad immark # provides --MARK-- message capability
# provides UDP syslog reception
#$ModLoad imudp
#$UDPServerRun 514
# provides TCP syslog reception
#$ModLoad imtcp
#$InputTCPServerRun 514
###########################
#### GLOBAL DIRECTIVES ####
###########################
#
# Use default timestamp format.
# To enable high precision timestamps, comment out the following line.
#
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
#
# Set the default permissions for all log files.
#
$FileOwner root
$FileGroup adm
$FileCreateMode 0640
#
# Include all config files in /etc/rsyslog.d/
#
$IncludeConfig /etc/rsyslog.d/*.conf
###############
#### RULES ####
###############
#
# First some standard log files. Log by facility.
#
auth,authpriv.* /var/log/auth.log
*.*;auth,authpriv.none -/var/log/syslog
#cron.* /var/log/cron.log
daemon.* -/var/log/daemon.log
kern.* -/var/log/kern.log
lpr.* -/var/log/lpr.log
#mail.* -/var/log/mail.log
user.* -/var/log/user.log
#
# Logging for the mail system. Split it up so that
# it is easy to write scripts to parse these files.
#
mail.info -/var/log/mail.info
mail.warn -/var/log/mail.warn
mail.err /var/log/mail.err
#
# Logging for INN news system.
#
news.crit /var/log/news/news.crit
news.err /var/log/news/news.err
news.notice -/var/log/news/news.notice
#
# Some "catch-all" log files.
#
*.=debug;\
auth,authpriv.none;\
news.none;mail.none -/var/log/debug
*.=info;*.=notice;*.=warn;\
auth,authpriv.none;\
cron,daemon.none;\
mail,news.none -/var/log/messages
#
# Emergencies are sent to everybody logged in.
#
*.emerg *
#
# I like to have messages displayed on the console, but only on a virtual
# console I usually leave idle.
#
#daemon,mail.*;\
# news.=crit;news.=err;news.=notice;\
# *.=debug;*.=info;\
# *.=notice;*.=warn /dev/tty8
# The named pipe /dev/xconsole is for the `xconsole' utility. To use it,
# you must invoke `xconsole' with the `-file' option:
#
# $ xconsole -file /dev/xconsole [...]
#
# NOTE: adjust the list below, or you'll go crazy if you have a reasonably
# busy site..
#
daemon.*;mail.*;\
news.err;\
*.=debug;*.=info;\
*.=notice;*.=warn |/dev/xconsole
-------------- next part --------------
#$ModLoad omrelp
$WorkDirectory /var/log/rsyslog
$ActionQueueType LinkedList # use asynchronous processing
$ActionQueueFileName srvrfwd # set file name, also enables disk mode
$ActionResumeRetryCount -1 # infinite retries on insert failure
$ActionQueueSaveOnShutdown on # save in-memory data if rsyslog shuts down
mail.* @@<remote_addr>:514
$ActionExecOnlyIfPreviousIsSuspended on
& /var/log/rsyslog/failover.log
$ActionExecOnlyIfPreviousIsSuspended off
-------------- next part --------------
GNU gdb 6.8-debian
Copyright (C) 2008 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu"...
Reading symbols from /usr/lib/libz.so.1...done.
Loaded symbols for /usr/lib/libz.so.1
Reading symbols from /lib/libpthread.so.0...Reading symbols from /usr/lib/debug/lib/libpthread-2.7.so...done.
done.
Loaded symbols for /lib/libpthread.so.0
Reading symbols from /lib/libdl.so.2...Reading symbols from /usr/lib/debug/lib/libdl-2.7.so...done.
done.
Loaded symbols for /lib/libdl.so.2
Reading symbols from /lib/librt.so.1...Reading symbols from /usr/lib/debug/lib/librt-2.7.so...done.
done.
Loaded symbols for /lib/librt.so.1
Reading symbols from /lib/libc.so.6...Reading symbols from /usr/lib/debug/lib/libc-2.7.so...done.
done.
Loaded symbols for /lib/libc.so.6
Reading symbols from /lib/ld-linux-x86-64.so.2...Reading symbols from /usr/lib/debug/lib/ld-2.7.so...done.
done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /usr/lib/rsyslog/lmnet.so...done.
Loaded symbols for /usr/lib/rsyslog/lmnet.so
Reading symbols from /lib/libnss_files.so.2...Reading symbols from /usr/lib/debug/lib/libnss_files-2.7.so...done.
done.
Loaded symbols for /lib/libnss_files.so.2
Reading symbols from /usr/lib/rsyslog/lmtcpclt.so...done.
Loaded symbols for /usr/lib/rsyslog/lmtcpclt.so
Reading symbols from /usr/lib/rsyslog/imuxsock.so...done.
Loaded symbols for /usr/lib/rsyslog/imuxsock.so
Reading symbols from /usr/lib/rsyslog/imklog.so...done.
Loaded symbols for /usr/lib/rsyslog/imklog.so
Reading symbols from /lib/libnss_compat.so.2...Reading symbols from /usr/lib/debug/lib/libnss_compat-2.7.so...done.
done.
Loaded symbols for /lib/libnss_compat.so.2
Reading symbols from /lib/libnsl.so.1...Reading symbols from /usr/lib/debug/lib/libnsl-2.7.so...done.
done.
Loaded symbols for /lib/libnsl.so.1
Reading symbols from /lib/libnss_nis.so.2...Reading symbols from /usr/lib/debug/lib/libnss_nis-2.7.so...done.
done.
Loaded symbols for /lib/libnss_nis.so.2
Reading symbols from /lib/libgcc_s.so.1...done.
Loaded symbols for /lib/libgcc_s.so.1
Core was generated by `/usr/sbin/rsyslogd -c3'.
Program terminated with signal 6, Aborted.
[New process 23847]
[New process 23849]
[New process 23848]
[New process 23846]
[New process 23845]
#0 0x00002b479d1c3ef5 in raise () from /lib/libc.so.6
(gdb)
Thread 5 (process 23845):
#0 0x00002b479d25af12 in select () from /lib/libc.so.6
No symbol table info available.
#1 0x000000000040e0c9 in realMain (argc=<value optimized out>,
argv=<value optimized out>) at syslogd.c:2563
iRet = <value optimized out>
i = <value optimized out>
p = <value optimized out>
num_fds = <value optimized out>
ch = <value optimized out>
hent = <value optimized out>
sigAct = {__sigaction_handler = {sa_handler = 0x1,
sa_sigaction = 0x1}, sa_mask = {__val = {0 <repeats 16 times>}},
sa_flags = 0, sa_restorer = 0}
bIsFirstOption = <value optimized out>
bEOptionWasGiven = <value optimized out>
bImUxSockLoaded = <value optimized out>
arg = <value optimized out>
legacyConfLine = '\0' <repeats 79 times>
#2 0x00002b479d1b01a6 in __libc_start_main () from /lib/libc.so.6
No symbol table info available.
#3 0x0000000000409609 in _start ()
No locals.
Thread 4 (process 23846):
#0 0x00002b479cb72fad in pthread_cond_timedwait@@GLIBC_2.3.2 ()
from /lib/libpthread.so.0
No symbol table info available.
#1 0x0000000000418e00 in wtiWorker (pThis=0x64f4a0) at wti.c:403
__cancel_buf = {__cancel_jmp_buf = {{__cancel_jmp_buf = {1082130748,
9093895273491594810, 47586571945984, 47586571906816, 0, 4096,
9094035857682333242, 9093894727638318650}, __mask_was_saved = 0}},
__pad = {0x408000d0, 0x0, 0x40800950, 0x40800950}}
not_first_call = <value optimized out>
iCancelStateSave = 0
t = {tv_sec = 1221135378, tv_nsec = 723151645}
pWtp = (wtp_t *) 0x64f360
bInactivityTOOccured = 0
#2 0x0000000000418032 in wtpWorker (arg=<value optimized out>) at wtp.c:417
__cancel_buf = {__cancel_jmp_buf = {{__cancel_jmp_buf = {1082130512,
9093895273491594810, 47586571945984, 47586571906816, 0, 4096,
9094034911474613818, 9093894729333866042}, __mask_was_saved = 0}},
__pad = {0x408001e0, 0x0, 0x40800950, 0x40800950}}
not_first_call = <value optimized out>
iCancelStateSave = 0
pWti = (wti_t *) 0x64f4a0
pThis = (wtp_t *) 0x64f360
sigSet = {__val = {18446744067267100671,
18446744073709551615 <repeats 15 times>}}
#3 0x00002b479cb6efc7 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#4 0x00002b479d2617cd in clone () from /lib/libc.so.6
No symbol table info available.
#5 0x0000000000000000 in ?? ()
No symbol table info available.
Thread 3 (process 23848):
#0 0x00002b479d25af12 in select () from /lib/libc.so.6
No symbol table info available.
#1 0x00002b479db01413 in runInput (pThrd=<value optimized out>)
at imuxsock.c:235
maxfds = 0
nfds = <value optimized out>
i = <value optimized out>
fd = <value optimized out>
readfds = {fds_bits = {1, 0 <repeats 15 times>}}
#2 0x0000000000416420 in thrdStarter (arg=0x66bd70) at threads.c:139
iRet = <value optimized out>
sigSet = {__val = {18446744067267100671,
18446744073709551615 <repeats 15 times>}}
#3 0x00002b479cb6efc7 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#4 0x00002b479d2617cd in clone () from /lib/libc.so.6
No symbol table info available.
#5 0x0000000000000000 in ?? ()
No symbol table info available.
Thread 2 (process 23849):
#0 0x00002b479cb757eb in read () from /lib/libpthread.so.0
No symbol table info available.
#1 0x00002b479dd05c5f in klogLogKMsg () at linux.c:454
No locals.
#2 0x00002b479dd0517c in runInput (pThrd=0x66c1f0) at imklog.c:208
iRet = <value optimized out>
#3 0x0000000000416420 in thrdStarter (arg=0x66c1f0) at threads.c:139
iRet = <value optimized out>
sigSet = {__val = {18446744067267100671,
18446744073709551615 <repeats 15 times>}}
#4 0x00002b479cb6efc7 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#5 0x00002b479d2617cd in clone () from /lib/libc.so.6
No symbol table info available.
#6 0x0000000000000000 in ?? ()
No symbol table info available.
Thread 1 (process 23847):
#0 0x00002b479d1c3ef5 in raise () from /lib/libc.so.6
No symbol table info available.
#1 0x00002b479d1c5413 in abort () from /lib/libc.so.6
No symbol table info available.
#2 0x0000000000411abf in sigsegvHdlr (signum=6) at debug.c:741
signame = <value optimized out>
sigAct = {__sigaction_handler = {sa_handler = 0, sa_sigaction = 0},
sa_mask = {__val = {0 <repeats 16 times>}}, sa_flags = 0, sa_restorer = 0}
#3 <signal handler called>
No symbol table info available.
#4 0x00002b479d1c3ef5 in raise () from /lib/libc.so.6
No symbol table info available.
#5 0x00002b479d1c5413 in abort () from /lib/libc.so.6
No symbol table info available.
#6 0x00002b479d2003e8 in __libc_message () from /lib/libc.so.6
No symbol table info available.
#7 0x00002b479d205968 in malloc_printerr () from /lib/libc.so.6
No symbol table info available.
#8 0x00002b479d207a76 in free () from /lib/libc.so.6
No symbol table info available.
#9 0x0000000000420f2d in msgDestruct (ppThis=0x6532a0) at msg.c:284
iCancelStateSave = 0
pThis = (msg_t *) 0x2aaab401e480
#10 0x000000000042a797 in actionCallAction (pAction=0x653240,
pMsg=0x2aaab0020620) at action.c:661
__cancel_buf = {__cancel_jmp_buf = {{__cancel_jmp_buf = {1090522832,
9093895273491594810, 6631600, 1090522832, 4238944, 1090522856,
9094038209632009786, 9093894735514959418}, __mask_was_saved = 0}},
__pad = {0x41000f70, 0x0, 0xbf11, 0x0}}
__cancel_arg = (void *) 0x6532f0
not_first_call = <value optimized out>
iRet = <value optimized out>
iCancelStateSave = 0
#11 0x000000000040ae79 in processMsgDoActions (pData=0x5d25, pParam=0x41000ed0)
at syslogd.c:1094
iRet = <value optimized out>
iRetMod = <value optimized out>
#12 0x0000000000429b97 in llExecFunc (pThis=0x6530b0,
pFunc=0x40ae60 <processMsgDoActions>, pParam=0x41000ed0)
at linkedlist.c:390
iRet = RS_RET_OK
llCookie = (linkedListCookie_t) 0x652c60
llCookiePrev = (linkedListCookie_t) 0x0
#13 0x000000000040aab0 in msgConsumer (notNeeded=<value optimized out>,
pUsr=<value optimized out>) at syslogd.c:1137
pMsg = (msg_t *) 0x2aaab0020620
#14 0x000000000041aa31 in queueConsumerReg (pThis=0x656980, pWti=0x64bcf0,
iCancelStateSave=<value optimized out>) at queue.c:1570
iRet = RS_RET_OK
#15 0x0000000000418e6f in wtiWorker (pThis=0x64bcf0) at wti.c:413
__cancel_buf = {__cancel_jmp_buf = {{__cancel_jmp_buf = {1090523452,
9093895273491594810, 47586571945984, 47586571906816, 0, 4096,
9094038209713798714, 9093894727638318650}, __mask_was_saved = 0}},
__pad = {0x410010d0, 0x0, 0x0, 0x0}}
not_first_call = <value optimized out>
iCancelStateSave = 0
t = {tv_sec = 1221135378, tv_nsec = 595602238}
pWtp = (wtp_t *) 0x64bbb0
bInactivityTOOccured = 0
#16 0x0000000000418032 in wtpWorker (arg=<value optimized out>) at wtp.c:417
__cancel_buf = {__cancel_jmp_buf = {{__cancel_jmp_buf = {1090523216,
9093895273491594810, 47586571945984, 47586571906816, 0, 4096,
9094038210546368058, 9093894729333866042}, __mask_was_saved = 0}},
__pad = {0x410011e0, 0x0, 0x0, 0x0}}
not_first_call = <value optimized out>
iCancelStateSave = 0
pWti = (wti_t *) 0x64bcf0
pThis = (wtp_t *) 0x64bbb0
sigSet = {__val = {18446744067267100671,
18446744073709551615 <repeats 15 times>}}
#17 0x00002b479cb6efc7 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#18 0x00002b479d2617cd in clone () from /lib/libc.so.6
No symbol table info available.
#19 0x0000000000000000 in ?? ()
No symbol table info available.
(gdb) quit