Hi,
This is what I got from RSYSLOG_DebugFormat, sorry about newlines. I'm
using omprog and are expecting to receive a json for each message without
LF so each newline is taken from my error logs
PS, the data is anonymized
'Debug line with all properties:\n'
"FROMHOST: '172.18.0.2', fromhost-ip: '172.18.0.2', HOSTNAME: '172.18.0.2',
PRI: 15,\n"
"syslogtag '', programname: '', APP-NAME: '', PROCID: '-', MSGID: '-',\n"
"TIMESTAMP: 'May 12 17:53:30', STRUCTURED-DATA: '-',\n"
'msg: \'2022-05-12T17:53:30 Feb 23 22:09:40 laptop 1,2020/02/23
22:09:40,fcc5dd0de335,SYSTEM,general,0,2020/02/23
22:09:40,,general,,0,0,general,informational,"Connection to Update server:
updates.paloaltonetworks.com completed successfully, initiated by
172.28.125.10",139576,0x0,0,0,0,0,,laptop -\'\n'
'escaped msg: \'2022-05-12T17:53:30 Feb 23 22:09:40 laptop 1,2020/02/23
22:09:40,fcc5dd0de335,SYSTEM,general,0,2020/02/23
22:09:40,,general,,0,0,general,informational,"Connection to Update server:
updates.paloaltonetworks.com completed successfully, initiated by
172.28.125.10",139576,0x0,0,0,0,0,,laptop -\'\n'
'inputname: udp rawmsg: \'<15>1 2022-05-12T17:53:30 Feb 23 22:09:40 laptop
1,2020/02/23 22:09:40,fcc5dd0de335,SYSTEM,general,0,2020/02/23
22:09:40,,general,,0,0,general,informational,"Connection to Update server:
updates.paloaltonetworks.com completed successfully, initiated by
172.28.125.10",139576,0x0,0,0,0,0,,laptop -\'\n'
'$!:\n'
'$.:\n'
'$/:\n'
'\n'
It seems rsyslog fails to set variables since they are all empty?
I tried to run with minimal configuration with the exact same result.
Is there a global parameter to turn off variable support?
Down below is my entire configuration
Best regards Johan Ryberg
global(
WorkDirectory="/var/cache/syslog"
)
module(load="omprog")
module(load="mmutf8fix")
module(load="imtcp")
module(load="imudp")
input(type="imtcp"
name="tcp"
port="514"
ruleset="azure_syslog")
input(type="imudp"
name="udp"
port="514"
ruleset="azure_syslog")
module(load="imfile"
mode="inotify")
input(type="imfile"
file="/var/lib/docker/containers/*/*.log"
tag="docker"
reopenOnTruncate="on"
ruleset="azure_docker")
template(name="ms" type="string"
string="%timereported:1:3:date-subseconds%")
template(name="new_unix" type="string"
string="%timereported:::date-unixtimestamp%%$.ms%")
set $.ms = exec_template("ms");
if ($.ms == "0") then {
set $.ms = "000";
}
set $.new_unix = exec_template("new_unix");
set $.test = "test string";
template(name="blobstorage" type="list" option.jsonf="on") {
property(outname="timestamp" name="timereported" dateFormat="rfc3339"
format="jsonf")
property(outname="unixtime" name="timereported"
dateFormat="unixtimestamp" format="jsonf")
property(outname="host" name="fromhost-ip" format="jsonf")
property(outname="test" name="$.test" format="jsonf")
property(outname="message" name="rawmsg-after-pri" format="jsonf")
property(outname="log_id" name="uuid" format="jsonf")
}
ruleset(name="azure_syslog") {
action(type="mmutf8fix")
action(
type="omprog"
name="azure_omprog"
action.resumeInterval="5"
binary="/usr/local/bin/syslog.py syslog"
confirmMessages="on"
confirmTimeout="180000"
killUnresponsive="on"
closeTimeout="200000"
output="/dev/stdout"
queue.type="fixedArray"
queue.size="32768"
queue.dequeueBatchSize="4096"
queue.workerThreads="1"
queue.workerThreadMinimumMessages="4096"
template="RSYSLOG_DebugFormat"
)
}
ruleset(name="azure_docker") {
action(type="mmutf8fix")
action(
type="omprog"
name="azure_omprog"
action.resumeInterval="5"
binary="/usr/local/bin/syslog.py docker"
confirmMessages="on"
confirmTimeout="180000"
killUnresponsive="on"
closeTimeout="200000"
output="/dev/stdout"
queue.type="fixedArray"
queue.size="32768"
queue.dequeueBatchSize="4096"
queue.workerThreads="1"
queue.workerThreadMinimumMessages="4096"
template="blobstorage"
)
}
module(
load="impstats"
interval="60"
format="json"
resetCounters="off"
ruleset="process_stats"
)
ruleset(name="process_stats") {
action(
type="omprog"
name="to_exporter"
binary="/usr/local/bin/rsyslog_exporter --web.listen-address=
127.0.0.1:9104"
)
On Thu, 12 May 2022 at 18:47, David Lang <david@lang.hm> wrote:
> when you have issues like this, it's a good idea to log the message with
> the
> template RSYSLOG_DebugFormat as it shows the contents of almost all the
> variables you can be working with.
>
> David Lang
>
> On Thu, 12 May 2022, Johan Ryberg via rsyslog wrote:
>
> > Date: Thu, 12 May 2022 10:24:34 +0200
> > From: Johan Ryberg via rsyslog <rsyslog@lists.adiscon.com>
> > To: Rainer Gerhards <rgerhards@hq.adiscon.com>
> > Cc: Johan Ryberg <johan@securit.se>, rsyslog-users <
> rsyslog@lists.adiscon.com>
> > Subject: Re: [rsyslog] Ensure unixtimestamp with milliseconds?
> >
> > Please ignore comment about missing "host", that entry came from imfile,
> > local file digest.
> >
> > // Johan
> >
> > On Thu, 12 May 2022 at 10:20, Johan Ryberg <johan@securit.se> wrote:
> >
> >> Thanks for you reply.
> >>
> >> I added ms and that is empty as well
> >>
> >> template(name="ms" type="string"
> >> string="%timereported:1:3:date-subseconds%")
> >> template(name="new_unix" type="string"
> >> string="%timereported:::date-unixtimestamp%%$.ms%")
> >> set $.ms = exec_template("ms");
> >> if ($.ms == "0") then {
> >> set $.ms = "000";
> >> }
> >> set $.new_unix = exec_template("new_unix");
> >>
> >> template(name="blobstorage" type="list" option.jsonf="on") {
> >> property(outname="timestamp" name="timereported"
> dateFormat="rfc3339"
> >> format="jsonf")
> >> property(outname="unixtime" name="timereported"
> >> dateFormat="unixtimestamp" format="jsonf")
> >> property(outname="ms" name="$.ms" format="jsonf")
> >> property(outname="new_unix" name="$.new_unix" format="jsonf")
> >> property(outname="host" name="fromhost-ip" format="jsonf")
> >> property(outname="message" name="rawmsg-after-pri" format="jsonf")
> >> property(outname="log_id" name="uuid" format="jsonf")
> >> }
> >>
> >> Output: {"timestamp":"2022-05-12T08:13:32.250744+00:00",
> >> "unixtime":"1652343212", "ms":"", "new_unix":"", "host":"",
> >> "message":"{\"log\":<redacted>",
> >> "log_id":"D3E23BB3CAFE4F68BE2AE4804214228D"}
> >>
> >> host is also empty, fun part if I move "host" above "ms" then I got
> values
> >> for host.
> >>
> >> Any clue how to troubleshoot this?
> >>
> >> Best regards Johan Ryberg
> >>
> >> On Thu, 12 May 2022 at 09:37, Rainer Gerhards <rgerhards@hq.adiscon.com
> >
> >> wrote:
> >>
> >>> I would suggest to output $.ms as well. Also, simplify the new_unix
> >>> timeplate to just contain the default timestamp, see if it works and
> >>> than go from there with more complex processing.
> >>>
> >>> all in all, it doesn't look wrong - probably a detail.
> >>>
> >>> HTH
> >>> Rainer
> >>>
> >>> El jue, 12 may 2022 a las 9:28, Johan Ryberg via rsyslog
> >>> (<rsyslog@lists.adiscon.com>) escribió:
> >>> >
> >>> > Hi,
> >>> >
> >>> > Running latest version of rsyslog (8.2204.1)
> >>> >
> >>> > I'm trying to create a template that ensures unixtimestamp with
> >>> > milliseconds.
> >>> >
> >>> > This is as far as I got
> >>> >
> >>> > template(name="ms" type="string"
> >>> > string="%timereported:1:3:date-subseconds%")
> >>> > set $!ms = exec_template("ms");
> >>> > if ($.ms == "0") then {
> >>> > set $.ms = "000";
> >>> > }
> >>> > template(name="new_unix" type="string"
> >>> > string="%timereported:::date-unixtimestamp%%$.ms%")
> >>> > set $!new_unix = exec_template("new_unix");
> >>> >
> >>> > template(name="storage" type="list" option.jsonf="on") {
> >>> > property(outname="timestamp" name="timereported"
> >>> dateFormat="rfc3339"
> >>> > format="jsonf")
> >>> > property(outname="unixtime" name="$!new_unix" format="jsonf")
> >>> > property(outname="host" name="fromhost-ip" format="jsonf")
> >>> > property(outname="message" name="rawmsg-after-pri"
> format="jsonf")
> >>> > property(outname="log_id" name="uuid" format="jsonf")
> >>> > }
> >>> >
> >>> > Sadly the output of unixtime is always empty, "unixtime": ""
> >>> >
> >>> > I would very much appreciate some guidance how to make it work and if
> >>> > possible, more efficient
> >>> >
> >>> > Best regards Johan Ryberg
> >>> > _______________________________________________
> >>> > rsyslog mailing list
> >>> > https://lists.adiscon.net/mailman/listinfo/rsyslog
> >>> > http://www.rsyslog.com/professional-services/
> >>> > What's up with rsyslog? Follow https://twitter.com/rgerhards
> >>> > NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a
> >>> myriad of sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST
> if
> >>> you DON'T LIKE THAT.
> >>>
> >>
> > _______________________________________________
> > rsyslog mailing list
> > https://lists.adiscon.net/mailman/listinfo/rsyslog
> > http://www.rsyslog.com/professional-services/
> > What's up with rsyslog? Follow https://twitter.com/rgerhards
> > NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a myriad
> of sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if you
> DON'T LIKE THAT.
_______________________________________________
rsyslog mailing list
https://lists.adiscon.net/mailman/listinfo/rsyslog http://www.rsyslog.com/professional-services/ What's up with rsyslog? Follow
https://twitter.com/rgerhards NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a myriad of sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if you DON'T LIKE THAT.