Mailing List Archive

R: Re: R: Re: R: Re: received json log format
Hello
I have modified my template to have a "real" json lofg format
ruleset( name="udp-netlogs-tool"){
reset $.mstoredir = "/data/logs";
set $!log2json = replace( $msg, "=", ":");
set $!msg = replace( $!log2json, " LogRecord ", "");
#action( name="normalize" useRawMsg="off" type="mmnormalize"
rulebase="/etc/rsyslog.d/rulebase.rb" path="$!logrecord"
variable="$!msg")
#action( name="normalize" useRawMsg="off" type="mmnormalize"
rulebase="/etc/rsyslog.d/rulebase.rb" path="$!logrecord" )
action( name="normalize" useRawMsg="off" type="mmnormalize"
rulebase="/etc/rsyslog.d/rulebase.rb")
#action(type="mmnormalize" useRawMsg="off"
rulebase="/etc/rsyslog.d/rulebasetool.rb" path="$!normalized_msg"
variable="$!logrecord")
action(type="mmnormalize" useRawMsg="off"
rule=["rule=:%clienIp:ipv4%"] )
action( name="logs-udp"
type="omfile"
DynaFile="rmt_sys_netlogs_udp"
DirOwner="root"
DirGroup="netlogs"
FileOwner="root"
FileGroup="netlogs"
DirCreateMode="0750"
FileCreateMode="0640"
template="addFrmHstUDP"
#template="Foo"
#template="RSYSLOG_DebugFormat"
closeTimeout="2"
dynaFileCacheSize="2"
)
}

Now $msg it's: "msg": "{id:'null', date:1678183203520,
applicationInstanceId....inboundBytes:null, outboundBytes:null}",
and $! it's: "log2json": " LogRecord {id:'null',
date:1678186740986...inboundBytes:null, outboundBytes:null}",

I try to use rulebasejsape.rb:
rule=:%id:char-to%:, %date:date-rfc3164%,
%applicationInstanceId:char-to:,%, %domainName:word%, %serverIp:ipv4%,
%serverPort:number%, %clientIp:ipv4%, %clientPort:number%,
%sessionId:char-to:,%, %username:word%, %clientRequest:char-to:,%,
%clientMessage:char-to:,%, %serverStatus:char-to:,%,
%serverMessage:char-to:,%, %inboundBytes:number%, %outboundBytes:number%
but when I use this one the rsyslog generate a coredump...

Using my template to print log but I can't extract the values:
#template to add source ip
template( name="addFrmHstUDP" type="list")
{
property( name="fromhost-ip" )
constant( value=" " )
#property( name="$!logrecord" )
#constant( value=" " )
#property( name="$!.logrecord" )
#constant( value=" " )
#property( name="$!.!logrecord" )
#constant( value=" " )
#property( name="!logrecord" )
#constant( value=" " )
#property( name="!." )
#constant( value=" " )
property( name="$!msg!clientIp")
constant( value=" " )
property( name="!msg!clientIp")
constant( value=" " )
property( name="$!clientIp")
constant( value=" " )
property( name="!clientIp")
#property( name="!normalized_msg" format="json" )
property( name="$!msg!clientIp" format="json" )
#property( name="msg" )
constant( value="\n" )
}



------ Messaggio Originale ------
Da: amaury@tin.it
A: rsyslog@lists.adiscon.com
Inviato: venerdì 3 marzo 2023 15:39
Oggetto: Re: [rsyslog] R: Re: R: Re: received json log format

Hello
I try debugging with

with: action( name="normalize" type="mmnormalize" useRawMsg="off"
rulebase="/etc/rsyslog.d/rulebase.rb" path="$.logrecord" )
Debug line with all properties:
FROMHOST: 'public', fromhost-ip: 'xxx.xxx.xxx.xxx', HOSTNAME:
'xxx.xxx.xxx.xxx', PRI: 14,
syslogtag 'TOOL:', programname: 'TOOL', APP-NAME: 'TOOL', PROCID: '-',
MSGID: '-',
TIMESTAMP: 'Mar 3 09:18:40', STRUCTURED-DATA: '-',
msg: ' LogRecord {id='null', date=1677831520226,
applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
domainName='public', serverIp='yyy.yyy.yyy.yyy', serverPort=2222,
clientIp='hhh.hhh.hhh.hhh', clientPort=50748,
sessionId='6233bbbf-dda9-4627-b4b0-8b65012a5564', username='user02',
clientRequest='null', clientMessage='null', serverStatus='session
closed', serverMessage='SFTP/SCP', inboundBytes=null,
outboundBytes=null}'
escaped msg: ' LogRecord {id='null', date=1677831520226,
applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
domainName='public', serverIp='yyy.yyy.yyy.yyy', serverPort=2222,
clientIp='hhh.hhh.hhh.hhh', clientPort=50748,
sessionId='6233bbbf-dda9-4627-b4b0-8b65012a5564', username='user02',
clientRequest='null', clientMessage='null', serverStatus='session
closed', serverMessage='SFTP/SCP', inboundBytes=null,
outboundBytes=null}'
inputname: prt_65309udp rawmsg: '<14>Mar 3 09:18:40 xxx.xxx.xxx.xxx
TOOL: LogRecord {id='null', date=1677831520226,
applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
domainName='public', serverIp='yyy.yyy.yyy.yyy', serverPort=2222,
clientIp='hhh.hhh.hhh.hhh', clientPort=50748,
sessionId='6233bbbf-dda9-4627-b4b0-8b65012a5564', username='user02',
clientRequest='null', clientMessage='null', serverStatus='session
closed', serverMessage='SFTP/SCP', inboundBytes=null,
outboundBytes=null}'
$!:
$.:{ "mprefix": "public2", "=logrecord": { "originalmsg": " LogRecord
{id='null', date=1677831520226,
applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
domainName='public', serverIp='yyy.yyy.yyy.yyy', serverPort=2222,
clientIp='hhh.hhh.hhh.hhh', clientPort=50748,
sessionId='6233bbbf-dda9-4627-b4b0-8b65012a5564', username='user02',
clientRequest='null', clientMessage='null', serverStatus='session
closed', serverMessage='SFTP\/SCP', inboundBytes=null,
outboundBytes=null}", "unparsed-data": "{id='null', date=1677831520226,
applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
domainName='public', serverIp='yyy.yyy.yyy.yyy', serverPort=2222,
clientIp='hhh.hhh.hhh.hhh', clientPort=50748,
sessionId='6233bbbf-dda9-4627-b4b0-8b65012a5564', username='user02',
clientRequest='null', clientMessage='null', serverStatus='session
closed', serverMessage='SFTP\/SCP', inboundBytes=null,
outboundBytes=null}" } }
$/:


with: action( name="normalize" type="mmnormalize" useRawMsg="off"
rulebase="/etc/rsyslog.d/rulebase.rb" )
Debug line with all properties:
FROMHOST: 'public', fromhost-ip: 'xxx.xxx.xxx.xxx', HOSTNAME:
'xxx.xxx.xxx.xxx', PRI: 14,
syslogtag 'TOOL:', programname: 'TOOL', APP-NAME: 'TOOL', PROCID: '-',
MSGID: '-',
TIMESTAMP: 'Mar 3 09:25:25', STRUCTURED-DATA: '-',
msg: ' LogRecord {id='null', date=1677831925112,
applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
domainName='public', serverIp='yyy.yyy.yyy.yyy', serverPort=2222,
clientIp='zzz.zzz.zzz.zzz', clientPort=57022,
sessionId='60341819-0d0d-4c08-a61e-0e20450d2951', username='user01',
clientRequest='null', clientMessage='null', serverStatus='session
closed', serverMessage='SFTP/SCP', inboundBytes=null,
outboundBytes=null}'
escaped msg: ' LogRecord {id='null', date=1677831925112,
applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
domainName='public', serverIp='yyy.yyy.yyy.yyy', serverPort=2222,
clientIp='zzz.zzz.zzz.zzz', clientPort=57022,
sessionId='60341819-0d0d-4c08-a61e-0e20450d2951', username='user01',
clientRequest='null', clientMessage='null', serverStatus='session
closed', serverMessage='SFTP/SCP', inboundBytes=null,
outboundBytes=null}'
inputname: prt_65309udp rawmsg: '<14>Mar 3 09:25:25 xxx.xxx.xxx.xxx
TOOL: LogRecord {id='null', date=1677831925112,
applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
domainName='public', serverIp='yyy.yyy.yyy.yyy', serverPort=2222,
clientIp='zzz.zzz.zzz.zzz', clientPort=57022,
sessionId='60341819-0d0d-4c08-a61e-0e20450d2951', username='user01',
clientRequest='null', clientMessage='null', serverStatus='session
closed', serverMessage='SFTP/SCP', inboundBytes=null,
outboundBytes=null}'
$!:{ "originalmsg": " LogRecord {id='null', date=1677831925112,
applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
domainName='public', serverIp='yyy.yyy.yyy.yyy', serverPort=2222,
clientIp='zzz.zzz.zzz.zzz', clientPort=57022,
sessionId='60341819-0d0d-4c08-a61e-0e20450d2951', username='user01',
clientRequest='null', clientMessage='null', serverStatus='session
closed', serverMessage='SFTP\/SCP', inboundBytes=null,
outboundBytes=null}", "unparsed-data": "{id='null', date=1677831925112,
applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
domainName='public', serverIp='yyy.yyy.yyy.yyy', serverPort=2222,
clientIp='zzz.zzz.zzz.zzz', clientPort=57022,
sessionId='60341819-0d0d-4c08-a61e-0e20450d2951', username='user01',
clientRequest='null', clientMessage='null', serverStatus='session
closed', serverMessage='SFTP\/SCP', inboundBytes=null,
outboundBytes=null}" }
$.:{ "mprefix": "public2" }
$/:

When I try to get the values that I want to extract and order
No values I find

Below the configuration use:
rulebase: rule=: LogRecord %logrecord:json%

module(load="mmnormalize")

template( name="addFrmHstUDP" type="list")
{
property( name="fromhost-ip" )
constant( value=" " )
#property( name="$!logrecord" )
#constant( value=" " )
#property( name="$!.logrecord" )
#constant( value=" " )
#property( name="$!.!logrecord" )
#constant( value=" " )
#property( name="!logrecord" )
#constant( value=" " )
#property( name="!." )
#constant( value=" " )
#property( name="$!logrecord!clientIp" )
#property( name="$!.!clientIp" )
property( name="!normalized_msg" )
#property( name="msg" )
constant( value="\n" )
}

#template set log filename
template( name="rmt_sys_netlogs_udp" type="list")
{
property( name="$.mstoredir" )
constant( value="/" )
property( name="$.mprefix" )
constant( value="." )
property( name="timegenerated" dateformat="year" )
property( name="timegenerated" dateformat="month" )
property( name="timegenerated" dateformat="day" )
property( name="timegenerated" dateformat="hour" )
constant( value="00" )
}
#END TEMPLATE

ruleset( name="udp-netlogs-tool"){
reset $.mstoredir = "/data/logs;
action( name="normalize" useRawMsg="off" type="mmnormalize"
rulebase="/etc/rsyslog.d/rulebase.rb" path="$!logrecord")
set $!msg = $!logrecord;
action(type="mmnormalize" useRawMsg="off"
rule=["rule=:%clientIp:ipv4% "] path="$!normalized_msg"
variable="$!msg")
action( name="logs-udp"
type="omfile"
DynaFile="rmt_sys_netlogs_udp"
DirOwner="root"
DirGroup="netlogs"
FileOwner="root"
FileGroup="netlogs"
DirCreateMode="0750"
FileCreateMode="0640"
template="addFrmHstUDP"
#template="RSYSLOG_DebugFormat"
closeTimeout="2"
dynaFileCacheSize="2"
)
}

ruleset( name="dp65309udp" )
{
reset $.mprefix = "public2;
call udp-netlogs-tool
stop
}
input(name="prt_65309udp" type="imudp" port="65309"
ruleset="dp65309udp")

Cheers,
Maurizio

------ Messaggio Originale ------
Da: david@lang.hm
A: rsyslog@lists.adiscon.com
Cc: amaury@tin.it
Inviato: giovedì 2 marzo 2023 12:24
Oggetto: Re: [rsyslog] R: Re: R: Re: received json log format

On Thu, 2 Mar 2023, amaury--- via rsyslog wrote:

> Hi David
> thank you for your suggest.
> This is the debug:
>
> Debug line with all properties:
> FROMHOST: 'public.', fromhost-ip: 'xxx.xxx.xxx.xxx', HOSTNAME:
> 'xxx.xxx.xxx.xxx', PRI: 14,
> syslogtag 'TOOL:', programname: 'TOOL', APP-NAME: 'TOOL', PROCID:
'-',
> MSGID: '-',
> TIMESTAMP: 'Mar 2 10:10:45', STRUCTURED-DATA: '-',
> msg: ' LogRecord {id='null', date=1677748245215,
> applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
> domainName='public', serverIp='yyy.yyy.yyy.yyy', serverPort=9999,
> clientIp='94.138.173.87', clientPort=64012,
> sessionId='859b383e-7c51-48cf-9365-6e67a84cd29d', username='',
> clientRequest='null', clientMessage='null', serverStatus='session
> started', serverMessage='SFTP/SCP', inboundBytes=null,
> outboundBytes=null}'
> escaped msg: ' LogRecord {id='null', date=1677748245215,
> applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
> domainName='public', serverIp='yyy.yyy.yyy.yyy', serverPort=9999,
> clientIp='94.138.173.87', clientPort=64012,
> sessionId='859b383e-7c51-48cf-9365-6e67a84cd29d', username='',
> clientRequest='null', clientMessage='null', serverStatus='session
> started', serverMessage='SFTP/SCP', inboundBytes=null,
> outboundBytes=null}'
> inputname: prt_65309udp rawmsg: '<14>Mar 2 10:10:45
xxx.xxx.xxx.xxx
> TOOL: LogRecord {id='null', date=1677748245215,
> applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
> domainName='public', serverIp='yyy.yyy.yyy.yyy', serverPort=9999,
> clientIp='94.138.173.87', clientPort=64012,
> sessionId='859b383e-7c51-48cf-9365-6e67a84cd29d', username='',
> clientRequest='null', clientMessage='null', serverStatus='session
> started', serverMessage='SFTP/SCP', inboundBytes=null,
> outboundBytes=null}'
> $!:
> $.:{ "mprefix": "public2", "mstoredir": "\/data\/logs" }
> $/:
>
> I have create a file rulebase.rb:
> rule=:%date:date-rfc3164% %clientIp:ipv4% %clientPort:number%

where is the call to use this rule (the mmnormalize line)? are you
parsing msg or rawmsg?

msg: ' LogRecord {id='null', date=1677748245215,
applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
domainName='public', serverIp='yyy.yyy.yyy.yyy', serverPort=9999,
clientIp='94.138.173.87', clientPort=64012,
sessionId='859b383e-7c51-48cf-9365-6e67a84cd29d', username='',
clientRequest='null', clientMessage='null', serverStatus='session
started', serverMessage='SFTP/SCP', inboundBytes=null,
outboundBytes=null}'

rawmsg: '<14>Mar 2 10:10:45 xxx.xxx.xxx.xxx TOOL: LogRecord
{id='null', date=1677748245215,
applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
domainName='public', serverIp='yyy.yyy.yyy.yyy', serverPort=9999,
clientIp='94.138.173.87', clientPort=64012,
sessionId='859b383e-7c51-48cf-9365-6e67a84cd29d', username='',>
clientRequest='null', clientMessage='null', serverStatus='session
started', serverMessage='SFTP/SCP', inboundBytes=null,
outboundBytes=null}'

your rule doesn't match eithe one.

a rule like
rule=: LogRecord %logrecord:json%
would match the msg

rule=:<%pri:number%>%date:date-rfc3167% %hostname:word%
%syslogtag:word% LogRecorc %logrecord:json%
would match rawmsg


> With a template to print into file like this:
> #template to add source ip
> template( name="addFrmHstUDP" type="list")
> {
> property( name="fromhost-ip" )
> constant( value=" " )
> property( name="$!msg!date" )
> constant( value=" " )
> property( name="$!msg!clientIp" )
> constant( value=" " )
> property( name="$!msg!clientPort" )
> #property( name="msg" )
> constant( value="\n" )
> }
> but it's print only variable fromhost-ip.

you haven't shown that you have parsed anything yet, $! is blank (the
line $!: shows that.

David Lang

> Please how can I get the values?
> Thank you
> Cheers,
> Maurizio
>
>
>
>
> ------ Messaggio Originale ------
> Da: david@lang.hm
> A: rsyslog@lists.adiscon.com
> Cc: amaury@tin.it
> Inviato: mercoledì 1 marzo 2023 19:00
> Oggetto: Re: [rsyslog] R: Re: received json log format
>
> Ok, in that case you will need to use mmnormalize to parse
the
> log lines, and then a template to use the variables that you parse.
>
> the json type will be your friend for parsing the json, then log
the
> line with the RSYSLOG_DebugFormat to see all the variables it
creates
> and use them ($!foo!bar for multi-level json structures) and then
use
> those variables in a template.
>
> David Lang
>
> On Wed, 1 Mar 2023, amaury--- via rsyslog wrote:
>
>> Date: Wed, 1 Mar 2023 16:44:20 +0100 (CET)
>> From: amaury--- via rsyslog <rsyslog@lists.adiscon.com>
>> To: rsyslog@lists.adiscon.com
>> Cc: "amaury@tin.it"
>> Subject: [rsyslog] R: Re: received json log format
>>
>> Hi David
>> til the vendor have't changed the log ouput format I received the
> logs
>> with this:
>>
>> # Add this to reset the umask#
>> $umask 0000
>>
>> #BEGIN TEMPLATE
>>
>> #template to add source ip
>> template( name="addFrmHstUDP" type="list")
>> {
>> property( name="fromhost-ip" )
>> constant( value=" " )
>> property( name="msg" )
>> constant( value="\n" )
>> }
>>
>> #template set log filename
>> template( name="rmt_sys_netlogs_udp" type="list")
>> {
>> property( name="$.mstoredir" )
>> constant( value="/" )
>> property( name="$.mprefix" )
>> constant( value="." )
>> property( name="timegenerated" dateformat="year" )
>> property( name="timegenerated" dateformat="month" )
>> property( name="timegenerated" dateformat="day" )
>> property( name="timegenerated" dateformat="hour" )
>> constant( value="00" )
>> }
>> #END TEMPLATE
>>
>> ruleset( name="udp-netlogs-tool"){
>> reset $.mstoredir = "/data/logs;
>> action( name="logs-udp"
>> type="omfile"
>> DynaFile="rmt_sys_netlogs_udp"
>> DirOwner="root"
>> DirGroup="netlogs"
>> FileOwner="root"
>> FileGroup="netlogs"
>> DirCreateMode="0750"
>> FileCreateMode="0640"
>> template="addFrmHstUDP"
>> closeTimeout="2"
>> dynaFileCacheSize="2"
>> )
>> }
>>
>> ruleset( name="dp65309udp" )
>> {
>> reset $.mprefix = "public2;
>> call udp-netlogs-tool
>> }
>>
>> Now I receive the log that I have show you.
>>
>> Cheers,
>> Maurizio
>> ------ Messaggio Originale ------
>> Da: david@lang.hm
>> A: rsyslog@lists.adiscon.com
>> Cc: amaury@tin.it
>> Inviato: mercoledì 1 marzo 2023 14:10
>> Oggetto: Re: [rsyslog] received json log format
>>
>> please post your config so we can understand if what youa
> re
>> showing us is the result of your config or what is being sent to
> you.
>>
>> If it's what is being sent to you, you would use mmnormalize to
> parse
>> it into variables, then create a custom template to assemble the
> message
>> format that you want to write out and then output the message with
> your
>> template
>>
>> David Lang
>>
>> On Wed, 1 Mar 2023, amaury--- via rsyslog wrote:
>>
>>> Date: Wed, 1 Mar 2023 14:01:49 +0100 (CET)
>>> From: amaury--- via rsyslog <rsyslog@lists.adiscon.com>
>>> To: rsyslog@lists.adiscon.com
>>> Cc: "amaury@tin.it"
>>> Subject: [rsyslog] received json log format
>>>
>>> Hello
>>> I receive on rsyslog-8.2102 log json format like this:
>>> LogRecord {id='null', date=1677669932610,
>>> applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
>>> domainName='public', serverIp='null', serverPort=null,
>> clientIp='null',
>>> clientPort=null, sessionId='null', username='null',
>> clientRequest='null',
>>> clientMessage='null', serverStatus='trigger completed',
>>> serverMessage='trigger=move document;
>>> eventId=8a8d22d9-fd5c-451b-817b-699c706db5d6', inboundBytes=null,
>>> outboundBytes=null}
>>> LogRecord {id='null', date=1677669932644,
>>> applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
>>> domainName='public', serverIp='ddd.ddd.ddd.ddd', serverPort=hhhh,
>>> clientIp='kkk.kkk.kkk.kkk', clientPort=9999,
>>> sessionId='134e7eed-af8b-48a5-bd7c-0cb48013dfda',
> username='user01',
>>> clientRequest='null', clientMessage='null', serverStatus='logged
>> out',
>>> serverMessage='null', inboundBytes=null, outboundBytes=null}
>>> LogRecord {id='null', date=1677669932645,
>>> applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
>>> domainName='public', serverIp='ddd.ddd.ddd.ddd', serverPort=hhhh,
>>> clientIp='kkk.kkk.kkk.kkk', clientPort=9999,
>>> sessionId='134e7eed-af8b-48a5-bd7c-0cb48013dfda',
> username='user01',
>>> clientRequest='null', clientMessage='null', serverStatus='session
>> closed',
>>> serverMessage='SFTP/SCP', inboundBytes=null, outboundBytes=null}
>>>
>>> Please how I can to convert rewrite in log file something like
>>> null 1677669932610 5fc42f05-36ab-45ff-908d-e7b978a88269 public
null
>> null null
>>> null null null null trigger completed trigger=move document
>>> 8a8d22d9-fd5c-451b-817b-699c706db5d6 null null
>>> ?
>>> Thank you
>>> Maurizio
>>> _______________________________________________
>>> rsyslog mailing list
>>> https://lists.adiscon.net/mailman/listinfo/rsyslog
>>> http://www.rsyslog.com/professional-services/
>>> What's up with rsyslog? Follow https://twitter.com/rgerhards
>>> NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a
>> myriad of
>>> sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if
you
>> DON'T
>>> LIKE THAT.
>>>
>>
>>
>>
>>
>> _______________________________________________
>> rsyslog mailing list
>> https://lists.adiscon.net/mailman/listinfo/rsyslog
>> http://www.rsyslog.com/professional-services/
>> What's up with rsyslog? Follow https://twitter.com/rgerhards
>> NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a
> myriad of
>> sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if
you
> DON'T
>> LIKE THAT.
>
>
>
>
> _______________________________________________
> rsyslog mailing list
> https://lists.adiscon.net/mailman/listinfo/rsyslog
> http://www.rsyslog.com/professional-services/
> What's up with rsyslog? Follow https://twitter.com/rgerhards
> NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a
myriad of
> sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if you
DON'T
> LIKE THAT.








_______________________________________________
rsyslog mailing list
https://lists.adiscon.net/mailman/listinfo/rsyslog
http://www.rsyslog.com/professional-services/
What's up with rsyslog? Follow https://twitter.com/rgerhards
NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a myriad of sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if you DON'T LIKE THAT.
Re: R: Re: R: Re: R: Re: received json log format [ In reply to ]
On Tue, 7 Mar 2023, amaury--- via rsyslog wrote:

> Hello
> I have modified my template to have a "real" json lofg format
> ruleset( name="udp-netlogs-tool"){
> reset $.mstoredir = "/data/logs";
> set $!log2json = replace( $msg, "=", ":");
> set $!msg = replace( $!log2json, " LogRecord ", "");

what does a DebugFormat output look like at this point

> #action( name="normalize" useRawMsg="off" type="mmnormalize"
> rulebase="/etc/rsyslog.d/rulebase.rb" path="$!logrecord"
> variable="$!msg")
> #action( name="normalize" useRawMsg="off" type="mmnormalize"
> rulebase="/etc/rsyslog.d/rulebase.rb" path="$!logrecord" )
> action( name="normalize" useRawMsg="off" type="mmnormalize"
> rulebase="/etc/rsyslog.d/rulebase.rb")
> #action(type="mmnormalize" useRawMsg="off"
> rulebase="/etc/rsyslog.d/rulebasetool.rb" path="$!normalized_msg"
> variable="$!logrecord")
> action(type="mmnormalize" useRawMsg="off"
> rule=["rule=:%clienIp:ipv4%"] )

what are the various rb files, and what is the debugformat output after
attempting the parse

> Now $msg it's: "msg": "{id:'null', date:1678183203520,
> applicationInstanceId....inboundBytes:null, outboundBytes:null}",

this sounds like what $!msg should look like after the manipulations above

> and $! it's: "log2json": " LogRecord {id:'null',
> date:1678186740986...inboundBytes:null, outboundBytes:null}",

This matches what I would expect form the section above

> I try to use rulebasejsape.rb:
> rule=:%id:char-to%:, %date:date-rfc3164%,
> %applicationInstanceId:char-to:,%, %domainName:word%, %serverIp:ipv4%,
> %serverPort:number%, %clientIp:ipv4%, %clientPort:number%,
> %sessionId:char-to:,%, %username:word%, %clientRequest:char-to:,%,
> %clientMessage:char-to:,%, %serverStatus:char-to:,%,
> %serverMessage:char-to:,%, %inboundBytes:number%, %outboundBytes:number%
> but when I use this one the rsyslog generate a coredump...

what is the config where you tried to use this? does it coredump when it loads,
or when you get the first message that matches?

> Hello
> I try debugging with
>
> with: action( name="normalize" type="mmnormalize" useRawMsg="off"
> rulebase="/etc/rsyslog.d/rulebase.rb" path="$.logrecord" )
> Debug line with all properties:

what exactly was rulebase.rb when you tried this?

> msg: ' LogRecord {id='null', date=1677831520K226,
> applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
> domainName='public', serverIp='yyy.yyy.yyy.yyy', serverPort=2222,
> clientIp='hhh.hhh.hhh.hhh', clientPort=50748,
> sessionId='6233bbbf-dda9-4627-b4b0-8b65012a5564', username='user02',
> clientRequest='null', clientMessage='null', serverStatus='session
> closed', serverMessage='SFTP/SCP', inboundBytes=null,
> outboundBytes=null}'

when you have userawmsg off, this is what you are parsing

> rawmsg: '<14>Mar 3 09:18:40 xxx.xxx.xxx.xxx TOOL: LogRecord {id='null',
> date=1677831520226,
> applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
> domainName='public', serverIp='yyy.yyy.yyy.yyy', serverPort=2222,
> clientIp='hhh.hhh.hhh.hhh', clientPort=50748,
> sessionId='6233bbbf-dda9-4627-b4b0-8b65012a5564', username='user02',
> clientRequest='null', clientMessage='null', serverStatus='session closed',
> serverMessage='SFTP/SCP', inboundBytes=null, outboundBytes=null}'

if you were to parse the rawmsg, this is what you are seeing

neither of these match your rulebasejsape.rb you list above


> "=logrecord": { "originalmsg": " LogRecord {id='null', date=1677831520226,
> applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
> domainName='public', serverIp='yyy.yyy.yyy.yyy', serverPort=2222,
> clientIp='hhh.hhh.hhh.hhh', clientPort=50748,
> sessionId='6233bbbf-dda9-4627-b4b0-8b65012a5564', username='user02',
> clientRequest='null', clientMessage='null', serverStatus='session closed',
> serverMessage='SFTP\/SCP', inboundBytes=null, outboundBytes=null}",


> "unparsed-data": "{id='null', date=1677831520226,
> applicationInstanceId='5fc42f05-36ab-45ff-908d-e7b978a88269',
> domainName='public', serverIp='yyy.yyy.yyy.yyy', serverPort=2222,
> clientIp='hhh.hhh.hhh.hhh', clientPort=50748,
> sessionId='6233bbbf-dda9-4627-b4b0-8b65012a5564', username='user02',
> clientRequest='null', clientMessage='null', serverStatus='session closed',
> serverMessage='SFTP\/SCP', inboundBytes=null, outboundBytes=null}" } }

This section says that your rulebase is using the name '=logrecord' which seems
unintended.

the unparsed-data field is showing where it broke in parsing the message


David Lang
_______________________________________________
rsyslog mailing list
https://lists.adiscon.net/mailman/listinfo/rsyslog
http://www.rsyslog.com/professional-services/
What's up with rsyslog? Follow https://twitter.com/rgerhards
NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a myriad of sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if you DON'T LIKE THAT.