Mailing List Archive: Cant parse incoming json via relp

Hello

I have two rsyslog servers - sender and receiver
Sender get data, convert it to json and send via relp to receiver
But receiver can't parse json

1. Sender config
module(load="omrelp")

template(
name="json_syslog"
type="list"
option.json="on"
) {
constant(value="{")
constant(value="\"@timestamp\":\"") property(name="timereported"
dateFormat="rfc3339")
constant(value="\",\"type\":\"syslog_json")
constant(value="\",\"syslogtag\":\"") property(name="syslogtag"
format="json")
constant(value="\",\"relayhost\":\"") property(name="fromhost")
constant(value="\",\"relayip\":\"") property(name="fromhost-ip")
constant(value="\",\"logsource\":\"") property(name="source")
constant(value="\",\"hostname\":\"") property(name="hostname"
caseconversion="lower")
constant(value="\",\"programname\":\"")
property(name="programname")
constant(value="\",\"source\":\"") property(name="app-name"
caseConversion="lower" onEmpty="null")
constant(value="\",\"priority\":\"") property(name="pri")
constant(value="\",\"severity\":\"")
property(name="syslogseverity" caseConversion="upper")
constant(value="\",\"facility\":\"")
property(name="syslogfacility")
constant(value="\",\"severity_label\":\"")
property(name="syslogseverity-text")
constant(value="\",\"facility_label\":\"")
property(name="syslogfacility-text")
constant(value="\",\"msg\":\"") property(name="msg" format="json")
constant(value="\",\"end_msg\":\"")
constant(value="\"}\n")
}

ruleset(
name="syslogCollector"
) {

action(
type="omrelp"
target="docker-swarm.dc1.virtel.net" port="10514"
template="json_syslog"

queue.type="LinkedList"
queue.size="4000"
queue.timeoutEnqueue="0" # timeout for reject new messages if queue
is full

action.resumeRetryCount="0"
action.reportSuspension="on"
action.reportSuspensionContinuation="on"
action.resumeInterval="10"
)
}

2. Receiver config
module(load="imrelp")
input(
port="10514"
type="imrelp"
name="imrelp"
ruleset="relpCollector"
)

# ----------
# Rulesets
# Must be in main file only !!!!!!!!!!
# ----------
ruleset(
name="relpCollector"
) {

action(type="mmjsonparse")

if $parsesuccess == "OK" then {
$IncludeConfig /etc/rsyslog.d/*.conf
stop
}

action(
type="omfile"
file="/var/log/json_parse_error.log"
ioBufferSize="64k"
flushOnTXEnd="on"
)
}

3. Send message to first server
logger -n 192.168.9.10 -P 514 -T -t myapp "This is only test message -----
remote"

4. Server1 send to Server2 valid json like a
{"@timestamp":"2024-02-14T15:47:50.323104+03:00","type":"syslog_json","syslogtag":"myapp","relayhost":"172.26.0.1","relayip":"172.26.0.1","logsource":"
docker-swarm.dc1.virtel.net","hostname":"devhost","programname":"myapp","source":"myapp","priority":"13","severity":"5","facility":"1","severity_label":"notice","facility_label":"user","msg":"This
is only test message ----- remote","end_msg":""}

5. Server2 cant parse json and store message into
/var/log/json_parse_error.log
_______________________________________________
rsyslog mailing list
https://lists.adiscon.net/mailman/listinfo/rsyslog
http://www.rsyslog.com/professional-services/
What's up with rsyslog? Follow https://twitter.com/rgerhards
NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a myriad of sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if you DON'T LIKE THAT.

what is the config of the receiver?

I'll note that what you are sending is valid json, but not a valid syslog
message, that could be why you are running into grief.

log using the template RSYSLOG_DebugFormat on the receiver so we can see exactly
what it's getting.

Also, you have a comment that rulesets must be in the main file, that is
incorrect. Rsyslog combines all files before any config parsing is done, so at
the time that it's parsing the config, it doesn't know what file any line came
from. You could (but shouldn't) have your template() definition spread across
multiple files as an extreme example.

David Lang

On Wed, 14 Feb 2024, Alex via rsyslog wrote:

> Hello
>
> I have two rsyslog servers - sender and receiver
> Sender get data, convert it to json and send via relp to receiver
> But receiver can't parse json
>
> 1. Sender config
> module(load="omrelp")
>
> template(
> name="json_syslog"
> type="list"
> option.json="on"
> ) {
> constant(value="{")
> constant(value="\"@timestamp\":\"") property(name="timereported"
> dateFormat="rfc3339")
> constant(value="\",\"type\":\"syslog_json")
> constant(value="\",\"syslogtag\":\"") property(name="syslogtag"
> format="json")
> constant(value="\",\"relayhost\":\"") property(name="fromhost")
> constant(value="\",\"relayip\":\"") property(name="fromhost-ip")
> constant(value="\",\"logsource\":\"") property(name="source")
> constant(value="\",\"hostname\":\"") property(name="hostname"
> caseconversion="lower")
> constant(value="\",\"programname\":\"")
> property(name="programname")
> constant(value="\",\"source\":\"") property(name="app-name"
> caseConversion="lower" onEmpty="null")
> constant(value="\",\"priority\":\"") property(name="pri")
> constant(value="\",\"severity\":\"")
> property(name="syslogseverity" caseConversion="upper")
> constant(value="\",\"facility\":\"")
> property(name="syslogfacility")
> constant(value="\",\"severity_label\":\"")
> property(name="syslogseverity-text")
> constant(value="\",\"facility_label\":\"")
> property(name="syslogfacility-text")
> constant(value="\",\"msg\":\"") property(name="msg" format="json")
> constant(value="\",\"end_msg\":\"")
> constant(value="\"}\n")
> }
>
> ruleset(
> name="syslogCollector"
> ) {
>
> action(
> type="omrelp"
> target="docker-swarm.dc1.virtel.net" port="10514"
> template="json_syslog"
>
> queue.type="LinkedList"
> queue.size="4000"
> queue.timeoutEnqueue="0" # timeout for reject new messages if queue
> is full
>
> action.resumeRetryCount="0"
> action.reportSuspension="on"
> action.reportSuspensionContinuation="on"
> action.resumeInterval="10"
> )
> }
>
> 2. Receiver config
> module(load="imrelp")
> input(
> port="10514"
> type="imrelp"
> name="imrelp"
> ruleset="relpCollector"
> )
>
> # ----------
> # Rulesets
> # Must be in main file only !!!!!!!!!!
> # ----------
> ruleset(
> name="relpCollector"
> ) {
>
> action(type="mmjsonparse")
>
> if $parsesuccess == "OK" then {
> $IncludeConfig /etc/rsyslog.d/*.conf
> stop
> }
>
> action(
> type="omfile"
> file="/var/log/json_parse_error.log"
> ioBufferSize="64k"
> flushOnTXEnd="on"
> )
> }
>
> 3. Send message to first server
> logger -n 192.168.9.10 -P 514 -T -t myapp "This is only test message -----
> remote"
>
> 4. Server1 send to Server2 valid json like a
> {"@timestamp":"2024-02-14T15:47:50.323104+03:00","type":"syslog_json","syslogtag":"myapp","relayhost":"172.26.0.1","relayip":"172.26.0.1","logsource":"
> docker-swarm.dc1.virtel.net","hostname":"devhost","programname":"myapp","source":"myapp","priority":"13","severity":"5","facility":"1","severity_label":"notice","facility_label":"user","msg":"This
> is only test message ----- remote","end_msg":""}
>
> 5. Server2 cant parse json and store message into
> /var/log/json_parse_error.log
> _______________________________________________
> rsyslog mailing list
> https://lists.adiscon.net/mailman/listinfo/rsyslog
> http://www.rsyslog.com/professional-services/
> What's up with rsyslog? Follow https://twitter.com/rgerhards
> NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a myriad of sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if you DON'T LIKE THAT.
>
_______________________________________________
rsyslog mailing list
https://lists.adiscon.net/mailman/listinfo/rsyslog
http://www.rsyslog.com/professional-services/
What's up with rsyslog? Follow https://twitter.com/rgerhards
NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a myriad of sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if you DON'T LIKE THAT.