Mailing List Archive

Author: hege
Date: Thu May 19 09:47:40 2022
New Revision: 1901060

URL: http://svn.apache.org/viewvc?rev=1901060&view=rev
Log:
Some meta cleanups and optimizations (Bug 7987)
- Use rule_ready() everywhere instead of direct tests_already_hit modify
- Simple tracking of meta dependency hits, run do_meta_tests only when needed
- Do not run do_meta_tests on last priority, as finish_meta_tests will run anyway

Modified:
spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Conf/Parser.pm
spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/OneLineBodyRuleType.pm

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm?rev=1901060&r1=1901059&r2=1901060&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm Thu May 19 09:47:40 2022
@@ -4911,6 +4911,7 @@ sub new {

# meta dependencies
$self->{meta_dependencies} = {};
+ $self->{meta_deprules} = {};

# map eval function names to rulenames
$self->{eval_to_rule} = {};

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Conf/Parser.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Conf/Parser.pm?rev=1901060&r1=1901059&r2=1901060&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Conf/Parser.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Conf/Parser.pm Thu May 19 09:47:40 2022
@@ -1135,6 +1135,9 @@ sub compile_meta_rules {
foreach my $name (keys %meta) {
if (@{$rule_deps{$name}}) {
$conf->{meta_dependencies}->{$name} = $rule_deps{$name};
+ foreach my $deprule (@{$rule_deps{$name}}) {
+ $conf->{meta_deprules}->{$deprule}->{$name} = 1;
+ }
}
if ($unsolved_metas{$name}) {
$conf->{meta_tests}->{$name} = sub { 0 };

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm?rev=1901060&r1=1901059&r2=1901060&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm Thu May 19 09:47:40 2022
@@ -3121,7 +3121,8 @@ sub got_hit {
for $tflags_ref->{$rule};
};

- my $already_hit = $self->{tests_already_hit}->{$rule} || 0;
+ $self->rule_ready($rule, 1); # mark ready for metas
+ my $already_hit = $self->{tests_already_hit}->{$rule};
# don't count hits multiple times, unless 'tflags multiple' is on
if ($already_hit && ($tflags_ref->{$rule}||'') !~ /\bmultiple\b/) {
return;
@@ -3167,7 +3168,7 @@ sub got_hit {
return 1;
}

-=item $status->rule_ready ($rulename)
+=item $status->rule_ready ($rulename [, $no_async])

Mark an asynchronous rule ready, so it can be considered for meta rule
evaluation. Asynchronous rule is a rule whose eval-function returns undef,
@@ -3176,16 +3177,25 @@ $status->rule_ready() must be called lat
$status->got_hit() also does this. If neither is called, then any meta rule
that depends on this rule might not evaluate.

+Optional boolean $no_async skips checking if there are pending async DNS
+lookups for the rule.
+
=cut

sub rule_ready {
- my ($self, $rule) = @_;
+ my ($self, $rule, $no_async) = @_;

- if ($self->get_async_pending_rules($rule)) {
+ if (!$no_async && $self->get_async_pending_rules($rule)) {
# Can't be ready if there are pending DNS lookups, ignore for now.
return;
}

+ # record rules that depend on this, so do_meta_tests will be run
+ foreach (keys %{$self->{conf}->{meta_deprules}->{$rule}}) {
+ $self->{meta_check_ready}->{$_} = 1;
+ }
+
+ # mark ready
$self->{tests_already_hit}->{$rule} ||= 0;
}


Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm?rev=1901060&r1=1901059&r2=1901060&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm Thu May 19 09:47:40 2022
@@ -102,7 +102,8 @@ sub check_main {
# Make sure priority -100 exists for launching DNS
$pms->{conf}->{priorities}->{-100} ||= 1 if $do_dns;

- foreach my $priority (sort { $a <=> $b } keys %{$pms->{conf}->{priorities}}) {
+ my @priorities = sort { $a <=> $b } keys %{$pms->{conf}->{priorities}};
+ foreach my $priority (@priorities) {
# no need to run if there are no priorities at this level. This can
# happen in Conf.pm when we switch a rule from one priority to another
next unless ($pms->{conf}->{priorities}->{$priority} > 0);
@@ -182,7 +183,8 @@ sub check_main {
$pms->harvest_completed_queries() if $rbls_running;

# check for ready metas
- $self->do_meta_tests($pms, $priority);
+ # - if this is not last priority (finish_meta_tests will be run anyway)
+ $self->do_meta_tests($pms, $priority) if $priority != $priorities[-1];
}

# Finish DNS results
@@ -280,7 +282,9 @@ sub do_meta_tests {
}

return if $self->{am_compiling}; # nothing to compile here
+ return if !$pms->{meta_check_ready}; # nothing to check

+ my $mr = $pms->{meta_check_ready};
my $mp = $pms->{meta_pending};
my $md = $pms->{conf}->{meta_dependencies};
my $mt = $pms->{conf}->{meta_tests};
@@ -288,7 +292,7 @@ sub do_meta_tests {
my $retry;

RULE:
- foreach my $rulename (keys %$mp) {
+ foreach my $rulename (keys %$mr) {
# Meta is not ready if some dependency has not run yet
foreach my $deprule (@{$md->{$rulename}||[]}) {
if (!exists $h->{$deprule}) {
@@ -304,12 +308,15 @@ RULE:
dbg("rules-all: ran meta rule $rulename, no hit") if $would_log_rules_all;
$h->{$rulename} = 0; # mark meta done
}
+ delete $mr->{$rulename};
delete $mp->{$rulename};
# Reiterate all metas again, in case some meta depended on us
$retry = 1;
}

goto RULE if $retry--;
+
+ delete $pms->{meta_check_ready};
}

sub finish_meta_tests {
@@ -454,7 +461,6 @@ sub run_generic_tests {
# start_rules_plugin_code '.$ruletype.' '.$priority.'
my $scoresptr = $self->{conf}->{scores};
my $qrptr = $self->{conf}->{test_qrs}; my $test_qr;
- my $hitsptr = $self->{tests_already_hit};
');
if (defined $opts{pre_loop_body}) {
$opts{pre_loop_body}->($self, $pms, $conf, %nopts);
@@ -724,10 +730,10 @@ sub do_head_tests {
$expr = '$hval '.$op.' /$test_qr/'.$matchg.'op';
}

- # Make sure rule is marked ready for meta rules using $hitsptr
+ # Make sure rule is marked ready for meta rules
$self->add_evalstr($pms, '
if ($scoresptr->{q{'.$rulename.'}}) {
- $hitsptr->{q{'.$rulename.'}} ||= 0;
+ $self->rule_ready(q{'.$rulename.'}, 1);
'.($op_infix ? '$test_qr = $qrptr->{q{'.$rulename.'}};' : '').'
'.$self->capture_rules_code($conf, $rulename).'
'.$posline.'
@@ -822,10 +828,10 @@ sub do_body_tests {
';
}

- # Make sure rule is marked ready for meta rules using $hitsptr
+ # Make sure rule is marked ready for meta rules
$self->add_evalstr($pms, '
if ($scoresptr->{q{'.$rulename.'}}) {
- $hitsptr->{q{'.$rulename.'}} ||= 0;
+ $self->rule_ready(q{'.$rulename.'}, 1);
$test_qr = $qrptr->{q{'.$rulename.'}};
'.$sub.'
'.$self->ran_rule_plugin_code($rulename, "body").'
@@ -887,10 +893,10 @@ sub do_uri_tests {
';
}

- # Make sure rule is marked ready for meta rules using $hitsptr
+ # Make sure rule is marked ready for meta rules
$self->add_evalstr($pms, '
if ($scoresptr->{q{'.$rulename.'}}) {
- $hitsptr->{q{'.$rulename.'}} ||= 0;
+ $self->rule_ready(q{'.$rulename.'}, 1);
$test_qr = $qrptr->{q{'.$rulename.'}};
'.$sub.'
'.$self->ran_rule_plugin_code($rulename, "uri").'
@@ -951,10 +957,10 @@ sub do_rawbody_tests {
';
}

- # Make sure rule is marked ready for meta rules using $hitsptr
+ # Make sure rule is marked ready for meta rules
$self->add_evalstr($pms, '
if ($scoresptr->{q{'.$rulename.'}}) {
- $hitsptr->{q{'.$rulename.'}} ||= 0;
+ $self->rule_ready(q{'.$rulename.'}, 1);
$test_qr = $qrptr->{q{'.$rulename.'}};
'.$sub.'
'.$self->ran_rule_plugin_code($rulename, "rawbody").'
@@ -990,10 +996,10 @@ sub do_full_tests {
my ($max) = ($conf->{tflags}->{$rulename}||'') =~ /\bmaxhits=(\d+)\b/;
$max = untaint_var($max);
$max ||= 0;
- # Make sure rule is marked ready for meta rules using $hitsptr
+ # Make sure rule is marked ready for meta rules
$self->add_evalstr($pms, '
if ($scoresptr->{q{'.$rulename.'}}) {
- $hitsptr->{q{'.$rulename.'}} ||= 0;
+ $self->rule_ready(q{'.$rulename.'}, 1);
$test_qr = $qrptr->{q{'.$rulename.'}};
pos $$fullmsgref = 0;
'.$self->hash_line_for_rule($pms, $rulename).'
@@ -1192,8 +1198,8 @@ sub run_eval_tests {
';
}

- # If eval returns undef, it means rule is running async and $hitsptr
- # will be set later by rule_ready() or got_hit()
+ # If eval returns undef, it means rule is running async and
+ # will be marked ready later by rule_ready() or got_hit()
$evalstr .= '
if (defined $result) {
if ($result) {
@@ -1221,7 +1227,6 @@ sub run_eval_tests {

my \$testptr = \$self->{conf}->{$evalname}->{$priority};
my \$scoresptr = \$self->{conf}->{scores};
- my \$hitsptr = \$self->{tests_already_hit};
my \$prepend2desc = q#$prepend2desc#;
my \$rulename;
my \$result;

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/OneLineBodyRuleType.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/OneLineBodyRuleType.pm?rev=1901060&r1=1901059&r2=1901060&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/OneLineBodyRuleType.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/OneLineBodyRuleType.pm Thu May 19 09:47:40 2022
@@ -69,14 +69,13 @@ sub check_start {
sub check_cleanup {
my ($self, $params) = @_;
my $pms = $params->{permsgstatus};
- my $hitsptr = $pms->{tests_already_hit};
my $scoresptr = $pms->{conf}->{scores};

# Force all body rules ready for meta rules. Need to do it here in
# cleanup, because the body is scanned per line instead of per rule
if ($pms->{conf}->{skip_body_rules}) {
foreach (keys %{$pms->{conf}->{skip_body_rules}}) {
- $hitsptr->{$_} ||= 0 if $scoresptr->{$_};
+ $pms->rule_ready($_, 1) if $scoresptr->{$_};
}
}
}
@@ -145,9 +144,9 @@ sub do_one_line_body_tests {

}

- # Make sure rule is marked ready for meta rules using $hitsptr
+ # Make sure rule is marked ready for meta rules
$sub .= '
- $hitsptr->{q{'.$rulename.'}} ||= 0;
+ $self->rule_ready(q{'.$rulename.'}, 1);
';

return if ($opts{doing_user_rules} &&