Menu
▾
▴
Re: [AMaViS-user] bounce killer feature - a patch for 2.6.0-rc1
|
From: Patrick T. T. <pa...@pa...> - 2008-04-10 17:49:56
|
Dear Mark,
Thanks for your patch.
I am testing 2.6.0-rc1 code.
The current problem of mine is to setup a working penpal checking for
inbound emails.
I have been searching on internet and this mailling list, but turn out with
nothing useful to my setup.
The postgresql is working fine with @storage , I can check it logged all
incoming and outgoing emails.
The inbound emails from outside are redirected from postfix to port 10024,
from where the penpal must be checked but I cannot find anything on the
maillog.
For the internal users, the emails are forwarded to port 10044, from where
no spam is checked, but the internal users get penpal checked and scored !!
I put all penpal entries on the global location of /etc/amavisd-new.conf,
understanding that it should be used by default instance of port 10024.
The penpals should check inbound emails and ignore outbound emails.
I also raise the debug log level up to 5, but I found that no sql query for
outbound traffic.
It doesn't work as I expected.
Could you please provide me some hints?
Thanks
Patrick
----- Original Message -----
From: "Mark Martinec" <Mar...@ij...>
To: <ama...@li...>
Cc: "Patrick T. Tsang" <pa...@pa...>
Sent: Friday, April 11, 2008 1:17 AM
Subject: Re: [AMaViS-user] bounce killer feature - a patch for 2.6.0-rc1
> Patrick,
>
>> do you provide 2.5.3 , and 2.5.4 patch?
>> My clients are suffering from this problem.
>
> Try this one (against 2.5.4):
>
>
> --- amavisd.orig 2008-04-10 16:18:49.000000000 +0200
> +++ amavisd 2008-04-10 19:14:04.000000000 +0200
> @@ -227,5 +227,6 @@
> @av_scanners @av_scanners_backup $first_infected_stops_scan
> $sa_spam_report_header $sa_spam_level_char $sa_mail_body_size_limit
> - $penpals_bonus_score $penpals_halflife $reputation_factor
> + $penpals_bonus_score $penpals_halflife $bounce_killer_score
> + $reputation_factor
> $undecipherable_subject_tag $localpart_is_case_sensitive
> $recipient_delimiter $replace_existing_extension
> @@ -582,4 +583,6 @@
> # for testing and statistics gathering);
>
> + $bounce_killer_score = 100;
> +
> #
> # Receiving mail related
> @@ -6028,4 +6031,6 @@
> sub name_declared # string or a ref to a list of strings
> { my($self)=shift; !@_ ? $self->{nm_decl} : ($self->{nm_decl}=shift) };
> +sub report_type # a string, e.g. 'delivery-status', rfc3462
> + { my($self)=shift; !@_ ? $self->{rep_typ} :
> ($self->{rep_typ}=shift) };
> sub size
> { my($self)=shift; !@_ ? $self->{size} : ($self->{size}=shift) };
> @@ -6500,4 +6505,6 @@
> }
> $part->name_declared(@rn==1 ? $rn[0] : \@rn) if @rn;
> + $val = $head->mime_attr('content-type.report-type');
> + $part->report_type($val) if $val ne '';
> }
> mime_decode_pre_epi('epilogue', $entity->epilogue,
> @@ -6519,6 +6526,10 @@
> $parent_obj));
> $parser->ignore_errors(1); # also is the default
> -# $parser->extract_nested_messages(0);
> - $parser->extract_nested_messages("NEST"); # parse embedded
> message/rfc822
> + # if bounce killer is enabled, extract_nested_messages must be off,
> + # otherwise we lose headers of attached message/rfc822 messages
> + $parser->extract_nested_messages(0);
> +# $parser->extract_nested_messages("NEST"); # parse embedded
> message/rfc822
> + # "NEST" complains with "part did not end with expected boundary"
> when
> + # the outer message is message/partial and the inner message is
> chopped
> $parser->extract_uuencode(1); # to enable or not to enable
> ???
> $parser->max_parts($MAXFILES) if $MAXFILES > 0;
> @@ -8778,4 +8789,5 @@
> ($will_do_virus_scanning || $will_do_banned_checking);
>
> + my($bounce_header_fields_ref,$bounce_msgid);
> if ($will_do_parts_decoding) { # decoding parts can take a lot of
> time
> $which_section = "mime_decode-1";
> @@ -8786,4 +8798,15 @@
> prolong_timer($which_section);
>
> + if (c('bounce_killer_score') > 0) {
> + $which_section = "dsn_parse";
> + # analyze a bounce after MIME decoding but before further archive
> + # decoding, which often replaces decoded files
> + $bounce_header_fields_ref = inspect_a_bounce_message($msginfo);
> + $bounce_msgid = $bounce_header_fields_ref->{'message-id'}
> + if $bounce_header_fields_ref &&
> + exists $bounce_header_fields_ref->{'message-id'};
> + prolong_timer($which_section);
> + }
> +
> $which_section = "parts_decode_ext";
> snmp_count('OpsDec');
> @@ -9014,4 +9037,5 @@
>
> $which_section = "penpals_check";
> + my($pp_age);
> if (defined($sql_storage) && !@virusname) {
> my($pp_bonus) = c('penpals_bonus_score'); # score points
> @@ -9022,8 +9046,8 @@
> (defined $penpals_threshold_low || defined
> $penpals_threshold_high);
> if ($pp_bonus <= 0 || $pp_halflife <= 0) {}
> - elsif (defined($penpals_threshold_low) &&
> + elsif (defined($penpals_threshold_low) && !defined($bounce_msgid)
> &&
> $spam_level + max(@boost_list) < $penpals_threshold_low) {}
> # low score for all recipients, no need for aid
> - elsif (defined($penpals_threshold_high) &&
> + elsif (defined($penpals_threshold_high) && !defined($bounce_msgid)
> &&
> $spam_level + min(@boost_list) - $pp_bonus
> > $penpals_threshold_high) {}
> @@ -9043,9 +9067,11 @@
> $msginfo->orig_header_fields->{'references'};
> my(@refs) = $refs_str eq '' ? () :
> parse_message_id($refs_str);
> + push(@refs,$bounce_msgid) if defined $bounce_msgid;
> do_log(4,"penpals: references: %s", join(", ",@refs)) if
> @refs;
> # NOTE: swap $rid and $sid as args here, as we are now
> checking
> # for a potential reply mail - whether the current recipient
> has
> # recently sent any mail to the sender of the current mail:
> - my($pp_age,$pp_mail_id,$pp_subj) =
> + my($pp_mail_id,$pp_subj);
> + ($pp_age,$pp_mail_id,$pp_subj) =
>
> $sql_storage->penpals_find($rid,$sid,\@refs,$msginfo->rx_time);
> if (defined($pp_age)) { # found info about previous
> correspondence
> @@ -9077,4 +9103,43 @@
> }
>
> + $which_section = "bounce_killer";
> + if ($bounce_header_fields_ref) { # message looks like a DSN
> + my($bounce_rescued);
> + if (defined $pp_age) { # found by pen pals by a Message-ID in
> attachment
> + # is a bounce, refers to our previous outgoing message, treat it
> kindly
> + snmp_count('InMsgsBounceRescuedByPenPals');
> + $bounce_rescued = 'by penpals';
> + } elsif (defined($bounce_msgid) && $bounce_msgid =~
> /(\@[^\@>]+)>?\z/ &&
> + lookup(0,$1, @{ca('local_domains_maps')})) {
> + # not in pen pals, but domain in Message-ID is a local domain
> + snmp_count('InMsgsBounceRescuedByDomain');
> + $bounce_rescued = 'by domain';
> + }
> + ll(2) && do_log(2, "bounce %s, %s -> %s, %s",
> + defined $bounce_rescued ?'rescued '.$bounce_rescued
> :'killed',
> + qquote_rfc2821_local($sender),
> + join(',', qquote_rfc2821_local(@recips)),
> + join(', ', map { $_ . ': ' .
> $bounce_header_fields_ref->{$_} }
> + sort(keys %$bounce_header_fields_ref)) );
> + if (!$bounce_rescued) {
> + snmp_count('InMsgsBounceNotFromUsKilled');
> + my($bounce_killer_score) = c('bounce_killer_score');
> + for my $r (@{$msginfo->per_recip_data}) {
> + my($boost) = $r->recip_score_boost || 0;
> + $r->recip_score_boost($boost + $bounce_killer_score);
> + }
> + }
> + } elsif ($sender eq '' ||
> + $sender =~
> /^(?:postmaster|mailer-daemon)(?:\@|\z)/si ||
> + $rfc2822_from[0] =~
> /^(?:postmaster|mailer-daemon|double-bounce|
> + mailer|uucp) (?:\@|\z)/xsi) {
> + # message could be some kind of a non-standard bounce, but lacks
> + # recognizable structure and/or a header section from original mail
> + ll(3) && do_log(3, "bounce unverifiable, %s -> %s, ",
> + qquote_rfc2821_local($sender),
> + join(',', qquote_rfc2821_local(@recips)));
> + snmp_count('InMsgsBounceUnverifiable');
> + }
> +
> $which_section = "decide_mail_destiny";
> $snmp_db->register_proc(2,0,'r',$am_id) if defined $snmp_db; #
> results...
> @@ -9743,4 +9808,109 @@
> }
>
> +# Check if a message is a bounce, and if it is, try to obtain essential
> +# information from an attached header section of an attached original
> message,
> +# mainly the Message-ID.
> +#
> +sub inspect_a_bounce_message($) {
> + my($msginfo) = @_;
> + my(%header_field); my($is_true_bounce) = 0;
> + my($parts_root) = $msginfo->parts_root;
> + if (defined $parts_root) {
> + my($sender) = $msginfo->sender;
> + my($structure_type) = '?';
> + my($top) = $parts_root->children;
> + $top = $top->[0] if defined $top; # there should only be one top
> part
> + my(@parts); my($fname_ind,$fname); my($plaintext) = 0;
> + if (defined $top)
> + { my($ch) = $top->children; @parts = ($top, !defined $ch ? () :
> @$ch) }
> + my(@t) =
> + map { my($t)=$_->type_declared; lc(ref $t ? $t->[0] : $t) } @parts;
> + ll(5) && do_log(5, "inspect_dsn: parts: %s", join(", ",@t));
> + if ( @parts >= 2 && @parts <= 4 && # a root, with 2 or 3 leaves
> + $t[0] eq 'multipart/report' &&
> + ( $t[2] eq 'message/delivery-status' ||
> + $t[2] eq 'message/feedback-report' ) &&
> + $t[2] eq 'message/'.lc($parts[0]->report_type) &&
> + ( $t[3] eq 'text/rfc822-headers' || $t[3] eq 'message/rfc822' ))
> {
> + # standard DSN or feedback-report
> + $fname_ind = 3; $is_true_bounce = 1; $structure_type = 'standard
> DSN';
> + } elsif (@parts >= 3 && @parts <= 4 && # a root, with 2 or 3 leaves
> + $t[0] eq 'multipart/report' &&
> + lc($parts[0]->report_type) eq 'delivery-status' &&
> + ( $t[-1] eq 'text/rfc822-headers' || $t[-1] eq
> 'message/rfc822' )) {
> + # not quite std. DSN (missing message/delivery-status), but
> recognizable
> + $fname_ind = -1; $is_true_bounce = 1;
> + $structure_type = 'DSN, missing delivery-status part';
> + } elsif (@parts >= 3 && @parts <= 5 && $sender eq '' &&
> + $t[0] eq 'multipart/mixed' &&
> + ( $t[-1] eq 'text/rfc822-headers' || $t[-1] eq
> 'message/rfc822' )) {
> + # sometimes qmail
> + $fname_ind = -1; $structure_type = 'multipart/mixed with attached
> msg';
> + } elsif (@parts == 1 && $sender eq '') {
> + # nonstructured, possibly a non-standard bounce (qmail, gmail.com,
> ...)
> + $fname_ind = 0; $plaintext = 1; $structure_type = 'nonstructured';
> + }
> + if (defined $fname_ind) { # we have a header section from original
> mail
> + $fname_ind = $#parts if $fname_ind == -1;
> + $fname = $parts[$fname_ind]->full_name;
> + ll(5) && do_log(5, "inspect_dsn: %s, base_name(%s): %s, fname: %s",
> + $structure_type, $fname_ind, $parts[$fname_ind]->base_name,
> $fname);
> + defined $fname or die "inspect_dsn: no filename ($fname_ind), ".
> + "make sure the extract_nested_messages is
> off!";
> + my(%collectable_header_fields);
> + $collectable_header_fields{lc($_)} = 1
> + for qw(From Message-ID Return-Path);
> + my($fh) = IO::File->new;
> + $fh->open($fname,'<') or die "Can't open file $fname: $!";
> + binmode($fh,":bytes") or die "Can't cancel :utf8 mode: $!"
> + if $unicode_aware;
> + my($curr_head,$ln); my($nr) = 0; my($have_msgid) = 0; local($1,$2);
> + for ($! = 0; defined($ln = $fh->getline); $! = 0) {
> + $nr++; last if $nr > 1000; # safety measure
> + if ($ln =~ /^[ \t]/) { # folded
> + $curr_head .= $ln if length($curr_head) < 2000; # safety
> measure
> + } else { # a new header field, process previous if any
> + $header_field{lc($1)} = $2 if defined($curr_head) &&
> + $curr_head =~ /^([!-9;-\176]{1,30})[
> \t]*:(.*)\z/s &&
> + $collectable_header_fields{lc($1)};
> + $curr_head = $ln;
> + $have_msgid = 1 if defined $header_field{'from'} &&
> + defined $header_field{'message-id'};
> + last if ($ln eq "\n" || $ln =~ /^--/) && !$plaintext;
> + last if $have_msgid;
> + }
> + }
> + defined $ln || $!==0 or die "Error reading from $fname: $!";
> + $fh->close or die "Error closing $fname: $!";
> + $header_field{lc($1)} = $2 if defined($curr_head) &&
> + $curr_head =~ /^([!-9;-\176]{1,30})[
> \t]*:(.*)\z/s &&
> + $collectable_header_fields{lc($1)};
> + $have_msgid = 1 if defined $header_field{'from'} &&
> + defined $header_field{'message-id'};
> + $is_true_bounce = 1 if $have_msgid;
> + if ($is_true_bounce) {
> + for (@header_field{keys %header_field})
> + { s/\n(?=[ \t])//g; s/^[ \t]+//; s/[ \t\n]+\z// }
> + $header_field{'message-id'} =
> + (parse_message_id($header_field{'message-id'}))[0]
> + if defined $header_field{'message-id'};
> + }
> + section_time("inspect_dsn");
> + }
> + if ($is_true_bounce) {
> + do_log(3, "inspect_dsn: bounce, struct(%s): %s, <%s>, %s",
> + !defined($fname_ind) ? '-' : $fname_ind,
> + $structure_type, $sender,
> + join(", ", map { $_ . ": " . $header_field{$_} }
> + sort(keys %header_field)) ) if ll(3);
> + } elsif ($sender eq '') {
> + do_log(3, "inspect_dsn: not a bounce, struct(%s): %s, parts: %s",
> + !defined($fname_ind) ? '-' : $fname_ind,
> + $structure_type, join(", ",@t)) if ll(3);
> + }
> + }
> + !$is_true_bounce ? 0 : \%header_field;
> +}
> +
> sub add_forwarding_header_edits_common($$$$$$$) {
> my($conn, $msginfo, $hdr_edits, $hold, $any_undecipherable,
>
>
>
>
> Mark
>
|
