#!/usr/pkg/bin/perl
# -*- indent-tabs-mode: nil; -*-
# vim:ft=perl:et:sw=4
# $Id: sympa_msg.pl.in 12363 2015-09-21 10:26:26Z sikeda $

# Sympa - SYsteme de Multi-Postage Automatique
#
# Copyright (c) 1997, 1998, 1999 Institut Pasteur & Christophe Wolfhugel
# Copyright (c) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
# 2006, 2007, 2008, 2009, 2010, 2011 Comite Reseau des Universites
# Copyright (c) 2011, 2012, 2013, 2014, 2015 GIP RENATER
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

use lib split(/:/, $ENV{SYMPALIB} || ''), '/usr/pkg/sympa/bin';
use strict;
use warnings;
use English qw(-no_match_vars);
use File::Copy qw();
use Getopt::Long;
use Pod::Usage;
use POSIX qw();

use Sympa;
use Sympa::Alarm;
use Sympa::Commands;
use Conf;
use Sympa::Constants;
use Sympa::Crash;    # Show traceback.
use Sympa::DatabaseManager;
use Sympa::Language;
use Sympa::List;
use Sympa::Log;
use Sympa::Mailer;
use Sympa::Regexps;
use Sympa::Report;
use Sympa::Scenario;
use Sympa::Spool::Digest::Collection;
use Sympa::Spool::Incoming;
use Sympa::Tools::Daemon;
use Sympa::Tools::Data;

## Internal tuning
# delay between each read of the digestqueue
my $digestsleep = 5;

## Init random engine
srand(time());

# Default is to launch process as master daemon.
my $daemon_usage = 'DAEMON_MASTER';

# Check options.
my %options;
unless (
    GetOptions(
        \%main::options, 'debug|d',  'log_level=s', 'foreground',
        'config|f=s',    'lang|l=s', 'mail|m',      'keepcopy|k=s',
        'help|h',        'version|v',
    )
    ) {
    pod2usage(-exitval => 1, -output => \*STDERR);
}
if ($main::options{'help'}) {
    pod2usage(0);
} elsif ($main::options{'version'}) {
    printf "Sympa %s\n", Sympa::Constants::VERSION;
    exit 0;
}

if ($main::options{'debug'}) {
    $main::options{'log_level'} = 2 unless $main::options{'log_level'};
    $main::options{'foreground'} = 1;
}

my $log = Sympa::Log->instance;
$log->{log_to_stderr} = 'all' if $main::options{'foreground'};

my $default_lang;

my $language = Sympa::Language->instance;
my $mailer   = Sympa::Mailer->instance;

_load();

## Put ourselves in background if we're not in debug mode. That method
## works on many systems, although, it seems that Unix conceptors have
## decided that there won't be a single and easy way to detach a
## process from its controlling TTY.
unless ($main::options{'foreground'}) {
    if (open(TTY, "/dev/tty")) {
        ioctl(TTY, 0x20007471, 0);    # XXX s/b TIOCNOTTY()
        close(TTY);
    }
    open(STDIN,  ">> /dev/null");
    open(STDOUT, ">> /dev/null");
    open(STDERR, ">> /dev/null");

    setpgrp 0, 0;

    my $child_pid = fork;
    if ($child_pid) {
        $log->syslog('notice', 'Starting sympa/msg daemon, PID %d',
            $child_pid);
        exit 0;
    } elsif (not defined $child_pid) {
        die sprintf 'Cannot fork: %s', $ERRNO;
    }

    # Fork a new process dedicated to automatic list creation, if required.
    if ($Conf::Conf{'automatic_list_feature'} eq 'on') {
        my $child_pid = fork;
        if ($child_pid) {
            waitpid $child_pid, 0;
            $CHILD_ERROR and die;
        } elsif (not defined $child_pid) {
            die sprintf 'Cannot fork: %s', $ERRNO;
        } else {
            # We're in the specialized child process:
            # automatic lists creation.
            exec q{/usr/pkg/sympa/bin/sympa_automatic.pl}, map {
                defined $main::options{$_}
                    ? ("--$_", $main::options{$_})
                    : ()
            } qw(config log_level mail);
            die sprintf 'Cannot exec: %s', $ERRNO;
        }
    }
}

my $pidfile = 'sympa_msg';

$log->openlog($Conf::Conf{'syslog'}, $Conf::Conf{'log_socket_type'});

# If process is running in foreground, don't write STDERR to a
# dedicated file
my $options;
$options->{'stderr_to_tty'} = 1 if $main::options{'foreground'};
$options->{'multiple_process'} = 0;

# Save the PID number
Sympa::Tools::Daemon::write_pid($pidfile, $PID, $options);
unless ($options->{'stderr_to_tty'}) {
    Sympa::Tools::Daemon::direct_stderr_to_file(pid => $PID);
}

# Start multiple processes if required.
unless ($main::options{'foreground'}) {
    if ($daemon_usage eq 'DAEMON_MASTER'
        and ($Conf::Conf{'incoming_max_count'} || 0) > 1) {
        $options->{'multiple_process'} = 1;

        # Disconnect from database before fork to prevent DB handles
        # to be shared by different processes.  Sharing database
        # handles may crash sympa_msg.pl.
        Sympa::DatabaseManager->disconnect;

        for my $process_count (2 .. $Conf::Conf{'incoming_max_count'}) {
            my $child_pid = fork;
            if ($child_pid) {
                $log->syslog('info', 'Starting child daemon, PID %s',
                    $child_pid);
                # Saves the PID number
                Sympa::Tools::Daemon::write_pid($pidfile, $child_pid,
                    $options);
                #$created_children{$child_pid} = 1;
                sleep 1;
            } elsif (not defined $child_pid) {
                $log->syslog('err', 'Cannot fork: %m');
                last;
            } else {
                # We're in a child process
                close STDERR;
                Sympa::Tools::Daemon::direct_stderr_to_file(pid => $PID);
                $daemon_usage = 'DAEMON_SLAVE';    # child
                $log->openlog($Conf::Conf{'syslog'},
                    $Conf::Conf{'log_socket_type'});
                $log->syslog('info', 'Slave daemon started with PID %s',
                    $PID);
                last;
            }
        }

        # Restore persistent connection.
        Sympa::DatabaseManager->instance
            or die 'Reconnecting database failed';
    }
}

# Set the User ID & Group ID for the process
$GID = $EGID = (getgrnam(Sympa::Constants::GROUP))[2];
$UID = $EUID = (getpwnam(Sympa::Constants::USER))[2];

## Required on FreeBSD to change ALL IDs
## (effective UID + real UID + saved UID)
POSIX::setuid((getpwnam(Sympa::Constants::USER))[2]);
POSIX::setgid((getgrnam(Sympa::Constants::GROUP))[2]);

## Check if the UID has correctly been set (useful on OS X)
unless (($GID == (getgrnam(Sympa::Constants::GROUP))[2])
    && ($UID == (getpwnam(Sympa::Constants::USER))[2])) {
    die
        "Failed to change process user ID and group ID. Note that on some OS Perl scripts can't change their real UID. In such circumstances Sympa should be run via sudo.\n";
}

# Sets the UMASK
umask(oct($Conf::Conf{'umask'}));

## Most initializations have now been done.
$log->syslog('notice', 'Sympa/msg %s Started', Sympa::Constants::VERSION());

# Check for several files.
# Prevent that 2 processes perform checks at the same time...
#FIXME: This would be done in --health_check mode.
unless (Conf::checkfiles()) {
    die "Missing files.\n";
    ## No return.
}

## Do we have right access in the directory
if ($main::options{'keepcopy'}) {
    if (!-d $main::options{'keepcopy'}) {
        $log->syslog(
            'notice',
            'Cannot keep a copy of incoming messages: %s is not a directory',
            $main::options{'keepcopy'}
        );
        delete $main::options{'keepcopy'};
    } elsif (!-w $main::options{'keepcopy'}) {
        $log->syslog(
            'notice',
            'Cannot keep a copy of incoming messages: no write access to %s',
            $main::options{'keepcopy'}
        );
        delete $main::options{'keepcopy'};
    }
}

# Catch signals, in order to exit cleanly, whenever possible.
my $signal = 0;
$SIG{'TERM'} = 'sigterm';
$SIG{'INT'}  = 'sigterm';    # Interrupt from terminal.
$SIG{'HUP'}  = 'sighup';
$SIG{'PIPE'} = 'IGNORE';     # Ignore SIGPIPE ; prevents process from dying

# Main loop.
# This loop is run foreach HUP signal received.

my %loop_info;
my %msgid_table;
my $latest_msgid_table_cleanup = time;
my $index_queuedigest          = 0;      # verify the digest queue
my $index_cleanqueue           = 0;

# This is the main loop : look for files in the directory, handles
# them, sleeps a while and continues the good job.

my $spool = Sympa::Spool::Incoming->new;

while ($signal ne 'term') {
    Sympa::List::init_list_cache();
    # Process grouped notifications
    Sympa::Alarm->instance->flush;

    # process digest only in distribution mode.
    # Scan queuedigest.
    if (    $daemon_usage eq 'DAEMON_MASTER'
        and $index_queuedigest++ >= $digestsleep) {
        $index_queuedigest = 0;
        SendDigest();
    }

    # Cleanup in-memory msgid table, only in a while.
    if (time > $latest_msgid_table_cleanup +
        $Conf::Conf{'msgid_table_cleanup_frequency'}) {
        clean_msgid_table();
        $latest_msgid_table_cleanup = time;
    }

    my ($message, $handle) = $spool->next;

    if ($message and $handle) {
        my $status = process_message($message, $handle);
        unless (defined $status) {
            $spool->quarantine($handle);
        } elsif ($status) {
            if ($main::options{'keepcopy'}) {
                unless (
                    File::Copy::copy(
                        $spool->{directory} . '/' . $handle->basename,
                        $main::options{'keepcopy'} . '/' . $handle->basename
                    )
                    ) {
                    $log->syslog(
                        'notice',
                        'Could not rename %s/%s to %s/%s: %m',
                        $spool->{directory},
                        $handle->basename,
                        $main::options{'keepcopy'},
                        $handle->basename
                    );
                }
            }

            $spool->remove($handle);
        } else {
            # Processing aborted.  Redo later.
            $handle->close;
            next;
        }
    } elsif ($handle) {
        $log->syslog('err', 'Cannot parse message <%s>', $handle->basename);
        $spool->quarantine($handle);
        next;
    }

    # Free zombie sendmail process.
    Sympa::Mailer->instance->reaper;

    if ($signal eq 'hup') {
        # Disconnect from Database
        Sympa::DatabaseManager->disconnect;

        $log->syslog('notice', 'Sympa %s reload config',
            Sympa::Constants::VERSION);
        _load();
        $spool  = Sympa::Spool::Incoming->new;
        $signal = 0;
    }

    unless ($message or $handle) {
        # Sleep for a while if spool is empty.
        sleep $Conf::Conf{'sleep'};
    }
}    # while ($signal ne 'term')

# Purge grouped notifications
Sympa::Alarm->instance->flush(purge => 1);

$log->syslog('notice', 'Sympa/msg exited normally due to signal');
Sympa::Tools::Daemon::remove_pid($pidfile, $PID, $options);

exit(0);

# Load configuration.
sub _load {
    ## Load sympa.conf.
    unless (Conf::load(Conf::get_sympa_conf(), 'no_db')) {    #Site and Robot
        die sprintf
            "Unable to load sympa configuration, file %s or one of the vhost robot.conf files contain errors. Exiting.\n",
            Conf::get_sympa_conf();
    }

    ## Open the syslog and say we're read out stuff.
    $log->openlog($Conf::Conf{'syslog'}, $Conf::Conf{'log_socket_type'});

    # Enable SMTP logging if required
    $mailer->{log_smtp} = $main::options{'mail'}
        || Sympa::Tools::Data::smart_eq($Conf::Conf{'log_smtp'}, 'on');

    # setting log_level using conf unless it is set by calling option
    if (defined $main::options{'log_level'}) {
        $log->{level} = $main::options{'log_level'};
        $log->syslog(
            'info',
            'Configuration file read, log level set using options: %s',
            $main::options{'log_level'}
        );
    } else {
        $log->{level} = $Conf::Conf{'log_level'};
        $log->syslog(
            'info',
            'Configuration file read, default log level %s',
            $Conf::Conf{'log_level'}
        );
    }

    if (Conf::cookie_changed()) {
        die sprintf
            'sympa.conf/cookie parameter has changed. You may have severe inconsitencies into password storage. Restore previous cookie or write some tool to re-encrypt password in database and check spools contents (look at %s/cookies.history file).',
            $Conf::Conf{'etc'};
    }

    # Check database connectivity.
    unless (Sympa::DatabaseManager->instance) {
        die sprintf
            "Database %s defined in sympa.conf is unreachable. verify db_xxx parameters in sympa.conf\n",
            $Conf::Conf{'db_name'};
    }

    # Now trying to load full config (including database)
    unless (Conf::load()) {    #FIXME: load Site, then robot cache
        die sprintf
            "Unable to load Sympa configuration, file %s or any of the virtual host robot.conf files contain errors. Exiting.\n",
            Conf::get_sympa_conf();
    }

    ## Set locale configuration
    ## Compatibility with version < 2.3.3
    $main::options{'lang'} =~ s/\.cat$//
        if defined $main::options{'lang'};
    $default_lang =
        $language->set_lang($main::options{'lang'}, $Conf::Conf{'lang'},
        'en');

    ## Main program
    if (!chdir($Conf::Conf{'home'})) {
        die sprintf 'Can\'t chdir to %s: %s', $Conf::Conf{'home'}, $ERRNO;
        ## Function never returns.
    }

    ## Check for several files.
    unless (Conf::checkfiles_as_root()) {
        die "Missing files\n";
    }
}

############################################################
# sigterm
############################################################
#  When we catch signal, just changes the value of the $signal
#  loop variable.
#
# IN : -
#
# OUT : -
#
############################################################
sub sigterm {
    my ($sig) = @_;
    $log->syslog('notice',
        'Signal %s received, still processing current task', $sig);
    $signal = 'term';
}

############################################################
# sighup
############################################################
#  When we catch SIGHUP, changes the value of the $signal
#  loop variable and puts the "-mail" logging option
#
# IN : -
#
# OUT : -
#
###########################################################
sub sighup {
    if ($mailer->{log_smtp}) {
        $log->syslog('notice',
            'signal HUP received, switch of the "-mail" logging option and continue current task'
        );
        $mailer->{log_smtp} = undef;
    } else {
        $log->syslog('notice',
            'signal HUP received, switch on the "-mail" logging option and continue current task'
        );
        $mailer->{log_smtp} = 1;
    }
    $signal = 'hup';
}

############################################################
#  process_message
############################################################
#  Handles a file received and files in the queue directory.
#  This will read the file, separate the header and the body
#  of the message and call the adequate function wether we
#  have received a command or a message to be redistributed
#  to a list.
#
# IN : -$file (+): the file to handle
#
# OUT : $status
#     | undef
#
##############################################################
sub process_message {
    $log->syslog('debug', '(%s)', @_);
    my $message = shift;

    my $status;

    unless (defined $message) {
        $log->syslog('err', 'Unable to create Sympa::Message object');
        $log->db_log(
            #'robot'        => $robot,
            #'list'         => $listname,
            'action'       => 'process_message',
            'parameters'   => '',
            'target_email' => "",
            'msg_id'       => '',
            'status'       => 'error',
            'error_type'   => 'unable_create_message',
            'user_email'   => ''
        );
        return undef;
    }

    unless (defined $message->{'message_id'}
        and length $message->{'message_id'}) {
        $log->syslog('err', 'Message %s has no message ID', $message);
        $log->db_log(
            #'robot'        => $robot,
            #'list'         => $listname,
            'action'       => 'process_message',
            'parameters'   => $message->get_id,
            'target_email' => "",
            'msg_id'       => "",
            'status'       => 'error',
            'error_type'   => 'no_message_id',
            'user_email'   => $message->{'sender'}
        );
        return undef;
    }

    my $msg_id = $message->{message_id};

    $language->set_lang($default_lang);

    # Compatibility: Message with checksum by Sympa <=6.2a.40
    # They should be migrated.
    if ($message and $message->{checksum}) {
        $log->syslog('err',
            '%s: Message with old format.  Run upgrade_send_spool.pl',
            $message);
        return 0;    # Skip
    }

    $log->syslog(
        'notice',
        'Processing %s; envelope_sender=%s; message_id=%s; sender=%s',
        $message,
        $message->{envelope_sender},
        $message->{message_id},
        $message->{sender}
    );

    my $robot;
    my $listname;

    if (ref $message->{context} eq 'Sympa::List') {
        $robot = $message->{context}->{'domain'};
    } elsif ($message->{context} and $message->{context} ne '*') {
        $robot = $message->{context};
    } else {
        # Older "sympa" alias may not have "@domain" in argument of queue
        # program.
        $robot = $Conf::Conf{'domain'};
    }
    $listname = $message->{'listname'};

    ## Ignoring messages with no sender
    my $sender = $message->{'sender'};
    unless ($message->{'md5_check'} or $sender) {
        $log->syslog('err', 'No sender found in message %s', $message);
        $log->db_log(
            'robot'        => $robot,
            'list'         => $listname,
            'action'       => 'process_message',
            'parameters'   => "",
            'target_email' => "",
            'msg_id'       => $msg_id,
            'status'       => 'error',
            'error_type'   => 'no_sender',
            'user_email'   => $sender
        );
        return undef;
    }

    ## Unknown robot
    unless ($message->{'md5_check'} or Conf::valid_robot($robot)) {
        $log->syslog('err', 'Robot %s does not exist', $robot);
        Sympa::Report::reject_report_msg('user', 'list_unknown', $sender,
            {'listname' => $listname, 'message' => $message},
            '*', $message->as_string, '');
        $log->db_log(
            'robot'        => $robot,
            'list'         => $listname,
            'action'       => 'process_message',
            'parameters'   => "",
            'target_email' => "",
            'msg_id'       => $msg_id,
            'status'       => 'error',
            'error_type'   => 'unknown_robot',
            'user_email'   => $sender
        );
        return undef;
    }

    $language->set_lang(Conf::get_robot_conf($robot, 'lang'), $default_lang);

    # Load spam status.
    $message->check_spam_status;
    # Check DKIM signatures.
    $message->check_dkim_signature;
    # Check S/MIME signature.
    $message->check_smime_signature;
    # Decrypt message.  On success, check nested S/MIME signature.
    if ($message->smime_decrypt and not $message->{'smime_signed'}) {
        $message->check_smime_signature;
    }

    # *** Now message content may be altered. ***

    # Enable SMTP logging if required.
    $mailer->{log_smtp} = $main::options{'mail'}
        || Conf::get_robot_conf($robot, 'log_smtp');
    # setting log_level using conf unless it is set by calling option
    unless (defined $main::options{'log_level'}) {
        $log->{level} = Conf::get_robot_conf($robot, 'log_level');
        $log->syslog('debug',
            'Setting log level with %s configuration (or sympa.conf): %d',
            $robot, Conf::get_robot_conf($robot, 'log_level'));
    }

    ## Strip of the initial X-Sympa-To and X-Sympa-Checksum internal headers
    delete $message->{'rcpt'};
    delete $message->{'checksum'};

    ## Initialize command report
    Sympa::Report::init_report_cmd();

    my $list =
        (ref $message->{context} eq 'Sympa::List')
        ? $message->{context}
        : undef;
    my $type = $message->{'listtype'} || '';
    my $list_address;

    if ($type eq 'listmaster' or $type eq 'sympa') {
        $list_address =
            $message->{'localpart'} . '@'
            . Conf::get_robot_conf($robot, 'host');
    } else {
        unless (ref $list eq 'Sympa::List') {
            $log->syslog('err', 'List %s does not exist', $listname);
            Sympa::Report::reject_report_msg(
                'user',
                'list_unknown',
                $sender,
                {   'listname' => $listname,
                    'message'  => $message
                },
                $robot,
                $message->as_string,
                ''
            );
            $log->db_log(
                'robot'        => $robot,
                'list'         => $listname,
                'action'       => 'process_message',
                'parameters'   => "",
                'target_email' => "",
                'msg_id'       => $msg_id,
                'status'       => 'error',
                'error_type'   => 'unknown_list',
                'user_email'   => $sender
            );
            return undef;
        }
        $list_address = $list->get_list_address();
    }

    ## Loop prevention
    if (ref $list eq 'Sympa::List'
        and Sympa::Tools::Data::smart_eq(
            $list->{'admin'}{'reject_mail_from_automates_feature'}, 'on'
        )
        ) {
        my $conf_loop_prevention_regex;
        $conf_loop_prevention_regex =
            $list->{'admin'}{'loop_prevention_regex'};
        $conf_loop_prevention_regex ||=
            Conf::get_robot_conf($robot, 'loop_prevention_regex');
        if ($sender =~ /^($conf_loop_prevention_regex)(\@|$)/mi) {
            $log->syslog(
                'err',
                'Ignoring message which would cause a loop, sent by %s; matches loop_prevention_regex',
                $sender
            );
            return undef;
        }

        ## Ignore messages that would cause a loop
        ## Content-Identifier: Auto-replied is generated by some non standard
        ## X400 mailer
        if (grep {/Auto-replied/i} $message->get_header('Content-Identifier')
            or grep {/Auto Reply to/i}
            $message->get_header('X400-Content-Identifier')
            or grep { !/^no$/i } $message->get_header('Auto-Submitted')) {
            $log->syslog('err',
                "Ignoring message which would cause a loop; message appears to be an auto-reply"
            );
            return undef;
        }
    }

    ## Q- and B-decode subject
    my $subject_field = $message->{'decoded_subject'};

    ## Loop prevention
    my $loop;
    foreach $loop ($message->get_header('X-Loop')) {
        chomp $loop;
        $log->syslog('debug2', 'X-Loop: %s', $loop);
        #foreach my $l (split(/[\s,]+/, lc($loop))) {
        if ($loop eq lc($list_address)) {
            $log->syslog('err',
                'Ignoring message which would cause a loop (X-Loop: %s)',
                $loop);
            return undef;
        }
        #}
    }

    # Anti-virus
    my $rc = $message->check_virus_infection;
    if ($rc) {
        my $antivirus_notify =
            Conf::get_robot_conf($robot, 'antivirus_notify') || 'none';
        if ($antivirus_notify eq 'sender') {
            Sympa::send_file(
                $robot,
                'your_infected_msg',
                $sender,
                {   'virus_name'     => $rc,
                    'recipient'      => $list_address,
                    'sender'         => $message->{sender},
                    'lang'           => Conf::get_robot_conf($robot, 'lang'),
                    'auto_submitted' => 'auto-replied'
                }
            );
        } elsif ($antivirus_notify eq 'delivery_status') {
            Sympa::send_dsn(
                $message->{context},
                $message,
                {   'virus_name' => $rc,
                    'recipient'  => $list_address,
                    'sender'     => $message->{sender}
                },
                '5.7.0'
            );
        }
        $log->syslog('notice',
            "Message for %s from %s ignored, virus %s found",
            $list_address, $sender, $rc);
        $log->db_log(
            'robot'        => $robot,
            'list'         => $listname,
            'action'       => 'process_message',
            'parameters'   => "",
            'target_email' => "",
            'msg_id'       => $msg_id,
            'status'       => 'error',
            'error_type'   => 'virus',
            'user_email'   => $sender
        );
        return undef;
    } elsif (!defined($rc)) {
        Sympa::send_notify_to_listmaster(
            $robot,
            'antivirus_failed',
            [   sprintf
                    "Could not scan message %s; The message has been saved as BAD.",
                $message->get_id
            ]
        );

        return undef;
    }

    if ($type eq 'listmaster') {
        $status = DoForward('sympa', 'listmaster', $robot, $message);
    } elsif ($type eq 'sympa') {
        # Mail addressed to the robot is command.
        $status = DoCommand('sympa', $robot, $message);
    } elsif ($type eq 'subscribe' or $type eq 'unsubscribe') {
        # Mail to <list>-subscribe or <list>-unsubscribe is command.
        $status = DoCommand("$listname-$type", $robot, $message);
    } elsif ($type eq 'owner') {    # -request
        # Forward mails to <list>-request.
        # Simulate Smartlist behaviour with command in subject
        if (    $subject_field
            and $subject_field =~
            /^\s*(subscribe|unsubscribe)(\s*$listname)?\s*$/i) {
            my $command = $1;
            $status = DoCommand("$listname-$command", $robot, $message);
        } else {
            $status = DoForward($listname, 'owner', $robot, $message);
        }
    } elsif ($type eq 'return_path' or $type eq 'editor') {
        # Forward mails to <list>-owner etc.
        $status = DoForward($listname, $type, $robot, $message);
    } else {
        $status = DoMessage($message);
    }

    ## Mail back the result.
    if (Sympa::Report::is_there_any_report_cmd()) {
        ## Loop prevention

        ## Count reports sent to $sender
        $loop_info{$sender}{'count'}++;

        ## Sampling delay
        if ((time - ($loop_info{$sender}{'date_init'} || 0)) <
            $Conf::Conf{'loop_command_sampling_delay'}) {

            ## Notify listmaster of first rejection
            if ($loop_info{$sender}{'count'} ==
                $Conf::Conf{'loop_command_max'}) {
                ## Notify listmaster
                Sympa::send_notify_to_listmaster($robot, 'loop_command',
                    {'msg' => $message});
            }

            ## Too many reports sent => message skipped !!
            if ($loop_info{$sender}{'count'} >=
                $Conf::Conf{'loop_command_max'}) {
                $log->syslog(
                    'err',
                    'Ignoring message which would cause a loop, %d messages sent to %s; loop_command_max exceeded',
                    $loop_info{$sender}{'count'},
                    $sender
                );

                return undef;
            }
        } else {
            ## Sampling delay is over, reinit
            $loop_info{$sender}{'date_init'} = time;

            ## We apply Decrease factor if a loop occurred
            $loop_info{$sender}{'count'} *=
                $Conf::Conf{'loop_command_decrease_factor'};
        }

        ## Send the reply message
        Sympa::Report::send_report_cmd($sender, $robot);
        $log->db_log(
            'robot'        => $robot,
            'list'         => $listname,
            'action'       => 'process_message',
            'parameters'   => "",
            'target_email' => "",
            'msg_id'       => $msg_id,
            'status'       => 'success',
            'error_type'   => '',
            'user_email'   => $sender
        );

    }

    return $status;
}

#sub DoSendMessage($message);
#DEPRECATED: Run upgrade_send_spool.pl to migrate message with old format.

############################################################
#  DoForward
############################################################
#  Handles a message sent to [list]-editor : the list editor,
#  [list]-request : the list owner or the listmaster.
#  Message is forwarded according to $function
#
# IN : -$name : list name (+) if ($function <> 'listmaster')
#      -$function (+): 'listmaster'|'owner'|'editor'
#      -$robot (+): robot
#      -$msg (+): ref(message object).
#
# OUT : 1
#     | undef
#
############################################################
sub DoForward {
    my ($name, $function, $robot, $message) = @_;
    $log->syslog('debug', '(%s, %s)', $name, $function);

    my $msg        = $message->as_entity;        #FIXME: not required.
    my $messageid  = $message->{'message_id'};
    my $msg_string = $message->as_string;        #FIMXE: not required.
    my $sender     = $message->{'sender'};
    chomp $sender;

    if ($message->{'spam_status'} eq 'spam') {
        $log->syslog(
            'notice',
            'Message for %s-%s ignored, because tagued as spam (message ID: %s)',
            $name,
            $function,
            $messageid
        );
        return undef;
    }

    ##  Search for the list
    my ($list, $admin, $host, $recipient, $priority);

    if ($function eq 'listmaster') {
        $recipient =
            $Conf::Conf{'listmaster_email'} . '@'
            . Conf::get_robot_conf($robot, 'host');
        $priority = 0;
    } else {
        $list = $message->{context};
        unless (ref $list eq 'Sympa::List') {
            $log->syslog(
                'notice',
                'Message for %s function %s ignored, unknown list %s (message ID: %s)',
                $name,
                $function,
                $name,
                $messageid
            );
            my $sympa_email = Conf::get_robot_conf($robot, 'sympa');
            unless (
                Sympa::send_file(
                    $robot,
                    'list_unknown',
                    $sender,
                    {   'list' => $name,
                        'date' => POSIX::strftime(
                            "%d %b %Y  %H:%M",
                            localtime(time)
                        ),
                        'boundary'       => $sympa_email . time,
                        'header'         => $message->header_as_string,
                        'auto_submitted' => 'auto-replied'
                    }
                )
                ) {
                $log->syslog('notice',
                    'Unable to send template "list_unknown" to %s', $sender);
            }
            return undef;
        }

        $admin     = $list->{'admin'};
        $recipient = $list->get_list_address($function);
        $priority  = $admin->{'priority'};
    }

    my @rcpt;

    $log->syslog('info',
        'Processing %s; message_id=%s; priority=%s; recipient=%s',
        $message, $messageid, $priority, $recipient);

    delete $message->{'rcpt'};
    delete $message->{'family'};

    if ($function eq 'listmaster') {
        @rcpt = Sympa::get_listmasters_email($robot);
        $log->syslog('notice', 'Warning: No listmaster defined in sympa.conf')
            unless @rcpt;
    } elsif ($function eq 'owner') {    # -request
        @rcpt = $list->get_admins_email('receptive_owner');
        @rcpt = $list->get_admins_email('owner') unless @rcpt;
        $log->syslog('notice', 'Warning: No owner defined at all in list %s',
            $name)
            unless @rcpt;
    } elsif ($function eq 'editor') {
        @rcpt = $list->get_admins_email('receptive_editor');
        @rcpt = $list->get_admins_email('actual_editor') unless @rcpt;
        $log->syslog('notice',
            'Warning: No owner and editor defined at all in list %s', $name)
            unless @rcpt;
    }

    # Did we find a recipient?
    unless (@rcpt) {
        $log->syslog('err',
            'Message for %s function %s ignored, %s undefined in list %s',
            $name, $function, $function, $name);
        my $string =
            sprintf
            'Impossible to forward a message to %s function %s : undefined in this list',
            $name, $function;
        Sympa::Report::reject_report_msg(
            'intern', $string, $sender,
            {   'msg_id'   => $messageid,
                'entry'    => 'forward',
                'function' => $function,
                'message'  => $msg
            },
            $robot,
            $msg_string,
            $list
        );
        $log->db_log(
            'robot'        => $robot,
            'list'         => $list->{'name'},
            'action'       => 'DoForward',
            'parameters'   => "$name,$function",
            'target_email' => '',
            'msg_id'       => $messageid,
            'status'       => 'error',
            'error_type'   => 'internal',
            'user_email'   => $sender
        );
        return undef;
    }

    # Add or remove several headers to forward message safely.
    # - Add X-Loop: field to mitigate mail looping.
    # - The Sender: field should be added (overwritten) at least for Sender ID
    #   (a.k.a. SPF 2.0) compatibility.  Note that Resent-Sender: field will
    #   be removed.
    # - Apply DMARC protection if needed.
    #FIXME: Existing DKIM signature depends on these headers will be broken.
    #FIXME: Currently messages via -request and -editor addresses will be
    #       protected against DMARC if neccessary.  The listmaster address
    #       would be protected, too.
    $message->add_header('X-Loop', $recipient);
    $message->replace_header('Sender',
        Conf::get_robot_conf($robot, 'request'));
    $message->delete_header('Resent-Sender');
    if ($function eq 'owner' or $function eq 'editor') {
        $message->dmarc_protect if $list;
    }

    # Overwrite envelope sender.  It is REQUIRED for delivery.
    $message->{envelope_sender} = Conf::get_robot_conf($robot, 'request');

    unless (defined $mailer->store($message, \@rcpt)) {
        $log->syslog('err', 'Impossible to forward mail for %s function %s',
            $name, $function);
        my $string =
            sprintf 'Impossible to forward a message for %s function %s',
            $name, $function;
        Sympa::Report::reject_report_msg(
            'intern', $string, $sender,
            {   'msg_id'   => $messageid,
                'entry'    => 'forward',
                'function' => $function,
                'message'  => $msg
            },
            $robot,
            $msg_string,
            $list
        );
        $log->db_log(
            'robot'        => $robot,
            'list'         => $list->{'name'},
            'action'       => 'DoForward',
            'parameters'   => "$name,$function",
            'target_email' => '',
            'msg_id'       => $messageid,
            'status'       => 'error',
            'error_type'   => 'internal',
            'user_email'   => $sender
        );
        return undef;
    }
    $log->db_log(
        'robot'        => $robot,
        'list'         => $list->{'name'},
        'action'       => 'DoForward',
        'parameters'   => "$name,$function",
        'target_email' => '',
        'msg_id'       => $messageid,
        'status'       => 'success',
        'error_type'   => '',
        'user_email'   => $sender
    );

    return 1;
}

####################################################
#  DoMessage
####################################################
#  Handles a message sent to a list. (Those that can
#  make loop and those containing a command are
#  rejected)
#
# IN : -$which (+): 'listname@hostname' - concerned list
#      -$message (+): ref(Message) - sent message
#      -$robot (+): robot
#
# OUT : 1 if ok (in order to remove the file from the queue)
#     | undef
#
####################################################
sub DoMessage {
    $log->syslog('debug2', '(%s)', @_);
    my $message = shift;

    my ($list, $robot_id, $listname);
    if (ref($message->{context}) eq 'Sympa::List') {
        $list     = $message->{context};
        $robot_id = $list->{'domain'};
        $listname = $list->{'name'};
    } elsif ($message->{context} and $message->{context} ne '*') {
        $robot_id = $message->{context};
        $listname = $message->{'localpart'};
    } else {
        $robot_id = '*';
        $listname = $message->{'localpart'};
    }

    my $messageid  = $message->{'message_id'};
    my $msg        = $message->as_entity;        #FIMXE: not required.
    my $msg_string = $msg->as_string;            #FIXME: not required.

    my $sender = $message->{'sender'};

    ## List unknown
    unless ($list) {
        $log->syslog('notice', 'Unknown list %s', $listname);
        my $sympa_email = Conf::get_robot_conf($robot_id, 'sympa');

        unless (
            Sympa::send_file(
                $robot_id,
                'list_unknown',
                $sender,
                {   'list' => $listname,
                    'date' =>
                        POSIX::strftime("%d %b %Y  %H:%M", localtime(time)),
                    'boundary'       => $sympa_email . time,
                    'header'         => $message->header_as_string,
                    'auto_submitted' => 'auto-replied'
                }
            )
            ) {
            $log->syslog('notice',
                'Unable to send template "list_unknown" to %s', $sender);
        }
        return undef;
    }

    my $start_time = time;

    $language->set_lang(
        $list->{'admin'}{'lang'},
        Conf::get_robot_conf($robot_id, 'lang'),
        $Conf::Conf{'lang'}, 'en'
    );

    ## Now check if the sender is an authorized address.

    $log->syslog('info',
        "Processing message %s for %s with priority %s, <%s>",
        $message, $list, $list->{'admin'}{'priority'}, $messageid);

    #FIXME: Might reject message when no message ID found.
    if ($messageid and $msgid_table{$list->get_list_id()}{$messageid}) {
        $log->syslog(
            'err',
            'Found known Message-ID <%s>, ignoring message %s which would cause a loop',
            $messageid,
            $message
        );
        $log->db_log(
            'robot'        => $robot_id,
            'list'         => $list->{'name'},
            'action'       => 'DoMessage',
            'parameters'   => $message->get_id,
            'target_email' => '',
            'msg_id'       => $messageid,
            'status'       => 'error',
            'error_type'   => 'known_message',
            'user_email'   => $sender
        );
        return undef;
    }

    # Reject messages with commands
    if ($Conf::Conf{'misaddressed_commands'} =~ /reject/i) {
        # Check the message for commands and catch them.
        my $cmd = _check_command($message);
        if (defined $cmd) {
            $log->syslog('err',
                'Found command "%s" in message, ignoring message', $cmd);
            Sympa::Report::reject_report_msg('user', 'routing_error', $sender,
                {'message' => $message},
                $robot_id, $msg_string, $list);
            $log->db_log(
                'robot'        => $robot_id,
                'list'         => $list->{'name'},
                'action'       => 'DoMessage',
                'parameters'   => $message->get_id,
                'target_email' => '',
                'msg_id'       => $messageid,
                'status'       => 'error',
                'error_type'   => 'routing_error',
                'user_email'   => $sender
            );
            return undef;
        }
    }

    my $admin = $list->{'admin'};
    unless ($admin) {
        $log->syslog('err', 'List config is undefined');
        Sympa::Report::reject_report_msg('intern', '', $sender,
            {'message' => $message},
            $robot_id, $msg_string, $list);
        $log->db_log(
            'robot'        => $robot_id,
            'list'         => $list->{'name'},
            'action'       => 'DoMessage',
            'parameters'   => $message->get_id,
            'target_email' => '',
            'msg_id'       => $messageid,
            'status'       => 'error',
            'error_type'   => 'internal',
            'user_email'   => $sender
        );
        return undef;
    }

    my $customheader = $admin->{'custom_header'};

    # Check if the message is too large
    my $max_size = $list->{'admin'}{'max_size'};

    if ($max_size && $message->{'size'} > $max_size) {
        $log->syslog('info',
            'Message for %s from %s rejected because too large (%d > %d)',
            $listname, $sender, $message->{'size'}, $max_size);
        Sympa::Report::reject_report_msg(
            'user',
            'message_too_large',
            $sender,
            {   'msg_size' => int($message->{'size'} / 1024),
                'max_size' => int($max_size / 1024)
            },
            $robot_id,
            '', $list
        );
        $log->db_log(
            'robot'        => $robot_id,
            'list'         => $list->{'name'},
            'action'       => 'DoMessage',
            'parameters'   => $message->get_id,
            'target_email' => '',
            'msg_id'       => $messageid,
            'status'       => 'error',
            'error_type'   => 'message_too_large',
            'user_email'   => $sender
        );
        return undef;
    }

    my $rc;

    my $context = {
        'sender'  => $sender,
        'message' => $message
    };

    ## list msg topic
    if ($list->is_there_msg_topic()) {

        my $info_msg_topic =
            $list->load_msg_topic_file($messageid, $robot_id);

        # is msg already tagged ?
        if (ref($info_msg_topic) eq "HASH") {
            if ($info_msg_topic->{'method'} eq "sender") {
                $context->{'topic_sender'} = $info_msg_topic->{'topic'};

            } elsif ($info_msg_topic->{'method'} eq "editor") {
                $context->{'topic_editor'} = $info_msg_topic->{'topic'};

            } elsif ($info_msg_topic->{'method'} eq "auto") {
                $context->{'topic_auto'} = $info_msg_topic->{'topic'};
            }

            # not already tagged
        } else {
            $context->{'topic_auto'} = $list->automatic_tag($message);
        }

        $context->{'topic'} =
               $context->{'topic_auto'}
            || $context->{'topic_sender'}
            || $context->{'topic_editor'};
        $context->{'topic_needed'} =
            (!$context->{'topic'} && $list->is_msg_topic_tagging_required());
    }

    ## Call scenarii : auth_method MD5 do not have any sense in send
    ## scenarii because auth is perfom by distribute or reject command.

    my $action;
    my $result;

    # the order of the following 3 lines is important ! SMIME > DKIM > SMTP
    my $auth_method =
          $message->{'smime_signed'} ? 'smime'
        : $message->{'md5_check'}    ? 'md5'
        : $message->{'dkim_pass'}    ? 'dkim'
        :                              'smtp';

    $result = Sympa::Scenario::request_action($list, 'send', $auth_method,
        $context);
    $action = $result->{'action'} if (ref($result) eq 'HASH');

    unless (defined $action) {
        $log->syslog(
            'err',
            'Message (%s) ignored because unable to evaluate scenario "send" for list %s',
            $messageid,
            $listname
        );
        Sympa::Report::reject_report_msg(
            'intern',
            'Message ignored because scenario "send" cannot be evaluated',
            $sender,
            {'msg_id' => $messageid, 'message' => $message},
            $robot_id,
            $msg_string,
            $list
        );
        $log->db_log(
            'robot'        => $robot_id,
            'list'         => $list->{'name'},
            'action'       => 'DoMessage',
            'parameters'   => $message->get_id,
            'target_email' => '',
            'msg_id'       => $messageid,
            'status'       => 'error',
            'error_type'   => 'internal',
            'user_email'   => $sender
        );
        return undef;
    }

    ## message topic context
    if (($action =~ /^do_it/) && ($context->{'topic_needed'})) {
        $action = 'editorkey'
            if (
            $list->{'admin'}{'msg_topic_tagging'} eq 'required_moderator');
        $action = 'request_auth'
            if ($list->{'admin'}{'msg_topic_tagging'} eq 'required_sender');
    }

    if ($action =~ /^do_it/) {
        $message->{shelved}{dkim_sign} = 1
            if Sympa::Tools::Data::is_in_array(
            $list->{'admin'}{'dkim_signature_apply_on'}, 'any')
            or (
            Sympa::Tools::Data::is_in_array(
                $list->{'admin'}{'dkim_signature_apply_on'},
                'smime_authenticated_messages')
            and $message->{'smime_signed'}
            )
            or (
            Sympa::Tools::Data::is_in_array(
                $list->{'admin'}{'dkim_signature_apply_on'},
                'dkim_authenticated_messages')
            and $message->{'dkim_pass'}
            );

        ## Check TT2 syntax for merge_feature.
        unless ($message->test_personalize($list)) {
            $log->syslog(
                'err',
                'Failed to personalize. Message %s for list %s was rejected',
                $message,
                $list
            );
            Sympa::send_dsn($list, $message, {}, '5.6.5');
            return undef;
        }

        my $numsmtp;
        #XXX eval {
        $numsmtp = Sympa::List::distribute_msg($message);
        #XXX };
        ## Keep track of known message IDs...if any
        $msgid_table{$list->get_list_id()}{$messageid} = time
            if $messageid;

        unless (defined($numsmtp)) {
            $log->syslog('err', 'Unable to send message to list %s',
                $listname);
            Sympa::Report::reject_report_msg('intern', '', $sender,
                {'msg_id' => $messageid, 'message' => $message},
                $robot_id, $msg_string, $list);
            $log->db_log(
                'robot'        => $robot_id,
                'list'         => $list->{'name'},
                'action'       => 'DoMessage',
                'parameters'   => $message->get_id,
                'target_email' => '',
                'msg_id'       => $messageid,
                'status'       => 'error',
                'error_type'   => 'internal',
                'user_email'   => $sender
            );
            return undef;
        }
        $log->syslog(
            'info',
            'Message %s for %s from %s accepted (%d seconds, %d sessions, %d subscribers), message ID=%s, size=%d',
            $message,
            $listname,
            $sender,
            time - $start_time,
            $numsmtp,
            $list->get_total(),
            $messageid,
            $message->{'size'}
        );

        return 1;
    } elsif ($action =~ /^request_auth/) {
        ## Check syntax for merge_feature.
        unless ($message->test_personalize($list)) {
            $log->syslog(
                'err',
                'Failed to personalize. Message %s for list %s was rejected',
                $message,
                $list
            );
            Sympa::send_dsn($list, $message, {}, '5.6.5');
            return undef;
        }

        my $key = Sympa::List::send_confirm_to_sender($message);

        unless (defined $key) {
            $log->syslog('err',
                'Failed to send confirmation of %s for %s to sender %s',
                $message, $list, $sender);
            Sympa::Report::reject_report_msg(
                'intern', 'The request authentication sending failed',
                $sender, {'msg_id' => $messageid, 'message' => $message},
                $robot_id, $msg_string,
                $list
            );
            $log->db_log(
                'robot'        => $robot_id,
                'list'         => $list->{'name'},
                'action'       => 'DoMessage',
                'parameters'   => $message->get_id,
                'target_email' => '',
                'msg_id'       => $messageid,
                'status'       => 'error',
                'error_type'   => 'internal',
                'user_email'   => $sender
            );
            return undef;
        }
        $log->syslog('notice',
            'Message for %s from %s kept for authentication with key %s',
            $listname, $sender, $key);
        $log->db_log(
            'robot'        => $robot_id,
            'list'         => $list->{'name'},
            'action'       => 'DoMessage',
            'parameters'   => $message->get_id,
            'target_email' => '',
            'msg_id'       => $messageid,
            'status'       => 'success',
            'error_type'   => 'kept_for_auth',
            'user_email'   => $sender
        );
        return 1;
    } elsif ($action =~ /^editorkey(\s?,\s?(quiet))?/) {
        my $quiet = $2;

        ## Check syntax for merge_feature.
        unless ($message->test_personalize($list)) {
            $log->syslog(
                'err',
                'Failed to personalize. Message %s for list %s was rejected',
                $message,
                $list
            );
            Sympa::send_dsn($list, $message, {}, '5.6.5');
            return undef;
        }

        my $key = Sympa::List::send_confirm_to_editor($message, 'md5');

        unless (defined $key) {
            $log->syslog(
                'err',
                'Failed to moderation request of %s from %s for list %s to editor(s)',
                $message,
                $sender,
                $list
            );
            Sympa::Report::reject_report_msg(
                'intern',
                'The request moderation sending to moderator failed.',
                $sender,
                {'msg_id' => $messageid, 'message' => $message},
                $robot_id,
                $msg_string,
                $list
            );
            $log->db_log(
                'robot'        => $robot_id,
                'list'         => $list->{'name'},
                'action'       => 'DoMessage',
                'parameters'   => $message->get_id,
                'target_email' => '',
                'msg_id'       => $messageid,
                'status'       => 'error',
                'error_type'   => 'internal',
                'user_email'   => $sender
            );
            return undef;
        }

        $log->syslog('info',
            'Key %s of message %s for list %s from %s sent to editors',
            $key, $message, $listname, $sender);

        # do not report to the sender if the message was tagged as a spam
        unless ($quiet or $message->{'spam_status'} eq 'spam') {
            unless (
                Sympa::Report::notice_report_msg(
                    'moderating_message', $sender,
                    {'message' => $message}, $robot_id,
                    $list
                )
                ) {
                $log->syslog(
                    'notice',
                    'Unable to send template "message_report", entry "moderating_message" to %s',
                    $sender
                );
            }
        }
        return 1;
    } elsif ($action =~ /^editor(\s?,\s?(quiet))?/) {
        my $quiet = $2;

        ## Check syntax for merge_feature.
        unless ($message->test_personalize($list)) {
            $log->syslog(
                'err',
                'Failed to personalize. Message %s for list %s was rejected',
                $message,
                $list
            );
            Sympa::send_dsn($list, $message, {}, '5.6.5');
            return undef;
        }

        my $key = Sympa::List::send_confirm_to_editor($message, 'smtp');

        unless (defined $key) {
            $log->syslog(
                'err',
                'Failed to send moderation request of %s by %s for list %s to editor(s)',
                $message,
                $sender,
                $list
            );
            Sympa::Report::reject_report_msg(
                'intern',
                'The request moderation sending to moderator failed.',
                $sender,
                {'msg_id' => $messageid, 'message' => $message},
                $robot_id,
                $msg_string,
                $list
            );
            $log->db_log(
                'robot'        => $robot_id,
                'list'         => $list->{'name'},
                'action'       => 'DoMessage',
                'parameters'   => $message->get_id,
                'target_email' => '',
                'msg_id'       => $messageid,
                'status'       => 'error',
                'error_type'   => 'internal',
                'user_email'   => $sender
            );
            return undef;
        }

        $log->syslog('info', 'Message %s for %s from %s sent to editors',
            $message, $listname, $sender);

        # do not report to the sender if the message was tagged as a spam
        unless ($quiet or $message->{'spam_status'} eq 'spam') {
            unless (
                Sympa::Report::notice_report_msg(
                    'moderating_message', $sender,
                    {'message' => $message}, $robot_id,
                    $list
                )
                ) {
                $log->syslog('notice',
                    "sympa::DoMessage(): Unable to send template 'message_report', type 'success', entry 'moderating_message' to $sender"
                );
            }
        }
        return 1;
    } elsif ($action =~ /^reject(,(quiet))?/) {
        my $quiet = $2;

        $log->syslog(
            'notice',
            'Message for %s from %s rejected(%s) because sender not allowed',
            $listname,
            $sender,
            $result->{'tt2'}
        );

        # do not report to the sender if the message was tagued as a spam
        unless ($quiet or $message->{'spam_status'} eq 'spam') {
            if (defined $result->{'tt2'}) {
                unless (
                    Sympa::send_file(
                        $list, $result->{'tt2'},
                        $sender, {'auto_submitted' => 'auto-replied'}
                    )
                    ) {
                    $log->syslog('notice',
                        "sympa::DoMessage(): Unable to send template '$result->{'tt2'}' to $sender"
                    );
                }
            } else {
                unless (
                    Sympa::Report::reject_report_msg(
                        'auth', $result->{'reason'},
                        $sender, {'message' => $message},
                        $robot_id, $msg_string,
                        $list
                    )
                    ) {
                    $log->syslog(
                        'notice',
                        'Unable to send template "message_report", type "auth" to %s',
                        $sender
                    );
                }
            }
        }
        $log->db_log(
            'robot'        => $robot_id,
            'list'         => $list->{'name'},
            'action'       => 'DoMessage',
            'parameters'   => $message->get_id,
            'target_email' => '',
            'msg_id'       => $messageid,
            'status'       => 'error',
            'error_type'   => 'rejected_authorization',
            'user_email'   => $sender
        );
        return undef;
    } else {
        $log->syslog('err',
            'Unknown action %s returned by the scenario "send"', $action);
        Sympa::Report::reject_report_msg(
            'intern', 'Unknown action returned by the scenario "send"',
            $sender, {'msg_id' => $messageid, 'message' => $message},
            $robot_id, $msg_string,
            $list
        );
        $log->db_log(
            'robot'        => $robot_id,
            'list'         => $list->{'name'},
            'action'       => 'DoMessage',
            'parameters'   => $message->get_id,
            'target_email' => '',
            'msg_id'       => $messageid,
            'status'       => 'error',
            'error_type'   => 'internal',
            'user_email'   => $sender
        );
        return undef;
    }
}

# Checks command in the body of the message.
# If there are any commands in it, returns string.
#
# IN : -$message (+): ref(Sympa::Message) - message to check
#
# OUT : -string: Command found
#       -undef:  Else.
#
# Old name: tools::checkcommand().
sub _check_command {
    my $message = shift;

    my $commands_re = $Conf::Conf{'misaddressed_commands_regexp'};
    return undef unless defined $commands_re and length $commands_re;

    # Check for commands in the subject.
    my $subject_field = $message->{'decoded_subject'};
    $subject_field = '' unless defined $subject_field;
    $subject_field =~ s/\n//mg;    # multiline subjects
    my $re_regexp = Sympa::Regexps::re();
    $subject_field =~ s/^\s*(?:$re_regexp)?\s*(.*)\s*$/$1/i;

    if ($subject_field =~ /^($commands_re)$/im) {
        return $1;
    }

    my @body = map { s/\r\n|\n//; $_ } split /(?<=\n)/,
        ($message->get_plain_body || '');

    # More than 5 lines in the text.
    return undef if scalar @body > 5;

    foreach my $line (@body) {
        if ($line =~ /^($commands_re)\b/im) {
            return $1;
        }

        # Control is only applied to first non-blank line.
        last unless $line =~ /\A\s*\z/;
    }
    return undef;
}

############################################################
#  DoCommand
############################################################
#  Handles a command sent to the list manager.
#
# IN : -$rcpt : recipient | <listname>-<subscribe|unsubscribe>
#      -$robot (+): robot
#      -$message : ref(Message) with :
#        ->msg (+): ref(MIME::Entity) : message containing command
#        ->filename (+): file containing message
#
# OUT : $success
#     | undef
#
##############################################################
sub DoCommand {
    $log->syslog('debug', '(%s, %s, %s)', @_);
    my ($rcpt, $robot, $message) = @_;

    ## boolean
    my $cmd_found = 0;

    my $messageid = $message->{'message_id'};
    my ($success, $status);

    $log->syslog(
        'debug',
        "Processing command with priority %s, %s",
        $Conf::Conf{'sympa_priority'}, $messageid
    );

    my $sender = $message->{'sender'};

    if ($message->{'spam_status'} eq 'spam') {
        $log->syslog(
            'notice',
            'Message for list %s@%s ignored, because tagged as spam (message ID: %s)',
            $rcpt,
            $robot,
            $messageid
        );
        return undef;
    }

    ## Detect loops
    if ($msgid_table{'sympa@' . $robot}{$messageid}) {
        $log->syslog('err',
            'Found known Message-ID, ignoring command which would cause a loop'
        );
        $log->db_log(
            'robot'        => $robot,
            'list'         => $rcpt,
            'action'       => 'DoCommand',
            'parameters'   => "$rcpt,$robot,$message",
            'target_email' => '',
            'msg_id'       => $messageid,
            'status'       => 'error',
            'error_type'   => 'known_message',
            'user_email'   => $sender
        );
        return undef;
    }    ## Clean old files from spool

    ## Keep track of known message IDs...if any
    $msgid_table{'sympa@' . $robot}{$messageid} = time
        if ($messageid);

    ## If X-Sympa-To = <listname>-<subscribe|unsubscribe> parse as a unique
    ## command
    if ($rcpt =~ /^(\S+)-(subscribe|unsubscribe)(\@(\S+))?$/o) {
        my ($listname, $command) = ($1, $2);
        $log->syslog('debug', 'Processing message for %s-%s',
            $listname, $command);
        # FIXME: at this point $message->{'dkim_pass'} does not verify that
        # Subject: is part of the signature. It SHOULD !
        my $auth_level = $message->{'dkim_pass'} ? 'dkim' : undef;

        Sympa::Commands::parse($sender, $robot,
            sprintf('%s %s', $command, $listname),
            $auth_level, $message);
        $log->db_log(
            'robot'        => $robot,
            'list'         => $listname,
            'action'       => 'DoCommand',
            'parameters'   => "$rcpt,$robot,$message",
            'target_email' => '',
            'msg_id'       => $messageid,
            'status'       => 'success',
            'error_type'   => '',
            'user_email'   => $sender
        );
        return 1;
    }

    ## Process the Subject of the message
    ## Search and process a command in the Subject field
    my $subject_field = $message->{'decoded_subject'};
    $subject_field = '' unless defined $subject_field;
    $subject_field =~ s/\n//mg;    ## multiline subjects
    my $re_regexp = Sympa::Regexps::re();
    $subject_field =~ s/^\s*(?:$re_regexp)?\s*(.*)\s*$/$1/i;

    #FIXME
    my $auth_level =
          $message->{'smime_signed'} ? 'smime'
        : $message->{'dkim_pass'}    ? 'dkim'
        :                              undef;

    if (defined $subject_field and $subject_field =~ /\S/) {
        $success ||= Sympa::Commands::parse($sender, $robot, $subject_field,
            $auth_level, $message);
        unless ($success and $success eq 'unknown_cmd') {
            $cmd_found = 1;
        }
    }

    my $i;
    my $size;

    ## Process the body of the message
    ## unless subject contained commands or message has no body
    unless ($cmd_found) {
        my $body = $message->get_plain_body;
        unless (defined $body) {
            $log->syslog('err', 'Could not change multipart to singlepart');
            Sympa::Report::global_report_cmd('user', 'error_content_type',
                {});
            $log->db_log(
                'robot'        => $robot,
                'list'         => $rcpt,                     #FIXME
                'action'       => 'DoCommand',
                'parameters'   => "$rcpt,$robot,$message",
                'target_email' => '',
                'msg_id'       => $messageid,
                'status'       => 'error',
                'error_type'   => 'error_content_type',
                'user_email'   => $sender
            );
            return $success ? 1 : undef;
        }

        foreach $i (split /\r\n|\r|\n/, $body) {
            last if $i =~ /^-- $/;    ## ignore signature
            $i =~ s/^\s*>?\s*(.*)\s*$/$1/g;
            next unless length $i;    ## skip empty lines
            next if $i =~ /^\s*\#/;

            #FIXME
            $auth_level =
                  $message->{'smime_signed'} ? 'smime'
                : $message->{'dkim_pass'}    ? 'dkim'
                :                              $auth_level;
            $status = Sympa::Commands::parse($sender, $robot, $i, $auth_level,
                $message);

            $cmd_found = 1;    # if problem no_cmd_understood is sent here
            if ($status eq 'unknown_cmd') {
                $log->syslog('notice', 'Unknown command found: %s', $i);
                Sympa::Report::reject_report_cmd('user', 'not_understood', {},
                    $i);
                $log->db_log(
                    'robot'        => $robot,
                    'list'         => $rcpt,                     #FIXME
                    'action'       => 'DoCommand',
                    'parameters'   => "$rcpt,$robot,$message",
                    'target_email' => '',
                    'msg_id'       => $messageid,
                    'status'       => 'error',
                    'error_type'   => 'not_understood',
                    'user_email'   => $sender
                );
                last;
            }
            if ($i =~ /^(quit|end|stop|-)\s*$/io) {
                last;
            }

            $success ||= $status;
        }
    }

    ## No command found
    unless ($cmd_found) {
        $log->syslog('info', "No command found in message");
        Sympa::Report::global_report_cmd('user', 'no_cmd_found', {});
        $log->db_log(
            'robot'        => $robot,
            'list'         => $rcpt,                     #FIXME
            'action'       => 'DoCommand',
            'parameters'   => "$rcpt,$robot,$message",
            'target_email' => '',
            'msg_id'       => $messageid,
            'status'       => 'error',
            'error_type'   => 'no_cmd_found',
            'user_email'   => $sender
        );
        return undef;
    }

    return $success ? 1 : undef;
}

############################################################
#  SendDigest
############################################################
#  Read the queuedigest and send old digests to the subscribers
#  with the digest option.
#
# IN : -
#
# OUT : -
#     | undef
#
##############################################################
sub SendDigest {
    $log->syslog('debug', '(%s => %s, %s => %s)', @_);
    my %options     = @_;
    my $keep_digest = $options{keep_digest};
    my $send_now    = $options{send_now};

    my $collection = Sympa::Spool::Digest::Collection->new;
    while (1) {
        my ($spool, $handle) = $collection->next;
        last unless $handle;    # No more spools.
        next unless $spool;     # Malformed entry.

        my $list     = $spool->{context};
        my $robot_id = $list->{'domain'};

        $language->set_lang(
            $list->{'admin'}{'lang'},
            Conf::get_robot_conf($robot_id, 'lang'),
            $Conf::Conf{'lang'}, 'en'
        );

        if ($send_now or Sympa::List::may_distribute_digest($spool)) {
            # Blindly send the message to all users.
            $log->syslog('info', 'Sending digest to list %s', $list);
            my $start_time = time;
            Sympa::List::distribute_digest($collection, $spool, $handle,
                keep_digest => $keep_digest);

            $log->syslog('info', 'Digest of the list %s sent (%d seconds)',
                $list, time - $start_time);
            $log->db_log(
                'robot'        => $robot_id,
                'list'         => $list->{'name'},
                'action'       => 'SendDigest',
                'parameters'   => "",
                'target_email' => '',
                'msg_id'       => '',
                'status'       => 'success',
                'error_type'   => '',
                'user_email'   => ''
            );
        }
    }

    return 1;
}

## Cleanup the msgid_table every 'msgid_table_cleanup_frequency' sec
## Removes all entries older than 'msgid_table_cleanup_ttl' sec
sub clean_msgid_table {

    foreach my $rcpt (keys %msgid_table) {
        foreach my $msgid (keys %{$msgid_table{$rcpt}}) {
            if (time > $msgid_table{$rcpt}{$msgid} +
                $Conf::Conf{'msgid_table_cleanup_ttl'}) {
                delete $msgid_table{$rcpt}{$msgid};
            }
        }
    }

    return 1;
}

__END__

=encoding utf-8

=head1 NAME

sympa_msg, sympa_msg.pl - Daemon to handle incoming messages

=head1 SYNOPSIS

S<B<sympa_msg.pl> [ B<-d, --debug> ] [ B<-f, --file>=I<another.sympa.conf> ]>
      S<[ B<-k, --keepcopy>=I<directory> ]>
      S<[ B<-l, --lang>=I<lang> ]> [ B<-m, --mail> ]
      S<[ B<-h, --help> ]> [ B<-v, --version> ]

=head1 DESCRIPTION

Sympa_msg.pl is a program which scans permanently the incoming message spool
and processes each message.

Messages bound for the lists and authorized sending are modified as neccesity
and at last stored into digest spool, archive spool and bulk spool.
Those bound for command addresses are interpreted and appropriate actions are
taken.
Those bound for listmasters or list admins are forwarded to them.

=head1 OPTIONS

Sympa_msg.pl follows the usual GNU command line syntax,
with long options starting with two dashes (C<-->).  A summary of
options is included below.

=over 4

=item B<-d>, B<--debug>

Enable debug mode.

=item B<-f>, B<--config=>I<file>

Force Sympa to use an alternative configuration file instead
of F</usr/pkg/etc/sympa/sympa.conf>.

=item B<-l>, B<--lang=>I<lang>

Set this option to use a language for Sympa. The corresponding
gettext catalog file must be located in F</usr/pkg/share/locale>
directory.

=item B<--log_level=>I<level>

Sets Sympa log level.

=back

F<sympa_msg.pl> may run in daemon mode with following options.

=over 4

=item B<--foreground>

The process remains attached to the TTY.

=item B<-k>, B<--keepcopy=>F<directory>

This option tells Sympa to keep a copy of every incoming message, 
instead of deleting them. `directory' is the directory to 
store messages.

=item B<-m>, B<--mail>

Sympa will log calls to sendmail, including recipients. This option is
useful for keeping track of each mail sent (log files may grow faster
though).

=item B<--service=process_command>|B<process_message>|B<process_creation>

B<Note>:
This option was deprecated.

Process is dedicated to messages distribution, commands or to automatic lists
creation (default three of them).

=back

With following options F<sympa_msg.pl> will print some information and exit.

=over 4

=item B<-h>, B<--help>

Print this help message.

=item B<-v>, B<--version>

Print the version number.

=back

=head1 FILES

F</usr/pkg/etc/sympa/sympa.conf> main configuration file.

F</var/sympa/run/sympa_msg.pid> this file contains the process ID
of F<sympa_msg.pl>.

=head1 SEE ALSO

L<sympa.conf(5)>, L<sympa(1)>.

L<archived(8)>, L<bulk(8)>, L<bounced(8)>, L<sympa_automatic(8)>,
L<task_manager(8)>.

=head1 HISTORY

F<sympa.pl> was originally written by:

=over 4

=item Serge Aumont

ComitE<233> RE<233>seau des UniversitE<233>s

=item Olivier SalaE<252>n

ComitE<233> RE<233>seau des UniversitE<233>s

=back

As of Sympa 6.2b.4, it was split into three programs:
F<sympa.pl> command line utility, F<sympa_automatic.pl> daemon and
F<sympa_msg.pl> daemon.

=cut
