#!/opt/perl/bin/perl -w
# -*- perl -*-

=pod

=encoding UTF-8

=head1 NAME
  ze-filter_counters_

=head1 APPLICABLE SYSTEMS

  MTAs running ze-filter filter 

=head1 CONFIGURATION

  [ze-filter_counters_*]
    user root

The following environment variables are used by this plugin:

    env.disable

=over 4

=back

=head1 BUGS/GOTCHAS

  None known, but some improvements to do...

=head1 AUTHOR

    Jose-Marcio Martins da Cruz - mailto:Jose-Marcio.Martins@mines-paristech.fr
    Ecole Nationale Superieure des Mines de Paris

  This plugin was inspired by the work of Christophe Decor
    Christophe Decor - mailto:decor@supagro.inra.fr
    INRA - Institut National de Recherche Agronomique

=head1 COPYRIGHT

=head1 VERSION 


=head1 MAGIC MARKERS

 #%# family=contrib
 #%# capabilities=autoconf suggest

=cut

use strict;
use warnings;

my $SMCF     = "/etc/ze-filter/ze-filter.cf";
my $FSTATS   = "/var/ze-filter/files/ze-stats";
my $WORKDIR  = "/var/ze-filter/wdb";
my $CONSTDIR = "/var/ze-filter/cdb";

# ------------------------------------------------
#
#
my $debug = 0;

my $AppName = $0;
$AppName = `basename $AppName`;
chomp $AppName;

my $AppExt = undef;
if ($AppName =~ /_([a-z0-9.-]+)$/i) {
  $AppExt = $1;
}

my %AppDisable = ();
{
  my $disable = $ENV{'disable'} || '';

  printf "# disable $disable\n";
  %AppDisable = map {$_ => 1;} split " ", $disable;
}

my %StaticData = ();
GetStaticData(\%StaticData);

my $AppType = "";
if (defined $AppExt && exists $StaticData{config}{"$AppExt+type"}) {
  $AppType = $StaticData{config}{"$AppExt+type"};
}

my $StateFile = undef;

if (exists $ENV{MUNIN_PLUGSTATE} && defined $ENV{MUNIN_PLUGSTATE}) {
  $StateFile = "$ENV{MUNIN_PLUGSTATE}/$AppName.state";
}
if (exists $ENV{statefile} && defined $ENV{statefile}) {
  $StateFile = $ENV{statefile};
}

# ------------------------------------------------
#
#
if ($#ARGV >= 0 && $ARGV[0] eq "autoconf") {
  unless (-f $SMCF) {
    print "no\n";
    exit 0;
  }
  print "yes\n";
  exit 0;
}

# ------------------------------------------------
#
#
if ($#ARGV >= 0 && $ARGV[0] eq "suggest") {

  # results from /var/ze-filter/files/ze-stats
  if (-f $SMCF && -f $FSTATS) {
    my %Data = ();
    my $line = GetStatsLine(\%Data);
    exit 1 unless defined $line && $line ne "";

    foreach my $k (keys %{$StaticData{config}}) {
      next unless $k =~ /(.*)[+]type$/;
      my $graph = $1;
      my $type  = $StaticData{config}{$k};
      if ($type =~ /^(counters|ratio)$/) {
        next unless exists $StaticData{config}{$graph . "+data"};
        my $data = $StaticData{config}{$graph . "+data"};
        $data =~ tr#,/# #;
        my @fields = split " ", $data;
        my $ok     = 1;
        my $bad    = "";
        for my $f (@fields) {
          unless (exists $Data{$f}) {
            $ok = 0;
            $bad .= " $f";
          }
        }
        if ($ok) {
          print "$graph\n" if $ok;
        } else {
          printf "# %-12s - Lacking fields : %s\n", $graph, $bad;
        }
      }
    }
  }

  # results from /var/ze-filter/wdb
  if (-f $SMCF && (-d $WORKDIR || -d $CONSTDIR)) {
    foreach my $k (keys %{$StaticData{config}}) {
      next unless $k =~ /(.*)[+]type$/;
      my $graph = $1;
      my $type  = $StaticData{config}{$k};
      if ($type =~ /^(dbcounters)$/) {
        next unless exists $StaticData{config}{$graph . "+data"};
        my $data = $StaticData{config}{$graph . "+data"};
        $data =~ tr#,/# #;
        my @fields = split " ", $data;
        my $ok     = 1;
        my $bad    = "";
        for my $f (@fields) {
          unless (-f "$WORKDIR/$f.db" || -f "$CONSTDIR/$f.db") {
            $ok = 0;
            $bad .= " $f";
          }
        }
        if ($ok) {
          print "$graph\n" if $ok;
        } else {
          printf "# %-12s - Lacking fields : %s\n", $graph, $bad;
        }
      }
    }
  }
  exit 0;
}

# ------------------------------------------------
#
#
if (defined $AppExt && $AppType eq "counters") {
  my %Data = ();
  my $line = GetStatsLine(\%Data);
  exit 1 unless defined $line && $line ne "";

  my $DataAggregate = $StaticData{config}{"$AppExt+data"};
  printf "# Data Aggregate : %s\n", $DataAggregate;
  my @Values = split " ", $DataAggregate;
  if ($#ARGV >= 0 && $ARGV[0] eq "config") {
    my @Keys = grep {$_ =~ /^$AppExt-/;} keys %{$StaticData{config}};
    foreach my $k (@Keys) {
      my $v = $StaticData{config}{$k};
      $k =~ s/^$AppExt-//;
      printf "%-16s %s\n", $k, $v;
    }
    printf "graph_order %s\n", (join " ", sort @Values);
    foreach my $k (@Values) {
      if (exists $Data{$k}) {
        my $label = $k;
        if (exists $StaticData{label}{$k}) {
          $label = $StaticData{label}{$k};
        }
        printf "%s.label %s\n", $k, $label;
        if (exists $StaticData{label}{"$k.info"}) {
          $label = $StaticData{label}{"$k.info"};
        }
        printf "%s.info %s\n", $k, $label;
        printf "%s.min 0\n", $k;
        printf "%s.type DERIVE\n", $k;
      }
    }
    exit 0;
  }

  if ($#ARGV < 0) {
    foreach my $k (@Values) {
      if (exists $Data{$k}) {
        printf "%s.value %s\n", $k, $Data{$k};
      }
    }
    exit 0;
  }
  exit 1;
}

# ------------------------------------------------
#
#
if (defined $AppExt && $AppType eq "ratio") {
  my %Data = ();

  my $line = GetStatsLine(\%Data);
  exit 1 unless defined $line && $line ne "";

  my $DataAggregate = $StaticData{config}{"$AppExt+data"};
  printf "# Data Aggregate : %s\n", $DataAggregate;
  my ($Ratios, $Base, undef) = split "/", $DataAggregate;
  exit 1 unless defined $Ratios && defined $Base;

  my @Values = split ",",    $Ratios;
  my @Den    = split "[ ]+", $Base;

  if ($#ARGV >= 0 && $ARGV[0] eq "config") {
    my @Keys = grep {$_ =~ /^$AppExt-/;} keys %{$StaticData{config}};
    foreach my $k (@Keys) {
      my $v = $StaticData{config}{$k};
      $k =~ s/^$AppExt-//;
      printf "%-16s %s\n", $k, $v;
    }

    my $nr = 0;
    printf "graph_order %s\n", (join " ", sort @Values);
    foreach my $num (@Values) {
      printf "# running for %s\n", $num;
      my @Num = split "[ ]+", $num;

      my $k = sprintf "%s%02d", $AppExt, $nr++;
      my $label = $k;
      if (exists $StaticData{label}{$k}) {
        $label = $StaticData{label}{$k};
      }
      printf "%s.label %s\n", $k, $label;
      if (exists $StaticData{label}{"$k.info"}) {
        $label = $StaticData{label}{"$k.info"};
      }
      printf "%s.info %s\n", $k, $label;
      printf "%s.min 0\n",   $k;
      printf "%s.max 100\n", $k;
      printf "%s.type GAUGE\n", $k;
    }
    exit 0;
  }

  if ($#ARGV < 0) {
    my %State = ();
    my %New = map {$_ => $Data{$_}} keys %Data;
    if (EvalBoolean($StaticData{config}{"$AppExt+state"})) {
      ReadState(\%State);
      DiffFromState(\%State, \%New, \%Data);
    }
    if ($debug) {
      foreach my $k (sort keys %Data) {
        printf "# DIFF %-16s %12d %12d %12d\n", $k, $State{$k}, $New{$k},
          $Data{$k};
      }
    }
    my $nr = 0;
    foreach my $num (@Values) {
      printf "# running for %s\n", $num;
      my @Num = split "[ ]+", $num;
      my $n   = 0;
      my $d   = 0;
      foreach my $k (@Num) {
        if (exists $Data{$k}) {
          $n += $Data{$k};
        }
      }
      foreach my $k (@Den) {
        if (exists $Data{$k}) {
          $d += $Data{$k};
        }
      }
      my $k = sprintf "%s%02d", $AppExt, $nr++;
      printf "%s.value %s\n", $k, $d > 0 ? ((100. * $n) / $d) : 0;
    }
    if (EvalBoolean($StaticData{config}{"$AppExt+state"})) {
      SaveState(\%New);
    }
    exit 0;
  }
  exit 1;
}

# ------------------------------------------------
#
#
if (defined $AppExt && $AppType eq "dbcounters") {
  my $DataAggregate = $StaticData{config}{"$AppExt+data"};
  printf "# Data Aggregate : %s\n", $DataAggregate;
  my @Values = split " ", $DataAggregate;
  if ($#ARGV >= 0 && $ARGV[0] eq "config") {
    my @Keys = grep {$_ =~ /^$AppExt-/;} keys %{$StaticData{config}};
    foreach my $k (@Keys) {
      my $v = $StaticData{config}{$k};
      $k =~ s/^$AppExt-//;
      printf "%-16s %s\n", $k, $v;
    }
    printf "graph_order %s\n", (join " ", sort @Values);
    foreach my $k (@Values) {
      if (1) {
        my $label = $k;
        if (exists $StaticData{label}{$k}) {
          $label = $StaticData{label}{$k};
        }
        printf "%s.label %s\n", $k, $label;
        if (exists $StaticData{label}{"$k.info"}) {
          $label = $StaticData{label}{"$k.info"};
        }
        printf "%s.info %s\n", $k, $label;
        printf "%s.type GAUGE\n", $k;
      }
    }
    exit 0;
  }

  if ($#ARGV < 0) {
    foreach my $k (@Values) {
      my $dbfile = "$WORKDIR/$k.db";
      $dbfile = "$CONSTDIR/$k.db" unless -f $dbfile;
      next unless -f $dbfile;
      my @DATA = `ze-makemap -c -b $dbfile`;
      chomp @DATA;
      my $nrec = 0;
      foreach my $line (@DATA) {
        if ($line =~ /\s+(\d+)\s+records/) {
          $nrec = $1;
          printf "%s.value %s\n", $k, $nrec;
          last;
        }
      }
    }
    exit 0;
  }
  exit 1;
}

# ------------------------------------------------
#
#
sub GetStatsLine {
  my ($h, undef) = @_;
  return undef unless -f $FSTATS;
  my $line = `tail -1 $FSTATS`;
  return undef unless defined $line && $line ne "";

  my @Values = split / /, $line;
  my $time = time();
  if ($Values[0] =~ /^\d+$/) {
    $time = shift @Values;
  }
  $h->{timestamp} = $time;
  foreach my $lx (@Values) {
    $lx = lc $lx;
    if ($lx =~ /(\S+)=\((\d+)\)/) {
      $h->{$1} = $2;
    }
  }
  return $line;
}

# ------------------------------------------------
#
#
sub ReadState {
  my ($h, undef) = @_;
  return 0 unless defined $StateFile && $StateFile ne "" && -f $StateFile;

  open FSTATE, "< $StateFile" || return 0;
  while (my $s = <FSTATE>) {
    chomp $s;
    next if $s =~ /^\s*$/;
    next if $s =~ /^\s*#/;
    my ($a, $b) = split " ", $s;
    $b = 0 unless defined $b;
    $h->{$a} = $b;
  }
  close FSTATE;
  return 1;
}

# ------------------------------------------------
#
#
sub SaveState {
  my ($h, undef) = @_;
  return 0 unless defined $StateFile && $StateFile ne "";

  open FSTATE, "> $StateFile" || return 0;
  foreach my $k (sort keys %{$h}) {
    printf FSTATE "%-20s %s\n", $k, $h->{$k};
  }
  close FSTATE;
  return 1;
}

# ------------------------------------------------
#
#
sub DiffFromState {
  my ($old, $new, $delta, undef) = @_;
  return 0 unless defined $old && defined $new;
  $delta = $new unless defined $delta;

  return 0 unless exists $old->{timestamp};
  return 0 unless exists $new->{timestamp};
  my $dt = $new->{timestamp} - $old->{timestamp};
  return 0 unless $dt > 60;

  $delta->{timestamp} = $new->{timestamp};
  foreach my $k (sort keys %{$new}) {
    next if $k eq "timestamp";
    if (exists $old->{$k}) {
      $delta->{$k} = $new->{$k} - $old->{$k};
      $delta->{$k} *= 300. / $dt;
    } else {
      $delta->{$k} = 0.;
    }
  }
  return 1;
}

# ------------------------------------------------
#
#
sub GetStaticData {
  my ($h, undef) = @_;
  my @DATA = <DATA>;
  my @SOPT = qw();
  chomp @DATA;
  my $in  = 0;
  my $key = "";
  foreach my $s (@DATA) {
    if ($key eq "") {
      if ($s =~ /==\s+BEGIN\s+(\S+)\s+==/) {
        $key = $1;
      }
      next;
    }
    if ($s =~ /==\s+END\s+(\S*\s+)?==/) {
      $key = "";
      next;
    }
    next if $s =~ /^\s*$/;
    push @{$h->{$key}{_data_}}, $s;
    my ($a, $b) = split " ", $s, 2;
    next unless defined $a && $a ne "";
    $b = "" unless defined $b;
    $h->{$key}{$a} = $b;
  }
}

# ------------------------------------------------
#
#
sub EvalBoolean {
  my ($v, undef) = @_;

  # return defined $v && $v =~ /^(1|yes|true|oui|vrai)$/i;
  if (defined $v) {
    return 1 if $v =~ /^(1|yes|true|oui|vrai)$/i;
  }
  return 0;
}

# ------------------------------------------------
#
#
__DATA__

== BEGIN label ==
conn                      Connections
conn.info                 SMTP Connections per time unit
msgs                      Messages
msgs.info                 Messages per time unit 
rcpt                      Recipients
rcpt.info                 Recipients per time unit

greymsgs                  Greylisted Messages
greymsgs.info             Greylisted Messages per time unit
greyrcpt                  Greylisted Recipients
greyrcpt.info             Greylisted Recipients per time unit

bayesham                  Legitimate (Stat Filter)
bayesham.info             Legitimate Messages - Statistical Filter
bayesspam                 Unwanted (Stat Filter)
bayesspam.info            Unwanted Messages - Statistical Filter

matching                  Pattern Matching
matching.info             Unwanted Messages - Pattern Matching Filter
urlbl                     URL Blacklist
urlbl.info                Unwanted Messages - URL Blacklist

throttle                  Connection Rate
throttle.info             Connections rejected by connection rate
badrcpt                   Bad Recipients
badrcpt.info              Connections rejected - excessive bad recipients
localuser                 Internal addresses
localuser.info            Messages sent to internal use only addresses
badmx                     Bad Sender MX
badmx.info                Messages rejected - Bad Sender MX
rcptrate                  Recipient Rate
rcptrate.info             Messages rejected by recipient rate

files                     Files
files.info                Files
xfiles                    X-Files
xfiles.info               X-Files

kbytes                    Volume (KBytes)
kbytes.info               Volume (KBytes) per time unit

ze-greyvalid               Grey Validated records
ze-greyvalid.info          Grey Validated records
ze-greypend                Grey Waiting records 
ze-greypend.info           Grey Waiting records 
ze-greywhitelist           Grey Whitelisted records
ze-greywhitelist.info      Grey Whitelisted records
ze-greyblacklist           ze-greyblacklist
ze-greyblacklist.info      ze-greyblacklist

ratiospamham              Ratio Messages
ratiospamham.info         Ratio Messages
ratiospamham00            Legitimate messages
ratiospamham00.info       Legitimate messages
ratiospamham01            Unwanted messages
ratiospamham01.info       Unwanted messages

== END label ==


== BEGIN config ==
activity-graph_title        ze-filter - Filter activity
activity-graph_category     ze-filter
activity-graph_vlabel       counts per second
activity-graph_scale        no
activity+data               conn msgs rcpt
activity+type               counters
activity+enable             yes

greylisting-graph_title     ze-filter - Greylisting activity
greylisting-graph_category  ze-filter
greylisting-graph_vlabel    counts per second
greylisting-graph_scale     no
greylisting+data            greymsgs greyrcpt
greylisting+type            counters
greylisting+enable          yes

statfilter-graph_title      ze-filter - Statistical classification
statfilter-graph_category   ze-filter
statfilter-graph_vlabel     counts per second
statfilter-graph_scale      no
statfilter+data             bayesham bayesspam
statfilter+type             counters
statfilter+enable           yes

contentfilter-graph_title      ze-filter - Content Filtering
contentfilter-graph_category   ze-filter
contentfilter-graph_vlabel     counts per second
contentfilter-graph_scale      no
contentfilter+data             bayesspam matching urlbl
contentfilter+type             counters
contentfilter+enable           yes

behaviour-graph_title       ze-filter - Behaviour filtering - rejections
behaviour-graph_category    ze-filter
behaviour-graph_vlabel      counts per second
behaviour-graph_scale       no
behaviour+data              throttle badrcpt localuser badmx rcptrate
behaviour+type              counters
behaviour+enable            yes

xfiles-graph_title          ze-filter - Suspected files filtering
xfiles-graph_category       ze-filter
xfiles-graph_vlabel         counts per second
xfiles-graph_scale          no
xfiles+data                 files xfiles
xfiles+type                 counters
xfiles+enable               yes

volume-graph_title          ze-filter - Volume handled by the filter
volume-graph_category       ze-filter
volume-graph_vlabel         KBytes per second
volume-graph_scale          no
volume+data                 kbytes
volume+type                 counters
volume+enable               yes

ratiospamham-graph_title       ze-filter - Statistical classification
ratiospamham-graph_category    ze-filter
ratiospamham-graph_vlabel      Ratio (%)
ratiospamham-graph_scale       yes
ratiospamham-graph_args        --lower-limit 0 --upper-limit 100 --rigid
ratiospamham+data              bayesham, bayesspam / bayesham bayesspam
ratiospamham+type              ratio
ratiospamham+state             yes
ratiospamham+enable            yes

greydb-graph_title             ze-filter - Greylisting Database Size
greydb-graph_category          ze-filter
greydb-graph_vlabel            records
greydb-graph_scale             no
greydb-graph_args              --lower-limit 0
greydb+data                    ze-greypend ze-greyvalid ze-greywhitelist
greydb+type                    dbcounters
greydb+enable                  yes

cdb-graph_title                ze-filter - Static databases
cdb-graph_category             ze-filter
cdb-graph_vlabel               records
cdb-graph_scale                no
cdb-graph_args                 --lower-limit 0
cdb+data                       ze-policy ze-urlbl ze-bayes ze-rcpt
cdb+type                       dbcounters
cdb+enable                     yes

== END config ==