ze-filter  (ze-filter-0.8.0-develop-180218)
ze-oracle-scores.c
Go to the documentation of this file.
1 
2 /*
3  *
4  * ze-filter - Mail Server Filter for sendmail
5  *
6  * Copyright (c) 2001-2018 - Jose-Marcio Martins da Cruz
7  *
8  * Auteur : Jose Marcio Martins da Cruz
9  * jose.marcio.mc@gmail.org
10  *
11  * Historique :
12  * Creation : Mon Apr 24 22:00:19 CEST 2006
13  *
14  * This program is free software, but with restricted license :
15  *
16  *
17  * This program is distributed in the hope that it will be useful,
18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
20  *
21  * More details about ze-filter license can be found at ze-filter
22  * web site : http://foss.jose-marcio.org
23  */
24 
25 
26 #include <ze-sys.h>
27 #include <ze-filter.h>
28 #include <ze-oracle-scores.h>
29 
30 
31 /* ****************************************************************************
32  * *
33  * *
34  **************************************************************************** */
35 typedef struct {
36  int type;
37  int ind;
38  double score;
39  double pOdds;
40  double nOdds;
41  long count;
42  bool enable;
43  char *msg;
45 
46 static oracle_message_T oracle_messages[] = {
47  /*
48  ** CONNECTION CHECKS
49  */
50  {
53  0.4, 0.0, 0.0, 0, TRUE,
54  "SMTP client resolve failed"},
55  {
58  0.4, 0.0, 0.0, 0, TRUE,
59  "SMTP client resolve forged"},
60  {
63  0.5, 0.0, 0.0, 0, TRUE,
64  "SMTP client resolve tempfail"},
65  {
68  1.2, 0.0, 0.0, 0, TRUE,
69  "False localhost DNS declaration"},
70  {
73  1.2, 0.0, 0.0, 0, TRUE,
74  "SMTP client sending mail to spamtrap"},
75  {
78  1.2, 0.0, 0.0, 0, TRUE,
79  "Bad EHLO parameter"},
80  {
83  1.2, 0.0, 0.0, 0, TRUE,
84  "Myself EHLO parameter - forged"},
85  {
88  1.2, 0.0, 0.0, 0, TRUE,
89  "SMTP client listed at some RBL"},
90 
91  /*
92  ** MSG CHECKS
93  */
96  0.5, 0.0, 0.0, 0, TRUE,
97  "No HTML nor TEXT parts"},
98  {
101  0.2, 0.0, 0.0, 0, TRUE, /* 0.5 -> 0.2 14/02/2005 */
102  "text/html without text/plain"},
103  {
106  0.4, 50.0, 0.0, 0, TRUE,
107  "bad date"},
108  {
111  0.4, 50.0, 0.0, 0, TRUE,
112  "date in the future"},
113  {
116  0.4, 1.0, 0.0, 0, TRUE,
117  "too old message"},
118  {
121  0.4, 0.0, 0.0, 0, TRUE,
122  "unwanted charset"},
123  {
126  1., 0.0, 0.0, 0, TRUE, /* XXX */
127  "BAD EXPRESSIONS"},
128  {
131  1., 0.0, 0.0, 0, TRUE,
132  "Forged postmaster"},
133  {
136  1., 0.0, 0.0, 0, TRUE,
137  "Invalid sender address"},
138  {
141  1., 0.0, 0.0, 0, TRUE,
142  "Invalid domain address"},
143  {
146  1., 0.0, 0.0, 0, TRUE,
147  "No Subject header"},
148  {
151  1., 0.0, 0.0, 0, TRUE,
152  "RFC2822 headers compliance"},
153  {
156  0.5, 0.0, 0.0, 0, TRUE,
157  "Header syntax"},
158  {
161  1., 0.0, 0.0, 0, TRUE,
162  "Base 64 encoded message"},
163  {
166  0.5, 0.0, 0.0, 0, TRUE,
167  "Base 64 encoded Subject"},
168  {
171  1., 0.0, 0.0, 0, TRUE, /* XXX */
172  "multipart/* unwanted boundary"},
173  {
176  1.2, 0.0, 0.0, 0, TRUE,
177  "message with bad recipients"},
178  {
181  1., 0.0, 0.0, 0, TRUE,
182  "MIME errors"},
183  {
186  1., 0.0, 0.0, 0, TRUE,
187  "unwanted mailer"},
188  {
191  0.5, 0.0, 0.0, 0, TRUE,
192  "text text/html parts don't match"},
193  {
196  2., 0.0, 0.0, 0, TRUE,
197  "message with spamtrap recipient"},
198  {
201  1., 0.0, 0.0, 0, TRUE, /* XXX */
202  "message too short"},
203  {
206  2., 0.0, 0.0, 0, TRUE,
207  "bad NULL sender"},
208  {
211  0.5, 0.0, 0.0, 0, TRUE,
212  "HI CAPS Subject"},
213  {
216  2., 0.0, 0.0, 0, TRUE,
217  "Unwanted MIME part with Content-ID"},
218  {
221  2., 0.0, 0.0, 0, TRUE,
222  "Message with an empty attachment"},
223  {
226  0.5, 0.0, 0.0, 0, TRUE,
227  "No alpha Subject"},
228 
229  /*
230  ** TEXT/PLAIN CHECKS
231  */
232  {
235  1., 0.0, 0.0, 0, TRUE,
236  "text/plain empty"},
237  {
240  0.5, 0.0, 0.0, 0, TRUE,
241  "text/plain encoded base64"},
242  {
245  0.2, 0., 0.0, 0, TRUE,
246  "text/plain wo charset"},
247  {
250  0.2, 0.0, 0.0, 0, TRUE,
251  "text/plain too short"},
252 
253  /*
254  ** TEXT/HTML CHECKS
255  */
256  {
259  0.8, 0.0, 0.0, 0, TRUE,
260  "cleaned HTML part too short"},
261  {
264  0.5, 0.0, 0.0, 0, TRUE,
265  "text/html encoded base64"},
266  {
268  SPAM_HTML_UNWANTED_TAGS, /* XXX */
269  0.5, 0.0, 0.0, 0, TRUE, /* 1.0 -> 0.5 x nb of tags 28/09/2005 */
270  "HTML with unwanted tags"},
271  {
274  0.5, 0.0, 0.0, 0, TRUE, /* 1.0 -> 0.5 14/02/2005 */
275  "HTML tag/text ratio"},
276  {
279  0.3, 0.0, 0.0, 0, TRUE,
280  "invalid HTML tags"},
281 
282  {
285  0.0, 0.0, 0.0, 0, TRUE,
286  "number of tagged messages"},
287 
288  {-1, -1, 0., 0.0, 0.0, 0, TRUE, NULL}
289 };
290 
291 /* ****************************************************************************
292  * *
293  * *
294  **************************************************************************** */
295 static pthread_mutex_t ora_mutex = PTHREAD_MUTEX_INITIALIZER;
296 
297 #define ORACLE_DATA_LOCK() MUTEX_LOCK(&ora_mutex)
298 #define ORACLE_DATA_UNLOCK() MUTEX_UNLOCK(&ora_mutex)
299 
300 static kstats_T st_ora_global = KSTATS_INITIALIZER;
301 static kstats_T st_ora_tagged = KSTATS_INITIALIZER;
302 static long nb_tag = 0;
303 static long nb_tot = 0;
304 
305 /* ****************************************************************************
306  * *
307  * *
308  **************************************************************************** */
309 void
311  int score;
312 {
314  zeKStatsUpdate(&st_ora_global, (double) score);
315  nb_tot++;
316  if (score > 0) {
317  zeKStatsUpdate(&st_ora_tagged, (double) score);
318  nb_tag++;
319  }
321 }
322 
323 /* ****************************************************************************
324  * *
325  * *
326  **************************************************************************** */
327 void
328 oracle_stats_get(gm, gs, gnb, tm, ts, tnb)
329  double *gm;
330  double *gs;
331  long *gnb;
332  double *tm;
333  double *ts;
334  long *tnb;
335 {
337  if (gm != NULL)
338  *gm = zeKMean(&st_ora_global);
339  if (gs != 0)
340  *gs = zeKStdDev(&st_ora_global);
341  if (gnb != NULL)
342  *gnb = nb_tot;
343 
344  if (tm != NULL)
345  *tm = zeKMean(&st_ora_tagged);
346  if (ts != 0)
347  *ts = zeKStdDev(&st_ora_tagged);
348  if (tnb != NULL)
349  *tnb = nb_tag;
350 
352 }
353 
354 
355 /* ****************************************************************************
356  * *
357  * *
358  **************************************************************************** */
359 char *
361  int type;
362  int ind;
363 {
364  oracle_message_T *p = oracle_messages;
365 
366  while (p->type >= 0) {
367  if ((p->type == type) && (p->ind == ind))
368  return p->msg;
369  p++;
370  }
371  return "";
372 }
373 
374 /* ****************************************************************************
375  * *
376  * *
377  **************************************************************************** */
378 double
380  int type;
381  int ind;
382 {
383  double odds = 0.;
384  oracle_message_T *p = oracle_messages;
385 
387  while (p->type >= 0) {
388  if ((p->type == type) && (p->ind == ind)) {
389  odds = p->pOdds;
390  break;
391  }
392  p++;
393  }
395 
396  return odds;
397 }
398 
399 /* ****************************************************************************
400  * *
401  * *
402  **************************************************************************** */
403 bool
404 oracle_set_podds(type, ind, value)
405  int type;
406  int ind;
407  double value;
408 {
409  bool result = FALSE;
410  oracle_message_T *p = oracle_messages;
411 
413  while (p->type >= 0) {
414  if ((p->type == type) && (p->ind == ind)) {
415  p->pOdds = value;
416  result = TRUE;
417  break;
418  }
419  p++;
420  }
422 
423  return result;
424 }
425 
426 /* ****************************************************************************
427  * *
428  * *
429  **************************************************************************** */
430 double
432  int type;
433  int ind;
434 {
435  double odds = 0.;
436  oracle_message_T *p = oracle_messages;
437 
439  while (p->type >= 0) {
440  if ((p->type == type) && (p->ind == ind)) {
441  odds = p->nOdds;
442  break;
443  }
444  p++;
445  }
447 
448  return odds;
449 }
450 
451 /* ****************************************************************************
452  * *
453  * *
454  **************************************************************************** */
455 bool
456 oracle_set_nodds(type, ind, value)
457  int type;
458  int ind;
459  double value;
460 {
461  bool result = FALSE;
462  oracle_message_T *p = oracle_messages;
463 
465  while (p->type >= 0) {
466  if ((p->type == type) && (p->ind == ind)) {
467  p->nOdds = value;
468  result = TRUE;
469  break;
470  }
471  p++;
472  }
474 
475  return result;
476 }
477 
478 /* ****************************************************************************
479  * *
480  * *
481  **************************************************************************** */
482 double
484  int type;
485  int ind;
486 {
487  double score = 0.;
488  oracle_message_T *p = oracle_messages;
489 
491  while (p->type >= 0) {
492  if ((p->type == type) && (p->ind == ind)) {
493  score = p->score;
494  break;
495  }
496  p++;
497  }
499 
500  return score;
501 }
502 
503 /* ****************************************************************************
504  * *
505  * *
506  **************************************************************************** */
507 bool
508 oracle_set_score(type, ind, value)
509  int type;
510  int ind;
511  double value;
512 {
513  bool result = FALSE;
514  oracle_message_T *p = oracle_messages;
515 
517  while (p->type >= 0) {
518  if ((p->type == type) && (p->ind == ind)) {
519  p->score = value;
520  result = TRUE;
521  break;
522  }
523  p++;
524  }
526 
527  return result;
528 }
529 
530 /* ****************************************************************************
531  * *
532  * *
533  **************************************************************************** */
534 
535 long
537  int type;
538  int ind;
539 {
540  long count = 0;
541  oracle_message_T *p = oracle_messages;
542 
544  while (p->type >= 0) {
545  if ((p->type == type) && (p->ind == ind)) {
546  count = p->count;
547  break;
548  }
549  p++;
550  }
552 
553  return count;
554 }
555 
556 /* ****************************************************************************
557  * *
558  * *
559  **************************************************************************** */
560 long
561 oracle_set_count(type, ind, value)
562  int type;
563  int ind;
564  long value;
565 {
566  long count = 0;
567  oracle_message_T *p = oracle_messages;
568 
570  while (p->type >= 0) {
571  if ((p->type == type) && (p->ind == ind)) {
572  count = p->count = value;
573  break;
574  }
575  p++;
576  }
578 
579  return count;
580 }
581 
582 /* ****************************************************************************
583  * *
584  * *
585  **************************************************************************** */
586 long
588  int type;
589  int ind;
590 {
591  long count = 0;
592  oracle_message_T *p = oracle_messages;
593 
595  while (p->type >= 0) {
596  if ((p->type == type) && (p->ind == ind)) {
597  count = ++p->count;
598  break;
599  }
600  p++;
601  }
603 
604  return count;
605 }
606 
607 /* ****************************************************************************
608  * *
609  * *
610  **************************************************************************** */
611 #define MINCNT 0
612 
613 bool
615  int fd;
616  bool verbose;
617 {
618 #if 0
619  int type;
620  int ind;
621  oracle_message_T *p = oracle_messages;
622 #endif
623  char *p;
624 
625  bool CloseOnQuit = (fd < 0);
626 
627  if (fd < 0) {
628  char fname[256];
629 
630  if (((p = cf_get_str(CF_ORACLE_STATS_FILE)) == NULL) || (strlen(p) == 0))
631  return TRUE;
632 
633  if (p[0] == '/')
634  strlcpy(fname, p, sizeof (fname));
635  else
636  snprintf(fname, sizeof (fname), "%s/%s", cf_get_str(CF_WORKDIR), p);
637 
638  if ((fd = open(fname, O_WRONLY | O_TRUNC | O_CREAT, 0644)) < 0) {
639  ZE_LogSysError("Error opening %s file ", fname);
640  return FALSE;
641  }
642  }
643 
644  if (fd >= 0) {
645  char s[256];
646  struct tm tm;
647  time_t now = time(NULL);
648  int i;
649  long nb;
650  long total = 0;
651  char *label;
652 
653  if (localtime_r(&now, &tm) != NULL)
654  strftime(s, sizeof (s), "%b %e %T", &tm);
655  else
656  memset(s, 0, sizeof (s));
657 
658  FD_PRINTF(fd, "++++++++++++++++ %ld %s\n", (long) now, s);
659 
660  /*
661  **
662  */
663  for (i = 0; i < SPAM_CONN_NB; i++) {
665  if (verbose)
667  else
668  label = "";
669 
670  if (nb <= MINCNT)
671  continue;
672 
673  total += nb;
674 
675  FD_PRINTF(fd, "C%02d %6ld - %5.2f %s\n", i, nb,
676  oracle_get_score(ORACLE_TYPE_CONN, i), STRNULL(label, ""));
677  }
678 
679  /*
680  **
681  */
682  for (i = 0; i < SPAM_MSG_NB; i++) {
684  if (verbose)
685  label = oracle_get_label(ORACLE_TYPE_MSG, i);
686  else
687  label = "";
688 
689  if (nb <= MINCNT)
690  continue;
691 
692  total += nb;
693 
694  FD_PRINTF(fd, "M%02d %6ld - %5.2f %s\n", i, nb,
695  oracle_get_score(ORACLE_TYPE_MSG, i), STRNULL(label, ""));
696  }
697 
698  /*
699  **
700  */
701  for (i = 0; i < SPAM_PLAIN_NB; i++) {
703  if (verbose)
705  else
706  label = "";
707 
708  if (nb <= MINCNT)
709  continue;
710 
711  total += nb;
712 
713  FD_PRINTF(fd, "P%02d %6ld - %5.2f %s\n", i, nb,
714  oracle_get_score(ORACLE_TYPE_PLAIN, i), STRNULL(label, ""));
715  }
716 
717  /*
718  **
719  */
720  for (i = 0; i < SPAM_HTML_NB; i++) {
722  if (verbose)
724  else
725  label = "";
726 
727  if (nb <= MINCNT)
728  continue;
729 
730  total += nb;
731 
732  FD_PRINTF(fd, "H%02d %6ld - %5.2f %s\n", i, nb,
733  oracle_get_score(ORACLE_TYPE_HTML, i), STRNULL(label, ""));
734  }
735 
736  if (total > 0)
737  FD_PRINTF(fd, "TOTAL %6ld\n", total);
738 
739  {
740  double gm, gs, tm, ts;
741  long gnb, tnb;
742 
743  oracle_stats_get(&gm, &gs, &gnb, &tm, &ts, &tnb);
744 
745  FD_PRINTF(fd, "MSGS : TOTAL - NB %8ld - MEAN %6.3f - STDDEV %6.3f\n",
746  gnb, gm, gs);
747  FD_PRINTF(fd, "MSGS : TAGGED - NB %8ld - MEAN %6.3f - STDDEV %6.3f\n",
748  tnb, tm, ts);
749  }
750 
751  if (CloseOnQuit)
752  close(fd);
753  }
754 
755  return TRUE;
756 }
757 
758 /* ****************************************************************************
759  * *
760  * *
761  **************************************************************************** */
762 bool
764 {
765 #if 0
766  int type;
767  int ind;
768  oracle_message_T *p = oracle_messages;
769 
771 
773 #endif
774  return TRUE;
775 }
776 
777 /* ****************************************************************************
778  * *
779  * *
780  **************************************************************************** */
781 bool
783 {
784 #if 0
785  int type;
786  int ind;
787  oracle_message_T *p = oracle_messages;
788 
790 
792 #endif
793  return TRUE;
794 }
795 
796 /* ****************************************************************************
797  * *
798  * *
799  **************************************************************************** */
800 bool
802 {
803 #if 0
804  int type;
805  int ind;
806  oracle_message_T *p = oracle_messages;
807 
809 
811 #endif
812  return TRUE;
813 }
814 
815 /* ****************************************************************************
816  * *
817  * *
818  **************************************************************************** */
819 #define KEYVALUE "^[a-z0-9_-]+=[^ \t]+"
820 
821 static int
822 read_oracle_def_string(v, arg)
823  void *v;
824  void *arg;
825 {
826  char *s = (char *) v;
827  int type = -1;
828  int ind = -1;
829  bool enable = TRUE;
830  double score = -1.;
831  double odds = 0.;
832 
833  switch (tolower(*s)) {
834  case 'c':
835  type = ORACLE_TYPE_CONN;
836  break;
837  case 'm':
838  type = ORACLE_TYPE_MSG;
839  break;
840  case 'p':
841  type = ORACLE_TYPE_PLAIN;
842  break;
843  case 'h':
844  type = ORACLE_TYPE_HTML;
845  break;
846  default:
847  type = -1;
848  }
849  if (type < 0)
850  return 0;
851 
852  s++;
853  if (!isdigit(*s))
854  return 0;
855  ind = atoi(s);
856  SKIP_DIGITS(s);
857  if (!isspace(*s))
858  return 0;
859 
860  enable = FALSE;
861 
862  SKIP_SPACES(s);
863 
864  while (strlen(s) > 0) {
865  long pi, pf;
866 
867  if (strncasecmp(s, "ENABLE", strlen("ENABLE")) == 0) {
868  enable = TRUE;
869  SKIP_ALPHAS(s);
870  SKIP_SPACES(s);
871  continue;
872  }
873 
874  if (strncasecmp(s, "DISABLE", strlen("DISABLE")) == 0) {
875  enable = FALSE;
876  SKIP_ALPHAS(s);
877  SKIP_SPACES(s);
878  continue;
879  }
880 
881  while (zeStrRegex(s, KEYVALUE, &pi, &pf, TRUE)) {
882  char *key, *val;
883 
884  if (pi != 0)
885  break;
886 
887  key = s;
888  val = strchr(s, '=');
889  *val++ = '\0';
890 
891  s += pf;
892  *s++ = '\0';
893 
894  ZE_MessageInfo(19, "KEY = (%s) VALUE = (%s)\n", key, val);
895 
896  if (STRCASEEQUAL(key, "score")) {
897  if (strspn(val, "0123456789.") == strlen(val))
898  score = atof(val);
899  else
900  ZE_MessageWarning(9, "Non numeric value found... %s=%s", key, val);
901  }
902 
903  if (STRCASEEQUAL(key, "odds")) {
904  if (strspn(val, "0123456789.") == strlen(val)) {
905  double v;
906 
907  v = atof(val);
908  if (v > 0.)
909  odds = log(v);
910  } else
911  ZE_MessageWarning(9, "Non numeric value found... %s=%s", key, val);
912  }
913 #if 0
914  if (STRCASEEQUAL(key, "action")) {
915  strlcpy(r.action, val, sizeof (r.action));
916  }
917 #endif
918 
919  SKIP_SPACES(s);
920  continue;
921  }
922 
923  break;
924  }
925 
926  {
927  oracle_message_T *p = oracle_messages;
928 
930 
931  while (p->type >= 0) {
932  if ((p->type == type) && (p->ind == ind)) {
933  if (score >= 0)
934  p->score = score;
935  /*
936  * if (odds != 0.)
937  */
938  p->pOdds = odds;
939  p->enable = enable;
940  /*
941  * result = TRUE;
942  */
943  break;
944  }
945  p++;
946  }
947 
949  }
950 
951  return 0;
952 }
953 
954 static bool
955 read_it(path, tag)
956  char *path;
957  char *tag;
958 {
959  int r;
960 
961  r = zm_RdFile(path, tag, read_oracle_def_string, NULL);
962 
963  return r >= 0;
964 }
965 
966 bool
967 load_oracle_defs(cfdir, fname)
968  char *cfdir;
969  char *fname;
970 {
971  bool result;
972 
973  ASSERT(fname != NULL);
974 
975  result =
976  read_conf_data_file(cfdir, fname, "ze-oracle:oracle-scores", read_it);
977 
978  return result;
979 }
980 
981 /* ****************************************************************************
982  * *
983  * *
984  **************************************************************************** */
985 void
987 {
988  oracle_message_T *q = oracle_messages;
989  char *prefix = "RCMHP.";
990  int otype = -1;
991 
992  printf("\n");
993  printf("<ORACLE-SCORES>\n");
994  for (q = oracle_messages; q->type >= 0; q++) {
995  char c = '.';
996  char sa[16];
997 
998  if (otype != q->type)
999  printf("\n");
1000  otype = q->type;
1001 
1002  if (q->type >= 0 && q->type < strlen(prefix))
1003  c = prefix[q->type];
1004 
1005  if (c == '.')
1006  continue;
1007 
1008  memset(sa, 0, sizeof (sa));
1009  snprintf(sa, sizeof (sa), "odds=%.3f", exp(q->pOdds));
1010  printf("%c%02d %-8s %-15s %s\n", c, q->ind,
1011  STRBOOL(q->enable, "ENABLE", "DISABLE"), sa, q->msg);
1012  }
1013  printf("</ORACLE-SCORES>\n");
1014  printf("\n");
1015 }
1016 
1017 /* ****************************************************************************
1018  * *
1019  * *
1020  **************************************************************************** */
1021 bool
1023  int type;
1024  int ind;
1025 {
1026  bool enable = FALSE;
1027 
1028  oracle_message_T *p = oracle_messages;
1029 
1030  /*
1031  * XXX shall this loop be locked ???
1032  */
1033  ORACLE_DATA_LOCK();
1034  while (p->type >= 0) {
1035  if ((p->type == type) && (p->ind == ind)) {
1036  enable = p->enable;
1037  break;
1038  }
1039  p++;
1040  }
1042 
1043  return enable;
1044 }
1045 
1046 /* ****************************************************************************
1047  * *
1048  * *
1049  **************************************************************************** */
1050 int
1052  char *id;
1053  char *ip;
1054  spamchk_T *data;
1055 {
1056  int i;
1057  char *msg = NULL;
1058  double value = 0., score = 0., v;
1059 
1060  double odds;
1061 
1062  if (data == NULL)
1063  return 0;
1064 
1065  id = STRNULL(id, "00000000.000");
1066  ip = STRNULL(ip, "0.0.0.0");
1067 
1068  bestof_init(&data->best, 5, NULL);
1069 
1070  for (i = 0; i < (8 * sizeof (uint32_t)); i++) {
1072  if (GET_BIT(data->flags.conn, i)) {
1074  bestof_add(&data->best, odds);
1075 
1076  value = oracle_get_score(ORACLE_TYPE_CONN, i);
1077  if (TRUE || value > 0) {
1079  v = 1.;
1080  switch (i) {
1082  break;
1084  break;
1086  break;
1087  case SPAM_CONN_BL_SPAMTRAP:
1088  break;
1090  break;
1091  case SPAM_CONN_BAD_EHLO:
1092  break;
1093  case SPAM_CONN_FORGED_EHLO:
1094  break;
1095  case SPAM_CONN_RBL:
1096  break;
1097  }
1099  if (cf_get_int(CF_LOG_LEVEL_ORACLE) >= 1)
1100  ZE_MessageInfo(9, "%s ORACLE - C%02d %s (%6.1f)", id, i, msg,
1101  value * v);
1102  score += value * v;
1103  }
1104  } else {
1106  bestof_add(&data->best, odds);
1107  }
1108  }
1109  }
1110 
1111  for (i = 0; i < (8 * sizeof (uint32_t)); i++) {
1113  if (GET_BIT(data->flags.msg, i)) {
1114  odds = oracle_get_podds(ORACLE_TYPE_MSG, i);
1115  bestof_add(&data->best, odds);
1116 
1117  value = oracle_get_score(ORACLE_TYPE_MSG, i);
1118  if (TRUE || value > 0) {
1120  v = 1.;
1121  switch (i) {
1122  case SPAM_MSG_NO_TEXT_PART:
1123  break;
1125  break;
1126  case SPAM_MSG_BAD_DATE:
1127  break;
1128  case SPAM_MSG_FUTURE_DATE:
1129  break;
1130  case SPAM_MSG_TOO_OLD_DATE:
1131  break;
1133  break;
1135  v = data->msg_bad_expressions;
1136  break;
1138  break;
1140  break;
1142  break;
1143  case SPAM_MSG_NO_SUBJECT:
1144  break;
1146  v = data->nb_rfc2822_hdrs_errors;
1147  break;
1149  v = data->headers_syntax_errors;
1150  break;
1151  case SPAM_MSG_BASE64:
1152  break;
1154  break;
1156  break;
1157  case SPAM_MSG_HAS_BADRCPT:
1158  v = data->nb_badrcpt;
1159  if (v > 4.)
1160  v = 4.;
1161  break;
1162  case SPAM_MSG_MIME_ERRORS:
1163  v = (double) data->mime_errors;
1164  break;
1166  break;
1168  /*
1169  * v = data->nb_diff_html_plain;
1170  */
1171  break;
1172  case SPAM_MSG_HAS_SPAMTRAP:
1173  break;
1174  case SPAM_MSG_TOO_SHORT:
1175  break;
1177  break;
1179  break;
1180  case SPAM_MSG_CONTENT_ID:
1181  break;
1183  break;
1185  break;
1186  }
1188  if (cf_get_int(CF_LOG_LEVEL_ORACLE) >= 1)
1189  ZE_MessageInfo(9, "%s ORACLE - M%02d %s (%6.1f)", id, i, msg,
1190  value * v);
1191  score += value * v;
1192  }
1193  } else {
1194  odds = oracle_get_nodds(ORACLE_TYPE_MSG, i);
1195  bestof_add(&data->best, odds);
1196  }
1197  }
1198  }
1199 
1200  for (i = 0; i < (8 * sizeof (uint32_t)); i++) {
1202  if (GET_BIT(data->flags.plain, i)) {
1204  bestof_add(&data->best, odds);
1205 
1206  value = oracle_get_score(ORACLE_TYPE_PLAIN, i);
1207  if (TRUE || value > 0) {
1209  v = 1.;
1210  switch (i) {
1211  case SPAM_PLAIN_EMPTY:
1212  break;
1213  case SPAM_PLAIN_BASE64:
1214  break;
1215  case SPAM_PLAIN_NO_CHARSET:
1216  break;
1217  case SPAM_PLAIN_TOO_SHORT:
1218  break;
1219  }
1221  if (cf_get_int(CF_LOG_LEVEL_ORACLE) >= 1)
1222  ZE_MessageInfo(9, "%s ORACLE - P%02d %s (%6.1f)", id, i, msg,
1223  value * v);
1224  score += value * v;
1225  }
1226  } else {
1228  bestof_add(&data->best, odds);
1229  }
1230  }
1231  }
1232 
1233  for (i = 0; i < (8 * sizeof (uint32_t)); i++) {
1235  if (GET_BIT(data->flags.html, i)) {
1237  bestof_add(&data->best, odds);
1238 
1239  value = oracle_get_score(ORACLE_TYPE_HTML, i);
1240  if (TRUE || value > 0) {
1242  v = 1.;
1243  switch (i) {
1245  break;
1246  case SPAM_HTML_BASE64:
1247  break;
1249  v = data->html_unwanted_tags;
1250  break;
1251  case SPAM_HTML_TAGS_RATIO:
1252  break;
1254  v = data->html_invalid_tags;
1255  break;
1256  }
1258  if (cf_get_int(CF_LOG_LEVEL_ORACLE) >= 1)
1259  ZE_MessageInfo(9, "%s ORACLE - H%02d %s (%6.1f)", id, i, msg,
1260  value * v);
1261  score += value * v;
1262  }
1263  } else {
1265  bestof_add(&data->best, odds);
1266  }
1267  }
1268  }
1269 
1270  ZE_MessageInfo(12, "%s ->Computed ORACLE score is %5.2f ...", id, score);
1271 
1272  {
1273  double lam = 0.0;
1274 
1275  odds = bestof_average(&data->best);
1276  lam = 1. / (1 + exp(-odds));
1277 
1278  data->scores.noracle = lam;
1279 
1280  ZE_MessageInfo(12,
1281  "%s ->Computed ORACLE score is %5.2f odds=%6.3f lam=%6.3f...",
1282  id, score, odds, lam);
1283  }
1284 
1285  return ((int) floor(score + 0.5));
1286 }
int msg_bad_expressions
bool oracle_save_counters()
#define SKIP_DIGITS(s)
Definition: macros.h:185
#define STRBOOL(x, t, f)
Definition: macros.h:87
bool oracle_set_nodds(int type, int ind, double value)
int zm_RdFile(char *fname, char *tag, RDFILE_F f, void *arg)
Definition: zeRdFile.c:185
#define SPAM_MSG_NO_SUBJECT
int html_unwanted_tags
#define KEYVALUE
#define ORACLE_TYPE_GLOB
#define ASSERT(a)
Definition: macros.h:27
#define SPAM_MSG_BASE64
bool bestof_init(bestof_T *b, int dim, bestcomp_F bcmp)
Definition: ze-bestof-n.c:51
#define SPAM_MSG_TOO_OLD_DATE
#define SPAM_CONN_RESOLVE_FAIL
#define CF_ORACLE_STATS_FILE
Definition: cfh-defs.h:127
#define SPAM_MSG_BAD_SENDER_ADDRESS
#define SPAM_CONN_FALSE_LOCALHOST
#define SPAM_MSG_HAS_SPAMTRAP
#define STRNULL(x, r)
Definition: macros.h:81
#define SPAM_HTML_UNWANTED_TAGS
#define SPAM_MSG_TOO_MUCH_HTML
bool oracle_read_counters()
uint32_t conn
Definition: ze-chkcontent.h:36
#define SKIP_SPACES(s)
Definition: macros.h:171
int oracle_compute_score(char *id, char *ip, spamchk_T *data)
double bestof_average(bestof_T *b)
Definition: ze-bestof-n.c:97
#define FALSE
Definition: macros.h:160
#define SPAM_PLAIN_TOO_SHORT
#define ORACLE_DATA_LOCK()
long oracle_get_count(int type, int ind)
#define strlcpy
Definition: zeString.h:32
#define SPAM_MSG_TOO_SHORT
bool zeStrRegex(char *, char *, long *, long *, bool)
Definition: zeStrings.c:544
#define MINCNT
#define SPAM_MSG_HAS_BADRCPT
bool oracle_set_score(int type, int ind, double value)
#define CF_LOG_LEVEL_ORACLE
Definition: cfh-defs.h:126
int headers_syntax_errors
#define GET_BIT(p, i)
Definition: macros.h:168
#define SPAM_MSG_SUBJECT_NO_ALPHA
#define FD_PRINTF(fdp,...)
Definition: macros.h:45
int cf_get_int(int id)
Definition: ze-cf.c:803
#define SPAM_CONN_NB
bool read_conf_data_file(char *cfdir, char *fname, char *dfile, read_conf_data_file_F func)
#define SPAM_MSG_UNWANTED_CHARSET
#define SPAM_MSG_FUTURE_DATE
#define strchr
Definition: ze-sys.h:218
#define SPAM_PLAIN_NO_CHARSET
#define SPAM_GLOB_TAGGED
#define SPAM_MSG_EMPTY_ATTACHMENT
#define SPAM_MSG_BAD_DATE
void oracle_stats_get(double *gm, double *gs, long *gnb, double *tm, double *ts, long *tnb)
#define SPAM_HTML_NB
bool bestof_add(bestof_T *b, double v)
Definition: ze-bestof-n.c:75
#define ORACLE_TYPE_HTML
double oracle_get_score(int type, int ind)
#define SPAM_CONN_RESOLVE_TEMPFAIL
long oracle_inc_count(int type, int ind)
msg_flags_T flags
Definition: ze-chkcontent.h:68
#define SPAM_HTML_BASE64
#define SPAM_MSG_HEADERS_SYNTAX
void zeKStatsUpdate(kstats_T *, double)
Definition: zeKStats.c:101
#define SPAM_MSG_BAD_DOMAIN_ADDRESS
bool load_oracle_defs(char *cfdir, char *fname)
#define ZE_MessageInfo(level,...)
Definition: zeSyslog.h:90
int nb
Definition: ze-connopen.c:61
#define SPAM_MSG_BAD_NULL_SENDER
#define SKIP_ALPHAS(s)
Definition: macros.h:178
#define SPAM_MSG_UNWANTED_BOUNDARY
#define TRUE
Definition: macros.h:157
#define ZE_MessageWarning(level,...)
Definition: zeSyslog.h:92
#define SPAM_CONN_BL_SPAMTRAP
#define SPAM_CONN_FORGED_EHLO
#define SPAM_PLAIN_NB
long oracle_set_count(int type, int ind, long value)
#define SPAM_MSG_BAD_EXPRESSIONS
#define SPAM_CONN_BAD_EHLO
void oracle_stats_update(int score)
#define ZE_LogSysError(...)
Definition: zeSyslog.h:129
#define KSTATS_INITIALIZER
Definition: zeKStats.h:36
void dump_oracle_defs()
double oracle_get_nodds(int type, int ind)
char * cf_get_str(int id)
Definition: ze-cf.c:854
#define SPAM_MSG_SUBJECT_HI_CAPS
#define SPAM_MSG_FORGED_POSTMASTER
char * oracle_get_label(int type, int ind)
uint32_t msg
Definition: ze-chkcontent.h:41
#define SPAM_CONN_RESOLVE_FORGED
#define SPAM_CONN_RBL
double noracle
Definition: ze-msg-score.h:85
#define SPAM_MSG_RFC2822_HEADERS
bestof_T best
Definition: ze-chkcontent.h:69
#define ORACLE_TYPE_PLAIN
#define SPAM_MSG_NB
uint32_t html
Definition: ze-chkcontent.h:38
#define SPAM_MSG_CONTENT_ID
#define SPAM_MSG_UNWANTED_MAILER
#define SPAM_PLAIN_EMPTY
#define STRCASEEQUAL(a, b)
Definition: macros.h:72
int html_invalid_tags
#define SPAM_MSG_MATCH_MIME_PARTS
double zeKMean(kstats_T *s)
Definition: zeKStats.c:43
bool oracle_dump_counters(int fd, bool verbose)
#define SPAM_PLAIN_BASE64
double oracle_get_podds(int type, int ind)
#define SPAM_HTML_TAGS_RATIO
double zeKStdDev(kstats_T *s)
Definition: zeKStats.c:53
msg_scores_T scores
Definition: ze-chkcontent.h:72
int nb_badrcpt
Definition: ze-chkcontent.h:75
#define ORACLE_DATA_UNLOCK()
#define ORACLE_TYPE_MSG
#define SPAM_MSG_MIME_ERRORS
bool oracle_check_enabled(int type, int ind)
#define CF_WORKDIR
Definition: cfh-defs.h:67
bool oracle_read_scores()
#define SPAM_HTML_INVALID_TAGS
long uint32_t
Definition: ze-sys.h:489
int nb_rfc2822_hdrs_errors
int msg[MAX_SCORE+2]
Definition: ze-stats.c:41
#define ORACLE_TYPE_CONN
#define SPAM_HTML_CLEAN_TOO_SHORT
uint32_t plain
Definition: ze-chkcontent.h:39
#define SPAM_MSG_BASE64_SUBJECT
bool oracle_set_podds(int type, int ind, double value)
#define SPAM_MSG_NO_TEXT_PART