ze-filter  (ze-filter-0.8.0-develop-180218)
ze-oracle.c
Go to the documentation of this file.
1 
2 /*
3  *
4  * ze-filter - Mail Server Filter for sendmail
5  *
6  * Copyright (c) 2001-2018 - Jose-Marcio Martins da Cruz
7  *
8  * Auteur : Jose Marcio Martins da Cruz
9  * jose.marcio.mc@gmail.org
10  *
11  * Historique :
12  * Creation : janvier 2002
13  *
14  * This program is free software, but with restricted license :
15  *
16  *
17  * This program is distributed in the hope that it will be useful,
18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
20  *
21  * More details about ze-filter license can be found at ze-filter
22  * web site : http://foss.jose-marcio.org
23  */
24 
25 #include <ze-sys.h>
26 
27 #include "ze-filter.h"
28 
29 /* ***************************************************************************
30  * *
31  * *
32  *****************************************************************************/
33 #define ORADATA_LEN 256
34 
35 typedef struct {
36  char type[32];
37  char data[ORADATA_LEN];
38  char action[32];
39  float score;
40  double pOdds;
41  double nOdds;
42 } oradata_T;
43 
44 static zeTbl_T htbl;
45 
46 static pthread_mutex_t st_mutex = PTHREAD_MUTEX_INITIALIZER;
47 
48 #if 1
49 #define ORA_LOCK() MUTEX_LOCK(&st_mutex)
50 #define ORA_UNLOCK() MUTEX_UNLOCK(&st_mutex)
51 #else
52 
53 #define ORA_LOCK() \
54  if (pthread_mutex_lock(&st_mutex) != 0) { \
55  ZE_LogSysError("pthread_mutex_lock(st_mutex)"); \
56  }
57 
58 #define ORA_UNLOCK() \
59  if (pthread_mutex_unlock(&st_mutex) != 0) { \
60  ZE_LogSysError("pthread_mutex_unlock(st_mutex)"); \
61  }
62 #endif
63 
64 /* ***************************************************************************
65  * *
66  * *
67  *****************************************************************************/
68 void
70 {
71  oradata_T p;
72 
73  printf("# Let's dump j_oradata_table : \n");
74  if (zeTable_Get_First(&htbl, &p) == 0) {
75  do {
76  char sodds[32];
77 
78  snprintf(sodds, sizeof (sodds), "odds=%.3f", exp(p.pOdds));
79 #if 1
80  printf("%-10s %-20s %s\n", p.type, sodds, p.data);
81 #else
82  printf("-> %-10s %-8s %8.2f : %s\n", p.type, p.action, (double) p.pOdds,
83  p.data);
84 #endif
85  } while (zeTable_Get_Next(&htbl, &p) == 0);
86  }
87 }
88 
89 /* ***************************************************************************
90  * *
91  * *
92  *****************************************************************************/
93 static int
94 oradata_comp(pa, pb)
95  const void *pa;
96  const void *pb;
97 {
98  oradata_T *a = (oradata_T *) pa;
99  oradata_T *b = (oradata_T *) pb;
100  int res;
101 
102  if ((res = strcasecmp(a->type, b->type)) != 0)
103  return res;
104 
105  return strcasecmp(a->data, b->data);
106 }
107 
108 /* ***************************************************************************
109  * *
110  * *
111  *****************************************************************************/
112 #define KEYVALUE "^[a-z0-9_-]+=[^ \t]+"
113 
114 static int
115 add_oradata_rec(vs, arg)
116  void *vs;
117  void *arg;
118 {
119  char *s = vs;
120 
121  oradata_T r;
122  char *v = NULL, *k = NULL;
123 
124  long pi, pf;
125 
126  ASSERT(s != NULL);
127 
128  memset(&r, 0, sizeof (r));
129  strlcpy(r.action, "score", sizeof (r.action));
130  r.score = 1.;
131  r.pOdds = 0.;
132  r.nOdds = 0.;
133 
134  SKIP_SPACES(s);
135 
136  v = s;
137  SKIP_KEYCHARS(s);
138  if (*s != '\0')
139  *s++ = '\0';
140 
141  SKIP_SPACES(s);
142 
143  while (zeStrRegex(s, KEYVALUE, &pi, &pf, TRUE)) {
144  char *key, *val;
145 
146  if (pi != 0)
147  break;
148 
149  key = s;
150  val = strchr(s, '=');
151  *val++ = '\0';
152 
153  s += pf;
154  *s++ = '\0';
155 
156  ZE_MessageInfo(11, "KEY = (%s) VALUE = (%s)\n", key, val);
157 
158  if (STRCASEEQUAL(key, "score")) {
159  strlcpy(r.action, key, sizeof (r.action));
160  if (strspn(val, "0123456789.") == strlen(val))
161  r.score = atof(val);
162  else
163  ZE_MessageWarning(9, "Non numeric value found... %s=%s", key, val);
164  }
165  if (STRCASEEQUAL(key, "odds")) {
166  strlcpy(r.action, key, sizeof (r.action));
167  if (strspn(val, "0123456789.") == strlen(val)) {
168  double v;
169 
170  v = atof(val);
171  if (v > 0.)
172  r.pOdds = log(v);
173  } else
174  ZE_MessageWarning(9, "Non numeric value found... %s=%s", key, val);
175  }
176  if (STRCASEEQUAL(key, "action")) {
177  strlcpy(r.action, val, sizeof (r.action));
178  }
179 
180  SKIP_SPACES(s);
181  }
182 
183  k = s;
184 
185  strlcpy(r.data, k, sizeof (r.data));
186  strlcpy(r.type, v, sizeof (r.type));
187 
188  ZE_MessageInfo(12, "TYPE=%-15s SCORE=%.2f VALUE=%s\n", r.type,
189  (double) r.score, r.data);
190 
191  return zeTable_Add(&htbl, &r);
192 }
193 
194 /* ***************************************************************************
195  * *
196  * *
197  *****************************************************************************/
198 static bool
199 read_it(path, tag)
200  char *path;
201  char *tag;
202 {
203  int r;
204 
205  r = zm_RdFile(path, tag, add_oradata_rec, NULL);
206 
207  return r >= 0;
208 }
209 
210 bool
211 load_oradata_table(cfdir, fname)
212  char *cfdir;
213  char *fname;
214 {
215  int res = 0;
216  static int htbl_ok = FALSE;
217  bool result = FALSE;
218 
219  ZE_MessageInfo(12, "Will load %s file", STRNULL(fname, "(NULL)"));
220 
221  if ((fname == NULL) || (strlen(fname) == 0))
223 
224  if ((fname == NULL) || (strlen(fname) == 0))
225  fname = ZE_ORADATA_FILE;
226 
227  ORA_LOCK();
228 
229  if (htbl_ok == FALSE) {
230  memset(&htbl, 0, sizeof (htbl));
231  res = zeTable_Init(&htbl, sizeof (oradata_T), 256, oradata_comp);
232  if (res == 0)
233  htbl_ok = TRUE;
234  }
235  if (res == 0)
236  res = zeTable_Clear(&htbl);
237 
238  if (res == 0)
239  result =
240  read_conf_data_file(cfdir, fname, "ze-oracle:oracle-data", read_it);
241 
242  ORA_UNLOCK();
243 
244  return result;
245 }
246 
247 
248 /* ***************************************************************************
249  * *
250  * *
251  *****************************************************************************/
252 int
253 count_oradata(id, type, data, find, odds)
254  char *id;
255  char *type;
256  char *data;
257  bool find;
258  double *odds;
259 {
260  oradata_T p;
261  int nb = 0;
262  bestof_T best;
263  double mean = 0;
264 
265  id = STRNULL(id, "00000000.000");
266  data = STRNULL(data, "");
267  type = STRNULL(type, "");
268 
269  memset(&p, 0, sizeof (p));
270 
271  bestof_init(&best, 3, NULL);
272 
273  ORA_LOCK();
274  if (zeTable_Get_First(&htbl, &p) == 0) {
275  do {
276  if (strcasecmp(type, p.type) != 0)
277  continue;
278 
279  if (zeStrRegex(data, p.data, NULL, NULL, TRUE)) {
281  ZE_MessageNotice(10, "%s SPAM CHECK - UNWANTED %s : %s", id, type,
282  p.data);
283  nb++;
284  bestof_add(&best, p.pOdds);
285  if (find)
286  break;
287  }
288  } while (zeTable_Get_Next(&htbl, &p) == 0);
289  }
290  ORA_UNLOCK();
291 
292  mean = bestof_average(&best);
293  if (odds != NULL)
294  *odds = mean;
295 
296  ZE_MessageInfo(11, "%s ->Computed ORACLE %-9s odds is %5.2f ...", id, type,
297  mean);
298 
299  return nb;
300 }
301 
302 
303 /* ***************************************************************************
304  * *
305  * *
306  *****************************************************************************/
307 #if 0
308 
309 static pthread_mutex_t vc_mutex = PTHREAD_MUTEX_INITIALIZER;
310 
311 #define VC_LOCK() \
312  if (pthread_mutex_lock(&vc_mutex) != 0) { \
313  ZE_LogSysError("pthread_mutex_lock(vc_mutex)"); \
314  }
315 
316 #define VC_UNLOCK() \
317  if (pthread_mutex_unlock(&vc_mutex) != 0) { \
318  ZE_LogSysError("pthread_mutex_unlock(vc_mutex)"); \
319  }
320 #else
321 #define VC_LOCK()
322 #define VC_UNLOCK()
323 #endif
324 
325 double
326 vector_compare(a, b, dim)
327  double *a;
328  double *b;
329  int dim;
330 {
331  double *ta = NULL, *tb = NULL;
332  size_t sdim = 0;
333  double ma, mb, ab, res;
334  int i;
335 
336  res = 1.;
337 
338  if ((a == NULL) || (b == NULL) || (dim <= 0))
339  return res;
340 
341  VC_LOCK();
342 
343  if (1 || (dim > sdim)) {
344  FREE(ta);
345  FREE(tb);
346  if ((ta = (double *) malloc(dim * sizeof (double))) == NULL)
347  ZE_LogSysError("malloc(ta)");
348  if ((tb = (double *) malloc(dim * sizeof (double))) == NULL)
349  ZE_LogSysError("malloc(tb)");
350  sdim = dim;
351  }
352 
353  if ((ta != NULL) && (tb != NULL)) {
354  for (i = 0; i < dim; i++) {
355  if (a[i] == b[i]) {
356  if (a[i] != 0)
357  ta[i] = tb[i] = 1.;
358  else
359  ta[i] = tb[i] = 0.;
360 
361  ta[i] = tb[i] = 1.; /* JOE */
362  continue;
363  }
364  if (a[i] > b[i]) {
365  ta[i] = 1.;
366  tb[i] = b[i] / a[i];
367  } else {
368  ta[i] = a[i] / b[i];
369  tb[i] = 1.;
370  }
371  }
372 
373  ma = mb = ab = 0;
374  for (i = 0; i < dim; i++) {
375  ma += ta[i] * ta[i];
376  mb += tb[i] * tb[i];
377  ab += ta[i] * tb[i];
378  }
379  if ((ma > 0.) && (mb > 0.))
380  res = ab / sqrt(ma * mb);
381  else {
382  if (ma == mb)
383  res = 1.;
384  else
385  res = 0.;
386  res = 1.; /* JOE */
387  }
388  }
389 
390  /*
391  ** Shall free or let it allocated for the next round ???
392  */
393  FREE(ta);
394  FREE(tb);
395 
396  VC_UNLOCK();
397 
398  return res;
399 }
400 
401 
402 
403 /* ***************************************************************************
404  * *
405  * *
406  *****************************************************************************/
407 char *
409  char *buf;
410  size_t size;
411 {
412  char *p, *q, *nbuf;
413 
414  if ((buf == NULL) || (size == 0))
415  return NULL;
416 
417  if ((nbuf = strdup(buf)) == NULL) {
418  ZE_LogSysError("strdup");
419  return NULL;
420  }
421 
422  p = buf;
423  q = nbuf;
424  while (*++p != '\0') {
425  if (isalpha(*p)) {
426  *q++ = tolower(*p);
427  continue;
428  }
429  if (isdigit(*p)) {
430  if (isalpha(*(p - 1)) || isalpha(*(p + 1))) {
431  switch (*p) {
432  case '0':
433  *q++ = 'o';
434  break;
435  case '1':
436  *q++ = 'i';
437  break;
438  case '3':
439  *q++ = 'e';
440  break;
441  case '4':
442  *q++ = 'a';
443  break;
444  default:
445  *q++ = ' ';
446  break;
447  }
448  } else {
449  *q++ = ' ';
450  }
451  continue;
452  }
453  if (isspace(*p)) {
454  *q++ = ' ';
455  continue;
456  }
457  if (isalpha(*(p - 1)) || (0 && isalpha(*(p + 1)))) {
458  switch (*p) {
459  case '|':
460  *q++ = 'l';
461  break;
462  case '@':
463  *q++ = 'a';
464  break;
465  case '&':
466  *q++ = 'e';
467  break;
468  default:
469  break;
470  }
471  }
472 #if 0
473  else
474  *q++ = ' ';
475 #endif
476  continue;
477  }
478  *q = '\0';
479 
480  {
481  bool space = FALSE;
482 
483  for (p = q = nbuf; *p != '\0'; p++) {
484  if (isspace(*p)) {
485  if (!space) {
486  space = TRUE;
487  *q++ = *p;
488  }
489  continue;
490  }
491  space = FALSE;
492  *q++ = *p;
493  }
494  *q = '\0';
495  }
496  return nbuf;
497 }
bool load_oradata_table(char *cfdir, char *fname)
Definition: ze-oracle.c:211
int zm_RdFile(char *fname, char *tag, RDFILE_F f, void *arg)
Definition: zeRdFile.c:185
float score
Definition: ze-oracle.c:39
char * realcleanup_text_buf(char *buf, size_t size)
Definition: ze-oracle.c:408
#define ASSERT(a)
Definition: macros.h:27
#define ORADATA_LEN
Definition: ze-oracle.c:33
bool bestof_init(bestof_T *b, int dim, bestcomp_F bcmp)
Definition: ze-bestof-n.c:51
#define FREE(x)
Definition: macros.h:37
#define STRNULL(x, r)
Definition: macros.h:81
#define VC_LOCK()
Definition: ze-oracle.c:321
#define SKIP_SPACES(s)
Definition: macros.h:171
int count_oradata(char *id, char *type, char *data, bool find, double *odds)
Definition: ze-oracle.c:253
double bestof_average(bestof_T *b)
Definition: ze-bestof-n.c:97
#define FALSE
Definition: macros.h:160
#define strlcpy
Definition: zeString.h:32
double pOdds
Definition: ze-oracle.c:40
#define KEYVALUE
Definition: ze-oracle.c:112
bool zeStrRegex(char *, char *, long *, long *, bool)
Definition: zeStrings.c:544
#define ORA_LOCK()
Definition: ze-oracle.c:49
#define ORA_UNLOCK()
Definition: ze-oracle.c:50
#define CF_LOG_LEVEL_ORACLE
Definition: cfh-defs.h:126
int cf_get_int(int id)
Definition: ze-cf.c:803
bool read_conf_data_file(char *cfdir, char *fname, char *dfile, read_conf_data_file_F func)
#define strchr
Definition: ze-sys.h:218
int zeTable_Clear(zeTbl_T *)
Definition: zeTable.c:136
bool bestof_add(bestof_T *b, double v)
Definition: ze-bestof-n.c:75
#define ZE_MessageNotice(level,...)
Definition: zeSyslog.h:91
double vector_compare(double *a, double *b, int dim)
Definition: ze-oracle.c:326
char type[32]
Definition: ze-oracle.c:36
int zeTable_Add(zeTbl_T *, void *)
Definition: zeTable.c:155
int zeTable_Get_First(zeTbl_T *, void *)
Definition: zeTable.c:250
#define SKIP_KEYCHARS(s)
Definition: macros.h:199
#define ZE_MessageInfo(level,...)
Definition: zeSyslog.h:90
int nb
Definition: ze-connopen.c:61
#define TRUE
Definition: macros.h:157
#define ZE_MessageWarning(level,...)
Definition: zeSyslog.h:92
void dump_oradata_table()
Definition: ze-oracle.c:69
#define ZE_LogSysError(...)
Definition: zeSyslog.h:129
char * cf_get_str(int id)
Definition: ze-cf.c:854
int zeTable_Get_Next(zeTbl_T *, void *)
Definition: zeTable.c:277
char data[ORADATA_LEN]
Definition: ze-oracle.c:37
#define ZE_ORADATA_FILE
Definition: defs.h:54
#define VC_UNLOCK()
Definition: ze-oracle.c:322
int zeTable_Init(zeTbl_T *, size_t, int, int(*)(const void *, const void *))
double nOdds
Definition: ze-oracle.c:41
#define STRCASEEQUAL(a, b)
Definition: macros.h:72
#define CF_ORACLE_DATA_FILE
Definition: cfh-defs.h:125
char action[32]
Definition: ze-oracle.c:38