Changeset 260

Timestamp:

Dec 7, 2009, 10:00:29 PM (15 years ago)

Author:

katerina

Message:

Support event correlation (ticket #178).

Location:

trunk

Files:

• docs/MANUAL-2_3.html.tar (modified) ( previous)
• docs/MANUAL-2_3.pdf (modified) ( previous)
• include/sh_cat.h (modified) (1 diff)
• include/sh_log_evalrule.h (modified) (1 diff)
• include/sh_string.h (modified) (1 diff)
• src/sh_cat.c (modified) (2 diffs)
• src/sh_log_check.c (modified) (4 diffs)
• src/sh_log_evalrule.c (modified) (25 diffs)
• src/sh_string.c (modified) (1 diff)

trunk/include/sh_cat.h

@@ -167 +167 @@
  MSG_LOGMON_REP,
  MSG_LOGMON_SUM,
+ MSG_LOGMON_COR,
 #endif
 

trunk/include/sh_log_evalrule.h

@@ -35 +35 @@
 int sh_eval_gend (const char * str);
 
+/* Process a single log record
+ */
 int sh_eval_process_msg(struct sh_logrecord * record);
+
+/* Match correlated rules
+ */
+void sh_keep_match();
+
 #endif

trunk/include/sh_string.h

@@ -84 +84 @@
 char ** split_array_list(char *line, unsigned int * nfields, size_t * lengths);
 
+/* Return a split_array_list() of a list contained in 'PREFIX\s*( list ).*'
+ */
+char ** split_array_braced (char *line, const char * prefix,
+                            unsigned int * nfields, size_t * lengths);
+
 /* Replaces fields in s with 'replacement'. Fields are given
  * in the ordered array ovector, comprising ovecnum pairs 

trunk/src/sh_cat.c

@@ -159 +159 @@
   { MSG_LOGMON_REP,  SH_ERR_SEVERE,  EVENT, N_("msg=\"POLICY [Logfile] %s\" time=\"%s\" host=\"%s\" path=\"%s\"") },
   { MSG_LOGMON_SUM,  SH_ERR_SEVERE,  EVENT, N_("msg=\"POLICY [Logfile] %s\" host=\"%s\" path=\"%s\"") },
+  { MSG_LOGMON_COR,  SH_ERR_SEVERE,  EVENT, N_("msg=\"POLICY [Logfile] Correlated events %s\"") },
 #endif
 
@@ -488 +489 @@
   { MSG_LOGMON_REP,  SH_ERR_SEVERE,  EVENT, N_("msg=<POLICY [Logfile] %s> time=<%s> host=<%s> path=<%s>") },
   { MSG_LOGMON_SUM,  SH_ERR_SEVERE,  EVENT, N_("msg=<POLICY [Logfile] %s> host=<%s> path=<%s>") },
+  { MSG_LOGMON_COR,  SH_ERR_SEVERE,  EVENT, N_("msg=<POLICY [Logfile] Correlated events %s>") },
 #endif
 

trunk/src/sh_log_check.c

@@ -28 +28 @@
 
 /* List of supported logfile types, format is
- * { "TYPE_CODE", Reader_Callback_Function, Parser_Callback_function }
+ * { 
+ *   "TYPE_CODE", 
+ *   Reader_Callback_Function, 
+ *   Parser_Callback_function,
+ *   Evaluate_Callback_Function 
+ * }
  * If Reader_Callback_Function is NULL, the default (line-oriented
  * text file) reader is used.
@@ -78 +83 @@
   plen = strlen(save_dir);
 
-  if (SL_TRUE == sl_ok_adds(plen, 129))
-    {
-      plen += 129; /* 64 + 64 + 1 */
+  if (SL_TRUE == sl_ok_adds(plen, 130))
+    {
+      plen += 130; /* 64 + 64 + 2 */
       path = SH_ALLOC(plen);
       (void) sl_snprintf(path, plen, "%s/%lu_%lu", save_dir,
@@ -860 +865 @@
     {
       sh_check_watches();
+      sh_keep_match();
     }
   SH_MUTEX_UNLOCK(mutex_logmon_check);
@@ -1036 +1042 @@
 
 /* Define a check rule.
- * Format: Queue_Label : Regex
- * TYPE must be 'report' or 'sum'
+ * Format: [KEEP(seconds,label):]Queue_Label : Regex
+ * KEEP indicates that we keep the label, to perform
+ *      correlation matching
  */
 static int sh_logmon_add_rule (const char * str)

trunk/src/sh_log_evalrule.c

@@ -89 +89 @@
 };
 
+enum {
+  RFL_ISRULE  = 1 << 0,
+  RFL_ISGROUP = 1 << 1,
+  RFL_KEEP    = 1 << 2
+};
+
+/*--------------------------------------------------------------*/
+
+struct sh_keep
+{
+  sh_string       * label;           /* label of keep rule   */
+  unsigned long     delay;           /* valid delay             */
+  time_t            last;            /* seen at                 */
+  struct sh_keep *  next; 
+};
+
+static struct sh_keep * keeplist  = NULL;
+static struct sh_keep * keeplast  = NULL;
+static unsigned long    keepcount = 0;
+
+static void sh_keep_free(void * item)
+{
+  struct sh_keep * keep = (struct sh_keep *) item;
+  if (!keep)
+    return;
+  sh_string_destroy(&(keep->label));
+  SH_FREE(keep);
+}
+
+static void sh_keep_destroy()
+{
+  struct sh_keep * keep;
+
+  while (keeplist)
+    {
+      keep = keeplist;
+      keeplist = keep->next;
+      sh_keep_free(keep);
+      --keepcount;
+    }
+  keeplist  = NULL;
+  keeplast  = NULL;
+  keepcount = 0;
+}
+
+static int sh_keep_add(sh_string * label, unsigned long delay, time_t last)
+{
+  struct sh_keep * keep = SH_ALLOC(sizeof(struct sh_keep));
+
+  keep->label = sh_string_copy(label);
+  keep->delay = delay;
+  keep->last  = last;
+  keep->next  = NULL;
+
+  if (keeplast && keeplist)
+    {
+      keeplast->next = keep;
+      keeplast       = keep;
+    }
+  else
+    {
+      keeplist = keep;
+      keeplast = keeplist;
+    }
+  ++keepcount;
+  return 0;
+}
+
+int sh_keep_comp(const void * a, const void * b)
+{
+  return ( (int)(((struct sh_keep *)a)->last) - 
+           (int)(((struct sh_keep *)b)->last) );
+}
+
+static sh_string * sh_keep_eval()
+{
+  unsigned long count   = 0;
+  sh_string * res       = NULL;
+  time_t now            = time(NULL);
+  struct sh_keep * keep = keeplist;
+  struct sh_keep * prev = keeplist;
+  struct sh_keep * arr;
+
+  if (keepcount > 0)
+    {
+      arr = SH_ALLOC (keepcount * sizeof(struct sh_keep));
+
+      while (count < keepcount && keep)
+        {
+          if ((now > keep->last) && ((unsigned long)(now - keep->last) <= keep->delay))
+            {
+              memcpy(&(arr[count]), keep, sizeof(struct sh_keep));
+              ++count;
+              prev = keep;
+              keep = keep->next;
+            }
+          else /* Too old or in future, delete it */
+            {
+              if (keep != keeplist)
+                {
+                  prev->next = keep->next;
+                  sh_keep_free(keep);
+                  keep = prev->next;
+                  --keepcount;
+                }
+              else /* list head */
+                {
+                  keeplist = keep->next;
+                  prev     = keeplist;
+                  sh_keep_free(keep);
+                  keep     = keeplist;
+                  --keepcount;
+                }
+            }
+        }
+
+      if (count > 0)
+        {
+          unsigned long i;
+          qsort(arr, count, sizeof(struct sh_keep), sh_keep_comp);
+          res = sh_string_copy(arr[0].label);
+          for (i = 1; i < count; ++i)
+            res = sh_string_add(res, arr[i].label);
+        }
+      SH_FREE(arr);
+    }
+  return res;
+}
+
+struct sh_mkeep
+{
+  sh_string       * label;           /* label of match rule     */
+  pcre            * rule;            /* compiled regex for rule */
+  struct sh_qeval * queue;           /* assigned queue          */
+  struct sh_mkeep * next; 
+};
+
+struct sh_mkeep * mkeep_list = NULL;
+
+static struct sh_qeval * find_queue(const char * str);
+
+static int sh_keep_match_add(const char * str, const char * queue, const char * pattern)
+{
+  unsigned int nfields = 1; /* seconds:label */
+  size_t       lengths[1];
+  char *       new    = sh_util_strdup(str);
+  char **      splits = split_array_braced(new, _("CORRELATE"), &nfields, lengths);
+
+  if (nfields == 1 && lengths[0] > 0)
+    {
+      struct sh_mkeep * mkeep = SH_ALLOC(sizeof(struct sh_mkeep));
+      const char * error;
+      int          erroffset;
+      struct sh_qeval * rqueue = NULL;
+
+      mkeep->rule = pcre_compile(pattern, PCRE_NO_AUTO_CAPTURE, 
+                             &error, &erroffset, NULL);
+      if (!(mkeep->rule))
+        {
+          sh_string * msg =  sh_string_new(0);
+          sh_string_add_from_char(msg, _("Bad regex: "));
+          sh_string_add_from_char(msg, pattern);
+          
+          SH_MUTEX_LOCK(mutex_thread_nolog);
+          sh_error_handle(SH_ERR_ERR, FIL__, __LINE__, 0, MSG_E_SUBGEN,
+                          sh_string_str(msg),
+                          _("sh_keep_match_add"));
+          SH_MUTEX_UNLOCK(mutex_thread_nolog);
+          sh_string_destroy(&msg);
+          
+          SH_FREE(splits);
+          SH_FREE(mkeep);
+          SH_FREE(new);
+          return -1;
+        }
+
+      if (0 != strcmp(queue, _("trash")))
+        {
+
+          rqueue = find_queue(queue);
+          if (!rqueue)
+            {
+              pcre_free(mkeep->rule);
+              SH_FREE(splits);
+              SH_FREE(mkeep);
+              SH_FREE(new);
+              return -1;
+            }
+        }
+
+      mkeep->queue = rqueue;
+      mkeep->label = sh_string_new_from_lchar(splits[0], strlen(splits[0]));
+      mkeep->next  = mkeep_list;
+      mkeep_list   = mkeep;
+    }
+  SH_FREE(new);
+  return 0;
+}
+
+static void sh_keep_match_del()
+{
+  struct sh_mkeep * mkeep = mkeep_list;
+  while (mkeep)
+    {
+      mkeep_list = mkeep->next;
+      sh_string_destroy(&(mkeep->label));
+      pcre_free(mkeep->rule);
+      mkeep = mkeep_list;
+    }
+  mkeep_list = NULL;
+}
+
+static struct sh_mkeep ** dummy_mkeep;
+
+void sh_keep_match()
+{
+  if (mkeep_list)
+    {
+      sh_string       * res = sh_keep_eval();
+
+      if (res)
+        {
+          struct sh_mkeep * mkeep = mkeep_list;
+
+          dummy_mkeep = &mkeep;
+
+          while (mkeep)
+            {
+              int val = pcre_exec(mkeep->rule, NULL, 
+                                  sh_string_str(res), (int)sh_string_len(res), 
+                                  0, 0, NULL, 0);
+              if (val >= 0)
+                {
+                  char * tmp;
+                  SH_MUTEX_LOCK(mutex_thread_nolog);
+                  tmp = sh_util_safe_name (sh_string_str(mkeep->label));
+                  sh_error_handle (mkeep->queue->severity, FIL__, __LINE__, 0, 
+                                   MSG_LOGMON_COR, tmp);
+                  SH_FREE(tmp);
+                  SH_MUTEX_UNLOCK(mutex_thread_nolog);
+                }
+              mkeep = mkeep->next;
+            }
+          sh_string_destroy(&res);
+        }
+    }
+  return;
+}
+
+/*--------------------------------------------------------------*/
+
 struct sh_geval  /* Group of rules (may be a single rule) */
 {
@@ -97 +348 @@
   int               ovecnum;         /* how many captured       */
   int               captures;        /* (captures+1)*3 required */
+  int               flags;           /* bit flags               */
+  unsigned long     delay;           /* delay for keep rules    */
   zAVLTree        * counterlist;     /* counters if EVAL_SUM    */
   struct sh_qeval * queue;           /* queue for this rule     */
@@ -178 +431 @@
   ng = SH_ALLOC(sizeof(struct sh_geval));
   ng->label       = sh_string_new_from_lchar(splits[0], lengths[0]);
+  ng->flags       = RFL_ISGROUP;
+
   ng->rule        = group;
   ng->rule_extra  = group_extra;
@@ -384 +639 @@
 }
 
+char * get_keep(char * str, unsigned long * seconds)
+{
+  char       * res    = NULL;
+  char       * endptr = NULL;
+
+  unsigned int nfields = 2; /* seconds:label */
+  size_t       lengths[2];
+  char *       new    = sh_util_strdup(str);
+  char **      splits = split_array_braced(new, _("KEEP"), &nfields, lengths);
+
+  if (nfields == 2 && lengths[0] > 0 && lengths[1] > 0)
+    {
+      *seconds = strtoul(splits[0], &endptr, 10);
+      if ((endptr == '\0' || endptr != splits[0]) && (*seconds != ULONG_MAX))
+        {
+          res = sh_util_strdup(splits[1]);
+        }
+    }
+  if (splits)
+    SH_FREE(splits);
+  SH_FREE(new);
+  return res;
+}
+
 static struct sh_qeval ** dummy_queue;
+static char            ** dummy_dstr;
 
 int sh_eval_radd (const char * str)
@@ -398 +678 @@
   unsigned int nfields = 2; /* queue:regex */
   size_t       lengths[2];
-  char *       new = sh_util_strdup(str);
+  char *       new    = sh_util_strdup(str);
   char **      splits = split_array(new, &nfields, ':', lengths);
 
+  int           qpos = 0;
+  int           rpos = 1;
+  unsigned long dsec = 0;
+  char *        dstr = NULL;
+
   dummy_queue = &queue;
-
-  if (nfields != 2)
+  dummy_dstr  = &dstr;
+
+  if (nfields < 2 || nfields > 3)
     {
       SH_FREE(splits);
@@ -410 +696 @@
     }
 
-  if (0 != strcmp(splits[0], _("trash")))
-      {
-        queue = find_queue(splits[0]);
-        if (!queue)
-          {
-            SH_FREE(splits);
-            SH_FREE(new);
-            return -1;
-          }
-      }
-
-  rule = pcre_compile(splits[1], 0, 
+  if (nfields == 3)
+    {
+      /* KEEP(nsec):queue:regex
+       */
+      dstr = get_keep(splits[0], &dsec);
+      if (!dstr)
+        {
+          /* CORRELATE:queue:regex 
+           */
+          int retval = sh_keep_match_add(splits[0], splits[1], splits[2]);
+          SH_FREE(splits);
+          SH_FREE(new);
+          return retval;
+        }
+      ++qpos; ++rpos;
+    }
+
+  if (0 != strcmp(splits[qpos], _("trash")))
+    {
+      queue = find_queue(splits[qpos]);
+      if (!queue)
+        {
+          SH_FREE(splits);
+          SH_FREE(new);
+          return -1;
+        }
+    }
+
+  rule = pcre_compile(splits[rpos], 0, 
                       &error, &erroffset, NULL);
   if (!rule)
@@ -427 +730 @@
       sh_string * msg =  sh_string_new(0);
       sh_string_add_from_char(msg, _("Bad regex: "));
-      sh_string_add_from_char(msg, splits[1]);
+      sh_string_add_from_char(msg, splits[rpos]);
       
       SH_MUTEX_LOCK(mutex_thread_nolog);
@@ -446 +749 @@
     {
       char * emsg = SH_ALLOC(SH_ERRBUF_SIZE);
-      sl_snprintf(emsg,  SH_ERRBUF_SIZE, _("Adding rule: |%s| with %d captures"), 
-                  splits[1], captures);
+      if (dstr)
+        sl_snprintf(emsg,  SH_ERRBUF_SIZE, _("Adding rule: |%s| with %d captures, keep(%lu,%s)"), 
+                    splits[rpos], captures, dsec, dstr);
+      else
+        sl_snprintf(emsg,  SH_ERRBUF_SIZE, _("Adding rule: |%s| with %d captures"), 
+                    splits[rpos], captures);
       SH_MUTEX_LOCK(mutex_thread_nolog);
       sh_error_handle(SH_ERR_ALL, FIL__, __LINE__, 0, MSG_E_SUBGEN,
@@ -455 +762 @@
     }
 
-  DEBUG("adding rule: |%s| with %d captures\n", splits[1], captures);
+  DEBUG("adding rule: |%s| with %d captures\n", splits[rpos], captures);
 
   SH_FREE(splits);
@@ -462 +769 @@
   nr = SH_ALLOC(sizeof(struct sh_geval));
   nr->label       = NULL;
+  nr->flags       = RFL_ISRULE;
+  nr->delay       = 0;
+
   nr->rule        = rule;
   nr->rule_extra  = rule_extra;
@@ -473 +783 @@
   nr->gnext       = NULL;
 
+
+  if (dstr)
+    {
+      nr->label   = sh_string_new_from_lchar(dstr, strlen(dstr));
+      nr->flags  |= RFL_KEEP;
+      nr->delay   = dsec;
+      SH_FREE(dstr);
+    }
+
   /* 
    * If there is an open group, add it to its
@@ -513 +832 @@
           if (0 != sh_eval_hadd("^.*"))
             {
+              if (nr->label)
+                sh_string_destroy(&(nr->label));
               SH_FREE(nr->ovector);
               SH_FREE(nr);
@@ -559 +880 @@
       else
         {
+          if (nr->label)
+            sh_string_destroy(&(nr->label));
           SH_FREE(nr->ovector);
           SH_FREE(nr);
@@ -623 +946 @@
     }
 
+  sh_keep_destroy();
+  sh_keep_match_del();
+
+  return;
 }
 
@@ -636 +963 @@
 static struct sh_geval ** dummy1;
 
-static struct sh_geval * test_rule (struct sh_geval * rule, sh_string *msg)
+static struct sh_geval * test_rule (struct sh_geval * rule, sh_string *msg, time_t tstamp)
 {
   int res; 
-  volatile int count;
+  volatile int    count;
+  volatile time_t timestamp = tstamp;
 
   dummy1 = &rule;
@@ -674 +1002 @@
               {
                 char * emsg = SH_ALLOC(SH_ERRBUF_SIZE);
-                sl_snprintf(emsg,  SH_ERRBUF_SIZE, _("Rule %d matches, result = %d"), 
-                            count, res);
+                if ( rule->flags & RFL_KEEP )
+                  sl_snprintf(emsg,  SH_ERRBUF_SIZE, _("Rule %d matches, result = %d (keep)"), 
+                              count, res);
+                else
+                  sl_snprintf(emsg,  SH_ERRBUF_SIZE, _("Rule %d matches, result = %d"), 
+                              count, res);
                 SH_MUTEX_LOCK(mutex_thread_nolog);
                 sh_error_handle(SH_ERR_ALL, FIL__, __LINE__, 0, MSG_E_SUBGEN,
@@ -682 +1014 @@
                 SH_FREE(emsg);
               }
-            DEBUG("debug: rule %d matches, result = %d\n", count, res);
+
+            if ( rule->flags & RFL_KEEP )
+              {
+                DEBUG("debug: rule %d matches (keep)\n", count);
+                sh_keep_add(rule->label, rule->delay, 
+                            timestamp == 0 ? time(NULL) : timestamp);
+              }
+
             break; /* return the matching rule; ovector is filled in */
           }
@@ -698 +1037 @@
           }
         DEBUG("debug: rule %d did not match\n", count);
+
         rule = rule->nextrule; ++count;
       } while (rule);
@@ -714 +1054 @@
 
 static struct sh_geval * test_grules (struct sh_heval * host, 
-                                      sh_string *msg)
+                                      sh_string       * msg,
+                                      time_t            timestamp)
 {
   struct sh_geval * result = NULL;
@@ -726 +1067 @@
       DEBUG("debug: if group\n");
       do {
-        if(group->label != NULL) 
+        if( (group->label != NULL) && (0 != (group->flags & RFL_ISGROUP))) 
           {
-            /* this is a rule group; only groups have labels */
+            /* this is a rule group */
 
             if (flag_err_debug == SL_TRUE)
@@ -747 +1088 @@
                           0, 0, NULL, 0) >= 0)
               {
-                result = test_rule(group->nextrule, msg);
+                result = test_rule(group->nextrule, msg, timestamp);
                 if (result)
                   break;
@@ -770 +1111 @@
 
             DEBUG("debug: else (single rule)\n");
-            result = test_rule(group, msg);
+            result = test_rule(group, msg, timestamp);
             if (result)
               break;
@@ -783 +1124 @@
  */
 static struct sh_geval * find_rule (sh_string *host, 
-                                    sh_string *msg)
+                                    sh_string *msg,
+                                    time_t     timestamp)
 {
   struct sh_geval * result = NULL;
@@ -796 +1138 @@
           {
             /* matching host, check rules/groups of rules */
-            result = test_grules(hlist, msg);
+            result = test_grules(hlist, msg, timestamp);
             if (result)
               break;
@@ -994 +1336 @@
     {
       struct sh_geval * rule = find_rule (record->host,
-                                          record->message);
+                                          record->message,
+                                          record->timestamp);
 
       if (rule)

trunk/src/sh_string.c

@@ -238 +238 @@
 {
   return split_array_ws_int (line, nfields, lengths, SH_SPLIT_LIST);
+}
+
+/* return a split() of a list contained in 'PREFIX\s*( list ).*'
+ */
+char ** split_array_braced (char *line, const char * prefix,
+                            unsigned int * nfields, size_t * lengths)
+{
+  char * s = line;
+  char * p;
+
+  while ( *s && isspace((int)*s) ) ++s;
+  if (prefix && 0 != strncmp(s, prefix, strlen(prefix)))
+    return NULL;
+  s = &s[strlen(prefix)];
+  while ( *s && isspace((int)*s) ) ++s;
+  if (!s || (*s != '('))
+    return NULL;
+  ++s;
+  p = strchr(s, ')');
+  if (!p || (*p == *s))
+    return NULL;
+  *p = '\0';
+  return split_array_list (s, nfields, lengths);
 }