Index: trunk/include/sh_cat.h =================================================================== --- trunk/include/sh_cat.h (revision 259) +++ trunk/include/sh_cat.h (revision 260) @@ -167,4 +167,5 @@ MSG_LOGMON_REP, MSG_LOGMON_SUM, + MSG_LOGMON_COR, #endif Index: trunk/include/sh_log_evalrule.h =================================================================== --- trunk/include/sh_log_evalrule.h (revision 259) +++ trunk/include/sh_log_evalrule.h (revision 260) @@ -35,4 +35,11 @@ int sh_eval_gend (const char * str); +/* Process a single log record + */ int sh_eval_process_msg(struct sh_logrecord * record); + +/* Match correlated rules + */ +void sh_keep_match(); + #endif Index: trunk/include/sh_string.h =================================================================== --- trunk/include/sh_string.h (revision 259) +++ trunk/include/sh_string.h (revision 260) @@ -84,4 +84,9 @@ char ** split_array_list(char *line, unsigned int * nfields, size_t * lengths); +/* Return a split_array_list() of a list contained in 'PREFIX\s*( list ).*' + */ +char ** split_array_braced (char *line, const char * prefix, + unsigned int * nfields, size_t * lengths); + /* Replaces fields in s with 'replacement'. Fields are given * in the ordered array ovector, comprising ovecnum pairs Index: trunk/src/sh_cat.c =================================================================== --- trunk/src/sh_cat.c (revision 259) +++ trunk/src/sh_cat.c (revision 260) @@ -159,4 +159,5 @@ { MSG_LOGMON_REP, SH_ERR_SEVERE, EVENT, N_("msg=\"POLICY [Logfile] %s\" time=\"%s\" host=\"%s\" path=\"%s\"") }, { MSG_LOGMON_SUM, SH_ERR_SEVERE, EVENT, N_("msg=\"POLICY [Logfile] %s\" host=\"%s\" path=\"%s\"") }, + { MSG_LOGMON_COR, SH_ERR_SEVERE, EVENT, N_("msg=\"POLICY [Logfile] Correlated events %s\"") }, #endif @@ -488,4 +489,5 @@ { MSG_LOGMON_REP, SH_ERR_SEVERE, EVENT, N_("msg= time=<%s> host=<%s> path=<%s>") }, { MSG_LOGMON_SUM, SH_ERR_SEVERE, EVENT, N_("msg= host=<%s> path=<%s>") }, + { MSG_LOGMON_COR, SH_ERR_SEVERE, EVENT, N_("msg=") }, #endif Index: trunk/src/sh_log_check.c =================================================================== --- trunk/src/sh_log_check.c (revision 259) +++ trunk/src/sh_log_check.c (revision 260) @@ -28,5 +28,10 @@ /* List of supported logfile types, format is - * { "TYPE_CODE", Reader_Callback_Function, Parser_Callback_function } + * { + * "TYPE_CODE", + * Reader_Callback_Function, + * Parser_Callback_function, + * Evaluate_Callback_Function + * } * If Reader_Callback_Function is NULL, the default (line-oriented * text file) reader is used. @@ -78,7 +83,7 @@ plen = strlen(save_dir); - if (SL_TRUE == sl_ok_adds(plen, 129)) - { - plen += 129; /* 64 + 64 + 1 */ + if (SL_TRUE == sl_ok_adds(plen, 130)) + { + plen += 130; /* 64 + 64 + 2 */ path = SH_ALLOC(plen); (void) sl_snprintf(path, plen, "%s/%lu_%lu", save_dir, @@ -860,4 +865,5 @@ { sh_check_watches(); + sh_keep_match(); } SH_MUTEX_UNLOCK(mutex_logmon_check); @@ -1036,6 +1042,7 @@ /* Define a check rule. - * Format: Queue_Label : Regex - * TYPE must be 'report' or 'sum' + * Format: [KEEP(seconds,label):]Queue_Label : Regex + * KEEP indicates that we keep the label, to perform + * correlation matching */ static int sh_logmon_add_rule (const char * str) Index: trunk/src/sh_log_evalrule.c =================================================================== --- trunk/src/sh_log_evalrule.c (revision 259) +++ trunk/src/sh_log_evalrule.c (revision 260) @@ -89,4 +89,255 @@ }; +enum { + RFL_ISRULE = 1 << 0, + RFL_ISGROUP = 1 << 1, + RFL_KEEP = 1 << 2 +}; + +/*--------------------------------------------------------------*/ + +struct sh_keep +{ + sh_string * label; /* label of keep rule */ + unsigned long delay; /* valid delay */ + time_t last; /* seen at */ + struct sh_keep * next; +}; + +static struct sh_keep * keeplist = NULL; +static struct sh_keep * keeplast = NULL; +static unsigned long keepcount = 0; + +static void sh_keep_free(void * item) +{ + struct sh_keep * keep = (struct sh_keep *) item; + if (!keep) + return; + sh_string_destroy(&(keep->label)); + SH_FREE(keep); +} + +static void sh_keep_destroy() +{ + struct sh_keep * keep; + + while (keeplist) + { + keep = keeplist; + keeplist = keep->next; + sh_keep_free(keep); + --keepcount; + } + keeplist = NULL; + keeplast = NULL; + keepcount = 0; +} + +static int sh_keep_add(sh_string * label, unsigned long delay, time_t last) +{ + struct sh_keep * keep = SH_ALLOC(sizeof(struct sh_keep)); + + keep->label = sh_string_copy(label); + keep->delay = delay; + keep->last = last; + keep->next = NULL; + + if (keeplast && keeplist) + { + keeplast->next = keep; + keeplast = keep; + } + else + { + keeplist = keep; + keeplast = keeplist; + } + ++keepcount; + return 0; +} + +int sh_keep_comp(const void * a, const void * b) +{ + return ( (int)(((struct sh_keep *)a)->last) - + (int)(((struct sh_keep *)b)->last) ); +} + +static sh_string * sh_keep_eval() +{ + unsigned long count = 0; + sh_string * res = NULL; + time_t now = time(NULL); + struct sh_keep * keep = keeplist; + struct sh_keep * prev = keeplist; + struct sh_keep * arr; + + if (keepcount > 0) + { + arr = SH_ALLOC (keepcount * sizeof(struct sh_keep)); + + while (count < keepcount && keep) + { + if ((now > keep->last) && ((unsigned long)(now - keep->last) <= keep->delay)) + { + memcpy(&(arr[count]), keep, sizeof(struct sh_keep)); + ++count; + prev = keep; + keep = keep->next; + } + else /* Too old or in future, delete it */ + { + if (keep != keeplist) + { + prev->next = keep->next; + sh_keep_free(keep); + keep = prev->next; + --keepcount; + } + else /* list head */ + { + keeplist = keep->next; + prev = keeplist; + sh_keep_free(keep); + keep = keeplist; + --keepcount; + } + } + } + + if (count > 0) + { + unsigned long i; + qsort(arr, count, sizeof(struct sh_keep), sh_keep_comp); + res = sh_string_copy(arr[0].label); + for (i = 1; i < count; ++i) + res = sh_string_add(res, arr[i].label); + } + SH_FREE(arr); + } + return res; +} + +struct sh_mkeep +{ + sh_string * label; /* label of match rule */ + pcre * rule; /* compiled regex for rule */ + struct sh_qeval * queue; /* assigned queue */ + struct sh_mkeep * next; +}; + +struct sh_mkeep * mkeep_list = NULL; + +static struct sh_qeval * find_queue(const char * str); + +static int sh_keep_match_add(const char * str, const char * queue, const char * pattern) +{ + unsigned int nfields = 1; /* seconds:label */ + size_t lengths[1]; + char * new = sh_util_strdup(str); + char ** splits = split_array_braced(new, _("CORRELATE"), &nfields, lengths); + + if (nfields == 1 && lengths[0] > 0) + { + struct sh_mkeep * mkeep = SH_ALLOC(sizeof(struct sh_mkeep)); + const char * error; + int erroffset; + struct sh_qeval * rqueue = NULL; + + mkeep->rule = pcre_compile(pattern, PCRE_NO_AUTO_CAPTURE, + &error, &erroffset, NULL); + if (!(mkeep->rule)) + { + sh_string * msg = sh_string_new(0); + sh_string_add_from_char(msg, _("Bad regex: ")); + sh_string_add_from_char(msg, pattern); + + SH_MUTEX_LOCK(mutex_thread_nolog); + sh_error_handle(SH_ERR_ERR, FIL__, __LINE__, 0, MSG_E_SUBGEN, + sh_string_str(msg), + _("sh_keep_match_add")); + SH_MUTEX_UNLOCK(mutex_thread_nolog); + sh_string_destroy(&msg); + + SH_FREE(splits); + SH_FREE(mkeep); + SH_FREE(new); + return -1; + } + + if (0 != strcmp(queue, _("trash"))) + { + + rqueue = find_queue(queue); + if (!rqueue) + { + pcre_free(mkeep->rule); + SH_FREE(splits); + SH_FREE(mkeep); + SH_FREE(new); + return -1; + } + } + + mkeep->queue = rqueue; + mkeep->label = sh_string_new_from_lchar(splits[0], strlen(splits[0])); + mkeep->next = mkeep_list; + mkeep_list = mkeep; + } + SH_FREE(new); + return 0; +} + +static void sh_keep_match_del() +{ + struct sh_mkeep * mkeep = mkeep_list; + while (mkeep) + { + mkeep_list = mkeep->next; + sh_string_destroy(&(mkeep->label)); + pcre_free(mkeep->rule); + mkeep = mkeep_list; + } + mkeep_list = NULL; +} + +static struct sh_mkeep ** dummy_mkeep; + +void sh_keep_match() +{ + if (mkeep_list) + { + sh_string * res = sh_keep_eval(); + + if (res) + { + struct sh_mkeep * mkeep = mkeep_list; + + dummy_mkeep = &mkeep; + + while (mkeep) + { + int val = pcre_exec(mkeep->rule, NULL, + sh_string_str(res), (int)sh_string_len(res), + 0, 0, NULL, 0); + if (val >= 0) + { + char * tmp; + SH_MUTEX_LOCK(mutex_thread_nolog); + tmp = sh_util_safe_name (sh_string_str(mkeep->label)); + sh_error_handle (mkeep->queue->severity, FIL__, __LINE__, 0, + MSG_LOGMON_COR, tmp); + SH_FREE(tmp); + SH_MUTEX_UNLOCK(mutex_thread_nolog); + } + mkeep = mkeep->next; + } + sh_string_destroy(&res); + } + } + return; +} + +/*--------------------------------------------------------------*/ + struct sh_geval /* Group of rules (may be a single rule) */ { @@ -97,4 +348,6 @@ int ovecnum; /* how many captured */ int captures; /* (captures+1)*3 required */ + int flags; /* bit flags */ + unsigned long delay; /* delay for keep rules */ zAVLTree * counterlist; /* counters if EVAL_SUM */ struct sh_qeval * queue; /* queue for this rule */ @@ -178,4 +431,6 @@ ng = SH_ALLOC(sizeof(struct sh_geval)); ng->label = sh_string_new_from_lchar(splits[0], lengths[0]); + ng->flags = RFL_ISGROUP; + ng->rule = group; ng->rule_extra = group_extra; @@ -384,5 +639,30 @@ } +char * get_keep(char * str, unsigned long * seconds) +{ + char * res = NULL; + char * endptr = NULL; + + unsigned int nfields = 2; /* seconds:label */ + size_t lengths[2]; + char * new = sh_util_strdup(str); + char ** splits = split_array_braced(new, _("KEEP"), &nfields, lengths); + + if (nfields == 2 && lengths[0] > 0 && lengths[1] > 0) + { + *seconds = strtoul(splits[0], &endptr, 10); + if ((endptr == '\0' || endptr != splits[0]) && (*seconds != ULONG_MAX)) + { + res = sh_util_strdup(splits[1]); + } + } + if (splits) + SH_FREE(splits); + SH_FREE(new); + return res; +} + static struct sh_qeval ** dummy_queue; +static char ** dummy_dstr; int sh_eval_radd (const char * str) @@ -398,10 +678,16 @@ unsigned int nfields = 2; /* queue:regex */ size_t lengths[2]; - char * new = sh_util_strdup(str); + char * new = sh_util_strdup(str); char ** splits = split_array(new, &nfields, ':', lengths); + int qpos = 0; + int rpos = 1; + unsigned long dsec = 0; + char * dstr = NULL; + dummy_queue = &queue; - - if (nfields != 2) + dummy_dstr = &dstr; + + if (nfields < 2 || nfields > 3) { SH_FREE(splits); @@ -410,16 +696,33 @@ } - if (0 != strcmp(splits[0], _("trash"))) - { - queue = find_queue(splits[0]); - if (!queue) - { - SH_FREE(splits); - SH_FREE(new); - return -1; - } - } - - rule = pcre_compile(splits[1], 0, + if (nfields == 3) + { + /* KEEP(nsec):queue:regex + */ + dstr = get_keep(splits[0], &dsec); + if (!dstr) + { + /* CORRELATE:queue:regex + */ + int retval = sh_keep_match_add(splits[0], splits[1], splits[2]); + SH_FREE(splits); + SH_FREE(new); + return retval; + } + ++qpos; ++rpos; + } + + if (0 != strcmp(splits[qpos], _("trash"))) + { + queue = find_queue(splits[qpos]); + if (!queue) + { + SH_FREE(splits); + SH_FREE(new); + return -1; + } + } + + rule = pcre_compile(splits[rpos], 0, &error, &erroffset, NULL); if (!rule) @@ -427,5 +730,5 @@ sh_string * msg = sh_string_new(0); sh_string_add_from_char(msg, _("Bad regex: ")); - sh_string_add_from_char(msg, splits[1]); + sh_string_add_from_char(msg, splits[rpos]); SH_MUTEX_LOCK(mutex_thread_nolog); @@ -446,6 +749,10 @@ { char * emsg = SH_ALLOC(SH_ERRBUF_SIZE); - sl_snprintf(emsg, SH_ERRBUF_SIZE, _("Adding rule: |%s| with %d captures"), - splits[1], captures); + if (dstr) + sl_snprintf(emsg, SH_ERRBUF_SIZE, _("Adding rule: |%s| with %d captures, keep(%lu,%s)"), + splits[rpos], captures, dsec, dstr); + else + sl_snprintf(emsg, SH_ERRBUF_SIZE, _("Adding rule: |%s| with %d captures"), + splits[rpos], captures); SH_MUTEX_LOCK(mutex_thread_nolog); sh_error_handle(SH_ERR_ALL, FIL__, __LINE__, 0, MSG_E_SUBGEN, @@ -455,5 +762,5 @@ } - DEBUG("adding rule: |%s| with %d captures\n", splits[1], captures); + DEBUG("adding rule: |%s| with %d captures\n", splits[rpos], captures); SH_FREE(splits); @@ -462,4 +769,7 @@ nr = SH_ALLOC(sizeof(struct sh_geval)); nr->label = NULL; + nr->flags = RFL_ISRULE; + nr->delay = 0; + nr->rule = rule; nr->rule_extra = rule_extra; @@ -473,4 +783,13 @@ nr->gnext = NULL; + + if (dstr) + { + nr->label = sh_string_new_from_lchar(dstr, strlen(dstr)); + nr->flags |= RFL_KEEP; + nr->delay = dsec; + SH_FREE(dstr); + } + /* * If there is an open group, add it to its @@ -513,4 +832,6 @@ if (0 != sh_eval_hadd("^.*")) { + if (nr->label) + sh_string_destroy(&(nr->label)); SH_FREE(nr->ovector); SH_FREE(nr); @@ -559,4 +880,6 @@ else { + if (nr->label) + sh_string_destroy(&(nr->label)); SH_FREE(nr->ovector); SH_FREE(nr); @@ -623,4 +946,8 @@ } + sh_keep_destroy(); + sh_keep_match_del(); + + return; } @@ -636,8 +963,9 @@ static struct sh_geval ** dummy1; -static struct sh_geval * test_rule (struct sh_geval * rule, sh_string *msg) +static struct sh_geval * test_rule (struct sh_geval * rule, sh_string *msg, time_t tstamp) { int res; - volatile int count; + volatile int count; + volatile time_t timestamp = tstamp; dummy1 = &rule; @@ -674,6 +1002,10 @@ { char * emsg = SH_ALLOC(SH_ERRBUF_SIZE); - sl_snprintf(emsg, SH_ERRBUF_SIZE, _("Rule %d matches, result = %d"), - count, res); + if ( rule->flags & RFL_KEEP ) + sl_snprintf(emsg, SH_ERRBUF_SIZE, _("Rule %d matches, result = %d (keep)"), + count, res); + else + sl_snprintf(emsg, SH_ERRBUF_SIZE, _("Rule %d matches, result = %d"), + count, res); SH_MUTEX_LOCK(mutex_thread_nolog); sh_error_handle(SH_ERR_ALL, FIL__, __LINE__, 0, MSG_E_SUBGEN, @@ -682,5 +1014,12 @@ SH_FREE(emsg); } - DEBUG("debug: rule %d matches, result = %d\n", count, res); + + if ( rule->flags & RFL_KEEP ) + { + DEBUG("debug: rule %d matches (keep)\n", count); + sh_keep_add(rule->label, rule->delay, + timestamp == 0 ? time(NULL) : timestamp); + } + break; /* return the matching rule; ovector is filled in */ } @@ -698,4 +1037,5 @@ } DEBUG("debug: rule %d did not match\n", count); + rule = rule->nextrule; ++count; } while (rule); @@ -714,5 +1054,6 @@ static struct sh_geval * test_grules (struct sh_heval * host, - sh_string *msg) + sh_string * msg, + time_t timestamp) { struct sh_geval * result = NULL; @@ -726,7 +1067,7 @@ DEBUG("debug: if group\n"); do { - if(group->label != NULL) + if( (group->label != NULL) && (0 != (group->flags & RFL_ISGROUP))) { - /* this is a rule group; only groups have labels */ + /* this is a rule group */ if (flag_err_debug == SL_TRUE) @@ -747,5 +1088,5 @@ 0, 0, NULL, 0) >= 0) { - result = test_rule(group->nextrule, msg); + result = test_rule(group->nextrule, msg, timestamp); if (result) break; @@ -770,5 +1111,5 @@ DEBUG("debug: else (single rule)\n"); - result = test_rule(group, msg); + result = test_rule(group, msg, timestamp); if (result) break; @@ -783,5 +1124,6 @@ */ static struct sh_geval * find_rule (sh_string *host, - sh_string *msg) + sh_string *msg, + time_t timestamp) { struct sh_geval * result = NULL; @@ -796,5 +1138,5 @@ { /* matching host, check rules/groups of rules */ - result = test_grules(hlist, msg); + result = test_grules(hlist, msg, timestamp); if (result) break; @@ -994,5 +1336,6 @@ { struct sh_geval * rule = find_rule (record->host, - record->message); + record->message, + record->timestamp); if (rule) Index: trunk/src/sh_string.c =================================================================== --- trunk/src/sh_string.c (revision 259) +++ trunk/src/sh_string.c (revision 260) @@ -238,4 +238,27 @@ { return split_array_ws_int (line, nfields, lengths, SH_SPLIT_LIST); +} + +/* return a split() of a list contained in 'PREFIX\s*( list ).*' + */ +char ** split_array_braced (char *line, const char * prefix, + unsigned int * nfields, size_t * lengths) +{ + char * s = line; + char * p; + + while ( *s && isspace((int)*s) ) ++s; + if (prefix && 0 != strncmp(s, prefix, strlen(prefix))) + return NULL; + s = &s[strlen(prefix)]; + while ( *s && isspace((int)*s) ) ++s; + if (!s || (*s != '(')) + return NULL; + ++s; + p = strchr(s, ')'); + if (!p || (*p == *s)) + return NULL; + *p = '\0'; + return split_array_list (s, nfields, lengths); }