Repository: incubator-senssoft-distill Updated Branches: refs/heads/master 114f3375a -> 2707c50f5
Fixed bugs in sankey and validated request args output Project: http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/commit/2707c50f Tree: http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/tree/2707c50f Diff: http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/diff/2707c50f Branch: refs/heads/master Commit: 2707c50f5560faa68300cb42db7f05681ad3cd62 Parents: 114f337 Author: msbeard <msbe...@apache.org> Authored: Mon Sep 11 10:53:42 2017 -0400 Committer: msbeard <msbe...@apache.org> Committed: Mon Sep 11 10:53:42 2017 -0400 ---------------------------------------------------------------------- distill/algorithms/graphs/graph.py | 59 ++++++++++++++++++++++++--------- distill/app.py | 31 +++++++++++------ 2 files changed, 64 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/2707c50f/distill/algorithms/graphs/graph.py ---------------------------------------------------------------------- diff --git a/distill/algorithms/graphs/graph.py b/distill/algorithms/graphs/graph.py index 4b282ea..ce463f8 100755 --- a/distill/algorithms/graphs/graph.py +++ b/distill/algorithms/graphs/graph.py @@ -31,7 +31,8 @@ class GraphAnalytics (object): def generate_graph(app, app_type='logs', log_type='raw', - target_events=[], + targets=[], + events=[], time_range=['now-1h', 'now'], size=20): """ @@ -62,15 +63,41 @@ class GraphAnalytics (object): ] # Filtering - # should_query = [] - if (target_events): - for event in target_events: - res = { - "term": { - "type": event - } + should_query = [] + must_query = [] + + # Include these events in the request + if events: + include_events = { + "terms": { + "type.keyword": events + } + } + filter_query.append(include_events) + + target_in = targets[0] + target_out = targets[1] + + if target_in: + include_targets = { + "terms": { + "target.keyword": target_in } - filter_query.append(res) + } + + filter_query.append(include_targets) + + # Remove these elementids from result set + for target in target_out: + res = { + "term": { + "target.keyword": target + } + } + must_not_query.append(res) + + # Finish off should query + # must_query.append({"bool": {"should": should_query}}) # Sort By Time sort_query = [ @@ -98,7 +125,7 @@ class GraphAnalytics (object): session_query = { "terms": { "field": "sessionID.keyword", - "min_doc_count": 1 + # "min_doc_count": 1 } } @@ -108,14 +135,14 @@ class GraphAnalytics (object): target_query = { "terms": { "field": "target.keyword", - "min_doc_count": 1, + # "min_doc_count": 1, "size": size }, "aggs": { "events": { "terms": { "field": "type.keyword", - "min_doc_count": 1, + # "min_doc_count": 1, "size": size } }, @@ -142,10 +169,11 @@ class GraphAnalytics (object): "sort": sort_query, "query": { "bool": { - # "must": must_match, + # "must": must_query, # "should": should_query, "filter": filter_query, "must_not": must_not_query, + # "minimum_should_match": len(should_query) - 1 } }, "_source": { @@ -165,7 +193,6 @@ class GraphAnalytics (object): # return query # Process Aggregate Results response = es.search(app, doc_type=app_type, body=query, size=0) - # Only want to look at aggregations sessions = response['aggregations']['sessions']['buckets'] # allSessions = { x['key']: [] for x in sessions } @@ -285,10 +312,10 @@ class GraphAnalytics (object): nodename2 = node2['target'] seqID = '%s->%s' % (nodename1, nodename2) - print(seqID) + #print(seqID) if nodename1 != nodename2: #double check again for self-loops - print(node1) + #print(node1) link = { 'sequenceID': seqID, 'sourceName': nodename1, http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/2707c50f/distill/app.py ---------------------------------------------------------------------- diff --git a/distill/app.py b/distill/app.py old mode 100644 new mode 100755 index d07396b..76bfb78 --- a/distill/app.py +++ b/distill/app.py @@ -83,26 +83,37 @@ def sankey(app_id): # Time range using date math from_range = 'now-15m' to_range = 'now' - ts_range = [from_range, to_range] - if 'from' in request.args: + + if 'from' in request.args and request.args.get('from') != '': from_range = request.args.get('from') - if 'to' in request.args: - to_range = request.args.get('to') - ts_range = [from_range, to_range] + if 'to' in request.args and request.args.get('to') != '': + to_range = request.args.get('to') + + ts_range = [from_range, to_range] # Size size = 20 - if 'size' in request.args: + if 'size' in request.args and request.args.get('size') != '': size = request.args.get('size') - # target events + # events (event_in) events = [] - if 'event' in request.args: - events.append(request.args.get('event')) + if 'event' in request.args and request.args.get('event') != '': + events = request.args.get('event').split(',') + + # filter in/out targets + target_in = [] + if 'target_in' in request.args and request.args.get('target_in') != '': + target_in = request.args.get('target_in').split(',') + + target_out = [] + if 'target_out' in request.args and request.args.get('target_out') != '': + target_out = request.args.get('target_out').split(',') return jsonify(GraphAnalytics.generate_graph(app_id, - target_events=events, + targets=[target_in, target_out], + events=events, time_range=ts_range, size=size))