diff --git a/topicexplorer/lib/util.py b/topicexplorer/lib/util.py index 96a0929c..c526b74a 100644 --- a/topicexplorer/lib/util.py +++ b/topicexplorer/lib/util.py @@ -72,3 +72,12 @@ def contains_pattern(directory, pattern): return True except StopIteration: return False + +def filter_until(predicate, n, ls): + ls = list(ls) + while n > 0 and ls: + item = ls.pop(0) + if predicate(item): + yield item + n -= 1 + diff --git a/topicexplorer/server.py b/topicexplorer/server.py index 38508031..bf2f378e 100644 --- a/topicexplorer/server.py +++ b/topicexplorer/server.py @@ -19,11 +19,14 @@ from bottle import request, response, route, run, static_file from topicexplorer.lib.ssl import SSLWSGIRefServer +from topicexplorer.lib.util import filter_until import numpy as np import pystache import topicexplorer.lib.color as colorlib + +stopwords = None def _set_acao_headers(f): """ Decorator to set Access-Control-Allow-Origin headers to enable cross-InPhO @@ -202,7 +205,11 @@ def topics(): # populate word values data = lda_v.topics() for i,topic in enumerate(data): - js[str(i)].update({'words' : dict([(w, p) for w,p in topic[:10]])}) + if stopwords: + js[str(i)].update({'words' : dict([(w, p) for w,p in + filter_until(lambda w_p: w_p[0] not in stopwords, 20, topic)])}) + else: + js[str(i)].update({'words' : dict([(w, p) for w,p in topic[:20]])}) return json.dumps(js) @@ -265,7 +272,12 @@ def get_docs(docs=None, id_as_key=False, query=None): def main(args): global context_type, lda_c, lda_m, lda_v, label, id_fn - + global stopwords + with open('../corpus.stopwords') as swfile: + stopwords = [] + for line in swfile: + stopwords.append(line.strip()) + # load in the configuration file config = ConfigParser({ 'certfile' : None,