diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbAnalyzer.java b/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbAnalyzer.java
index 65d021a34b255bda3f08155be05838fb7bae48e0..1845ac5a34c9f1bfac64f409963db55ada428968 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbAnalyzer.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbAnalyzer.java
@@ -101,11 +101,8 @@ public class JGibbAnalyzer extends Analyzer {
// discovered (line does not start with a tab). Cut off
// after k words are gathered.
String nextLine;
- int words = 0;
while ((nextLine = nextLine()) != null) {
if (nextLine.startsWith("\t")) {
- if (words > Constants.K_TOPIC_WORDS)
- continue;
String[] parts = nextLine.trim().split("\\s+");
try {
Word word = wordMap.get(parts[0]);
@@ -113,7 +110,6 @@ public class JGibbAnalyzer extends Analyzer {
Constants.LIKELINESS_PRECISION);
TopicWord topicWord = new TopicWord(word, likeliness);
topicWords.add(topicWord);
- words++;
} catch (NumberFormatException e) {
log.error("could not parse number in line: " + nextLine);
} catch (ArrayIndexOutOfBoundsException e) {
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ModelingCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ModelingCommand.java
index 8f0b79fa54d5e2135c7f0e9600e40865e3831496..25d948904b78de46aff916dcc581f4485a6b4e43 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ModelingCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ModelingCommand.java
@@ -107,7 +107,7 @@ public class ModelingCommand implements Command {
while (indexIter.hasNext() && topicRefsListIter.hasNext()) {
// get article from database
String id = indexIter.next();
- ArticleFull article = dbArticles.getSingle(MongoUtils.objectId(id));
+ ArticleFull article = dbArticles.getSingle(MongoUtils.objectId(id), "processedText");
if (article == null) {
log.error("no article found in db for id " + id);
continue;
diff --git a/vipra-ui/app/html/index-menu.html b/vipra-ui/app/html/index-menu.html
deleted file mode 100644
index aa1e3f4110b2f193a1dc3ee4ac68ad159ce9e254..0000000000000000000000000000000000000000
--- a/vipra-ui/app/html/index-menu.html
+++ /dev/null
@@ -1 +0,0 @@
-<li><a>Advanced</a></li>
\ No newline at end of file
diff --git a/vipra-ui/app/html/topics/show.html b/vipra-ui/app/html/topics/show.html
index 9ca80d168a892da0382fc54057687b90acad97bb..3f11638f931fdedfb5caa7f57bd70abf7297504c 100644
--- a/vipra-ui/app/html/topics/show.html
+++ b/vipra-ui/app/html/topics/show.html
@@ -38,7 +38,7 @@
</thead>
<tbody>
<tr ng-repeat="word in ::topic.words">
- <td><a ui-sref="words.show({id:word.word})" ng-bind="word.word"></a></td>
+ <td><a ui-sref="words.show({id:word.id})" ng-bind="word.id"></a></td>
<td ng-bind="word.likeliness"></td>
</tr>
</tbody>
diff --git a/vipra-ui/app/index.html b/vipra-ui/app/index.html
index e1232a7dc4598262e9687defb374b03946a94a17..215b50a1fc6e371451469ea385141febcce31fcb 100644
--- a/vipra-ui/app/index.html
+++ b/vipra-ui/app/index.html
@@ -55,7 +55,6 @@
<li ui-sref-active="active"><a ui-sref="topics.index">Topics</a></li>
<li ui-sref-active="active"><a ui-sref="words.index">Words</a></li>
</ul>
- <ul class="nav navbar-nav navbar-right" ui-view="menu"></ul>
</div><!-- /.navbar-collapse -->
</div><!-- /.container-fluid -->
</nav>
diff --git a/vipra-ui/app/js/directives.js b/vipra-ui/app/js/directives.js
index 4db6af286da54dc9f4b2f7966d18e2fa75380a4b..9ed3dd5a01c234025f4dc9f7b0c82e329c69b05a 100644
--- a/vipra-ui/app/js/directives.js
+++ b/vipra-ui/app/js/directives.js
@@ -76,6 +76,7 @@
edges = [],
articleColor = '#BBC9D2',
topicColor = '#DBB234',
+ wordColor = '#B6C7BD',
container = $element.find("#visgraph")[0];
$scope.articleColor = articleColor;
@@ -134,6 +135,41 @@
}
};
};
+
+ var wordNode = function(word) {
+ return {
+ id: ++id,
+ title: word.word,
+ label: word.word,
+ type: 'word',
+ word: word.word,
+ color: {
+ background: wordColor,
+ highlight: { background: wordColor }
+ }
+ };
+ };
+
+ var constructor = function(result, key, nodeFunction) {
+ if(result.data && result.data[key]) {
+ var data = result.data[key],
+ newNodes = [],
+ newEdges = [];
+ for(var i = 0; i < data.length; i++) {
+ var current = data[i];
+ if(ids.hasOwnProperty(current.id)) {
+ if(!$scope.nodes.get(ids[current.id]).loaded)
+ newEdges.push({from:ids[current.id], to:node.id});
+ } else {
+ newNodes.push(nodeFunction(current));
+ newEdges.push({from:id, to:node.id});
+ ids[current.id] = id;
+ }
+ }
+ if(newNodes.length) $scope.nodes.add(newNodes);
+ if(newEdges.length) $scope.edges.add(newEdges);
+ }
+ };
// on node select
var selectTimeout;
@@ -145,46 +181,17 @@
if(node.type === 'article') {
// node is article, load article to get topics
ArticleFactory.get({id:node.article}, function(res) {
- if(res.data && res.data.topics) {
- var topics = res.data.topics,
- newNodes = [],
- newEdges = [];
- for(var i = 0; i < topics.length; i++) {
- var topic = topics[i].topic;
- if(ids.hasOwnProperty(topic.id)) {
- if(!$scope.nodes.get(ids[topic.id]).loaded)
- newEdges.push({from:node.id, to:ids[topic.id]});
- } else {
- newNodes.push(topicNode(topic));
- newEdges.push({from:node.id, to:id});
- ids[topic.id] = id;
- }
- }
- if(newNodes.length) $scope.nodes.add(newNodes);
- if(newEdges.length) $scope.edges.add(newEdges);
- }
+ constructor(res, 'topics', topicNode);
});
- } else {
- // node is topic, load topic to get articles
+ } else if(node.type === 'topic') {
+ // node is topic, load topic to get words
TopicFactory.get({id:node.topic}, function(res) {
- if(res.data && res.data.articles) {
- var articles = res.data.articles,
- newNodes = [],
- newEdges = [];
- for(var i = 0; i < articles.length; i++) {
- var article = articles[i];
- if(ids.hasOwnProperty(article.id)) {
- if(!$scope.nodes.get(ids[article.id]).loaded)
- newEdges.push({from:ids[article.id], to:node.id});
- } else {
- newNodes.push(articleNode(article));
- newEdges.push({from:id, to:node.id});
- ids[article.id] = id;
- }
- }
- if(newNodes.length) $scope.nodes.add(newNodes);
- if(newEdges.length) $scope.edges.add(newEdges);
- }
+ constructor(res, 'words', wordNode);
+ });
+ } else if(node.type === 'word') {
+ // node is word, load word to get topics
+ WordFactory.get({id:node.word}, function(res) {
+ constructor(res, 'topics', topicNode);
});
}
node.loaded = true;
@@ -199,8 +206,10 @@
var node = $scope.nodes.get(props.nodes[0]);
if(node.type === 'article')
$state.transitionTo('articles.show', {id:node.article});
- else
+ else if(node.type === 'topic')
$state.transitionTo('topics.show', {id:node.topic});
+ else if(node.type === 'word')
+ $state.transitionTo('words.show', {id:node.word});
};
// watch for changes to model
@@ -210,8 +219,10 @@
// root node
if($scope.visType === 'articles')
nodes.push(articleNode($scope.ngModel));
- else
+ else if($scope.visType === 'topics')
nodes.push(topicNode($scope.ngModel));
+ else if($scope.visType === 'words')
+ nodes.push(wordNode($scope.ngModel));
ids[$scope.ngModel.id] = id;
// add nodes and edges
diff --git a/vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java b/vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java
index 4380c73381077fca6bee3b2e643552bc71ff2d8b..8a17356a72544aaf547474f3b41250fe0d104b70 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java
@@ -47,7 +47,7 @@ public class ArticleFull extends FileModel<ObjectId> implements Serializable {
private String text;
@ElasticIndex("text")
- @QueryIgnore(multi = true)
+ @QueryIgnore(all = true)
private String processedText;
private String url;
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TopicWord.java b/vipra-util/src/main/java/de/vipra/util/model/TopicWord.java
index bd79ed2b418461b4f33432d7537c765e479e80a1..ad75e6f5b947e577a7d68281babf051af087f92e 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/TopicWord.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/TopicWord.java
@@ -17,7 +17,7 @@ public class TopicWord implements Comparable<TopicWord>, Serializable {
@JsonIgnore
private Word word;
- @JsonProperty("word")
+ @JsonProperty("id")
private String wordString;
private Double likeliness;
diff --git a/vipra-util/src/main/java/de/vipra/util/service/MongoService.java b/vipra-util/src/main/java/de/vipra/util/service/MongoService.java
index 519d30efc8bd47809b2ca490437b7558b2ea5208..912be211f6e54187e9459023cb106fcd87b535b8 100644
--- a/vipra-util/src/main/java/de/vipra/util/service/MongoService.java
+++ b/vipra-util/src/main/java/de/vipra/util/service/MongoService.java
@@ -2,10 +2,7 @@ package de.vipra.util.service;
import java.lang.reflect.Field;
import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
import org.mongodb.morphia.Datastore;
import org.mongodb.morphia.query.Query;
@@ -54,7 +51,7 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
if (cache == null || (fields != null && fields.length > 0) || !cache.contains(id)) {
Query<Type> q = datastore.createQuery(clazz).field("_id").equal(id);
if (fields != null && fields.length > 0)
- q.retrievedFields(true, setMinus(fields, ignoredFieldsSingle));
+ q.retrievedFields(true, fields);
else if (ignoredFieldsSingle.length > 0)
q.retrievedFields(false, ignoredFieldsSingle);
t = q.get();
@@ -75,14 +72,7 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
public List<Type> getMultiple(Integer skip, Integer limit, String sortBy, Pair<String, Object> criteria,
String... fields) {
return getMultiple(
- QueryBuilder.builder().skip(skip).limit(limit).sortBy(sortBy).criteria(criteria).fields(true, fields));
- }
-
- @Override
- public List<Type> getMultiple(Integer skip, Integer limit, String sortBy, Pair<String, Object> criteria,
- boolean defaultIgnore, String... fields) {
- return getMultiple(QueryBuilder.builder().skip(skip).limit(limit).sortBy(sortBy).fields(true, fields)
- .criteria(criteria).defaultIgnore(defaultIgnore));
+ QueryBuilder.builder().skip(skip).limit(limit).sortBy(sortBy).fields(true, fields).criteria(criteria));
}
@Override
@@ -100,12 +90,8 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
if (builder.getFields() != null) {
String[] fields = builder.getFields();
if (builder.isInclude()) {
- if (builder.isDefaultIgnore() && ignoredFieldsMulti.length > 0)
- fields = setMinus(fields, ignoredFieldsMulti);
q.retrievedFields(true, fields);
} else {
- if (builder.isDefaultIgnore() && ignoredFieldsMulti.length > 0)
- fields = setPlus(fields, ignoredFieldsMulti);
q.retrievedFields(false, fields);
}
} else if (ignoredFieldsMulti.length > 0) {
@@ -173,24 +159,4 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
return new MongoService<Type, IdType>(mongo, clazz);
}
- private String[] setMinus(String[] a, String[] b) {
- if (a != null && b != null) {
- Set<String> sa = new HashSet<>(Arrays.asList(a));
- Set<String> sb = new HashSet<>(Arrays.asList(b));
- sa.removeAll(sb);
- return sa.toArray(new String[sa.size()]);
- }
- return a;
- }
-
- private String[] setPlus(String[] a, String[] b) {
- if (a != null && b != null) {
- Set<String> sa = new HashSet<>(Arrays.asList(a));
- Set<String> sb = new HashSet<>(Arrays.asList(b));
- sa.addAll(sb);
- return sa.toArray(new String[sa.size()]);
- }
- return a;
- }
-
}
diff --git a/vipra-util/src/main/java/de/vipra/util/service/Service.java b/vipra-util/src/main/java/de/vipra/util/service/Service.java
index d83fe03e223c1384538df773b06b4106f58f6afb..a02b434337c2c016be9ebce5e2254f8e4097888c 100644
--- a/vipra-util/src/main/java/de/vipra/util/service/Service.java
+++ b/vipra-util/src/main/java/de/vipra/util/service/Service.java
@@ -43,12 +43,6 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
List<Type> getMultiple(Integer skip, Integer limit, String sortBy, Pair<String, Object> criteria, String... fields)
throws E;
- /**
- * @see {@link Service#getMultiple(QueryBuilder)}
- */
- List<Type> getMultiple(Integer skip, Integer limit, String sortBy, Pair<String, Object> criteria,
- boolean noDefaultIgnore, String... fields) throws E;
-
/**
* Returns multiple entities from the database.
*
@@ -147,7 +141,6 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
private List<Pair<String, Object>> criteria;
private String[] fields;
private boolean include;
- private boolean defaultIgnore = true;
private QueryBuilder() {}
@@ -242,19 +235,6 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
return this;
}
- /**
- * Whether to use default ignoring fields or not. Some fields are
- * ignored by default, as per defined in the models.
- *
- * @param defaultIgnore
- * true to use default ignoring
- * @return QueryBuilder instance
- */
- public QueryBuilder defaultIgnore(boolean defaultIgnore) {
- this.defaultIgnore = defaultIgnore;
- return this;
- }
-
public Integer getSkip() {
return skip;
}
@@ -279,10 +259,6 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
return fields;
}
- public boolean isDefaultIgnore() {
- return defaultIgnore;
- }
-
}
}