problem with topic insertion

vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbAnalyzer.java

@@ -101,11 +101,8 @@ public class JGibbAnalyzer extends Analyzer {
 					// discovered (line does not start with a tab). Cut off
 					// after k words are gathered.
 					String nextLine;
-					int words = 0;
 					while ((nextLine = nextLine()) != null) {
 						if (nextLine.startsWith("\t")) {
-							if (words > Constants.K_TOPIC_WORDS)
-								continue;
 							String[] parts = nextLine.trim().split("\\s+");
 							try {
 								Word word = wordMap.get(parts[0]);
@@ -113,7 +110,6 @@ public class JGibbAnalyzer extends Analyzer {
 										Constants.LIKELINESS_PRECISION);
 								TopicWord topicWord = new TopicWord(word, likeliness);
 								topicWords.add(topicWord);
-								words++;
 							} catch (NumberFormatException e) {
 								log.error("could not parse number in line: " + nextLine);
 							} catch (ArrayIndexOutOfBoundsException e) {

vipra-cmd/src/main/java/de/vipra/cmd/option/ModelingCommand.java

@@ -107,7 +107,7 @@ public class ModelingCommand implements Command {
 		while (indexIter.hasNext() && topicRefsListIter.hasNext()) {
 			// get article from database
 			String id = indexIter.next();
-			ArticleFull article = dbArticles.getSingle(MongoUtils.objectId(id));
+			ArticleFull article = dbArticles.getSingle(MongoUtils.objectId(id), "processedText");
 			if (article == null) {
 				log.error("no article found in db for id " + id);
 				continue;

vipra-ui/app/html/index-menu.html

-<li><a>Advanced</a></li>
\ No newline at end of file

vipra-ui/app/html/topics/show.html

@@ -38,7 +38,7 @@
   </thead>
   <tbody>
     <tr ng-repeat="word in ::topic.words">
-      <td><a ui-sref="words.show({id:word.word})" ng-bind="word.word"></a></td>
+      <td><a ui-sref="words.show({id:word.id})" ng-bind="word.id"></a></td>
       <td ng-bind="word.likeliness"></td>
     </tr>
   </tbody>

vipra-ui/app/index.html

@@ -55,7 +55,6 @@
             <li ui-sref-active="active"><a ui-sref="topics.index">Topics</a></li>
             <li ui-sref-active="active"><a ui-sref="words.index">Words</a></li>
           </ul>
-          <ul class="nav navbar-nav navbar-right" ui-view="menu"></ul>
         </div><!-- /.navbar-collapse -->
       </div><!-- /.container-fluid -->
     </nav>

vipra-ui/app/js/directives.js

@@ -76,6 +76,7 @@
             edges = [],
             articleColor = '#BBC9D2',
             topicColor = '#DBB234',
+            wordColor = '#B6C7BD',
             container = $element.find("#visgraph")[0];
 
         $scope.articleColor = articleColor;
@@ -134,6 +135,41 @@
             }
           };
         };
+
+        var wordNode = function(word) {
+          return {
+            id: ++id,
+            title: word.word,
+            label: word.word,
+            type: 'word',
+            word: word.word,
+            color: {
+              background: wordColor,
+              highlight: { background: wordColor }
+            }
+          };
+        };
+
+        var constructor = function(result, key, nodeFunction) {
+          if(result.data && result.data[key]) {
+            var data = result.data[key],
+                newNodes = [],
+                newEdges = [];
+            for(var i = 0; i < data.length; i++) {
+              var current = data[i];
+              if(ids.hasOwnProperty(current.id)) {
+                if(!$scope.nodes.get(ids[current.id]).loaded)
+                  newEdges.push({from:ids[current.id], to:node.id});
+              } else {
+                newNodes.push(nodeFunction(current));
+                newEdges.push({from:id, to:node.id});
+                ids[current.id] = id;
+              }
+            }
+            if(newNodes.length) $scope.nodes.add(newNodes);
+            if(newEdges.length) $scope.edges.add(newEdges);
+          }
+        };
         
         // on node select
         var selectTimeout;
@@ -145,46 +181,17 @@
               if(node.type === 'article') {
                 // node is article, load article to get topics
                 ArticleFactory.get({id:node.article}, function(res) {
-                  if(res.data && res.data.topics) {
-                    var topics = res.data.topics,
-                        newNodes = [],
-                        newEdges = [];
-                    for(var i = 0; i < topics.length; i++) {
-                      var topic =  topics[i].topic;
-                      if(ids.hasOwnProperty(topic.id)) {
-                        if(!$scope.nodes.get(ids[topic.id]).loaded)
-                          newEdges.push({from:node.id, to:ids[topic.id]});
-                      } else {
-                        newNodes.push(topicNode(topic));
-                        newEdges.push({from:node.id, to:id});
-                        ids[topic.id] = id;
-                      }
-                    }
-                    if(newNodes.length) $scope.nodes.add(newNodes);
-                    if(newEdges.length) $scope.edges.add(newEdges);
-                  }
+                  constructor(res, 'topics', topicNode);
                 });
-              } else {
-                // node is topic, load topic to get articles
+              } else if(node.type === 'topic') {
+                // node is topic, load topic to get words
                 TopicFactory.get({id:node.topic}, function(res) {
-                  if(res.data && res.data.articles) {
-                    var articles = res.data.articles,
-                        newNodes = [],
-                        newEdges = [];
-                    for(var i = 0; i < articles.length; i++) {
-                      var article = articles[i];
-                      if(ids.hasOwnProperty(article.id)) {
-                        if(!$scope.nodes.get(ids[article.id]).loaded)
-                          newEdges.push({from:ids[article.id], to:node.id});
-                      } else {
-                        newNodes.push(articleNode(article));
-                        newEdges.push({from:id, to:node.id});
-                        ids[article.id] = id;
-                      }
-                    }
-                    if(newNodes.length) $scope.nodes.add(newNodes);
-                    if(newEdges.length) $scope.edges.add(newEdges);
-                  }
+                  constructor(res, 'words', wordNode);
+                });
+              } else if(node.type === 'word') {
+                // node is word, load word to get topics
+                WordFactory.get({id:node.word}, function(res) {
+                  constructor(res, 'topics', topicNode);
                 });
               }
               node.loaded = true;
@@ -199,8 +206,10 @@
           var node = $scope.nodes.get(props.nodes[0]);
           if(node.type === 'article')
             $state.transitionTo('articles.show', {id:node.article});
-          else
+          else if(node.type === 'topic')
             $state.transitionTo('topics.show', {id:node.topic});
+          else if(node.type === 'word')
+            $state.transitionTo('words.show', {id:node.word});
         };
 
         // watch for changes to model
@@ -210,8 +219,10 @@
           // root node
           if($scope.visType === 'articles')
             nodes.push(articleNode($scope.ngModel));
-          else
+          else if($scope.visType === 'topics')
             nodes.push(topicNode($scope.ngModel));
+          else if($scope.visType === 'words')
+            nodes.push(wordNode($scope.ngModel));
           ids[$scope.ngModel.id] = id;
 
           // add nodes and edges

vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java

@@ -47,7 +47,7 @@ public class ArticleFull extends FileModel<ObjectId> implements Serializable {
 	private String text;
 
 	@ElasticIndex("text")
-	@QueryIgnore(multi = true)
+	@QueryIgnore(all = true)
 	private String processedText;
 
 	private String url;

vipra-util/src/main/java/de/vipra/util/model/TopicWord.java

@@ -17,7 +17,7 @@ public class TopicWord implements Comparable<TopicWord>, Serializable {
 	@JsonIgnore
 	private Word word;
 
-	@JsonProperty("word")
+	@JsonProperty("id")
 	private String wordString;
 
 	private Double likeliness;

vipra-util/src/main/java/de/vipra/util/service/MongoService.java

@@ -2,10 +2,7 @@ package de.vipra.util.service;
 
 import java.lang.reflect.Field;
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 
 import org.mongodb.morphia.Datastore;
 import org.mongodb.morphia.query.Query;
@@ -54,7 +51,7 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
 		if (cache == null || (fields != null && fields.length > 0) || !cache.contains(id)) {
 			Query<Type> q = datastore.createQuery(clazz).field("_id").equal(id);
 			if (fields != null && fields.length > 0)
-				q.retrievedFields(true, setMinus(fields, ignoredFieldsSingle));
+				q.retrievedFields(true, fields);
 			else if (ignoredFieldsSingle.length > 0)
 				q.retrievedFields(false, ignoredFieldsSingle);
 			t = q.get();
@@ -75,14 +72,7 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
 	public List<Type> getMultiple(Integer skip, Integer limit, String sortBy, Pair<String, Object> criteria,
 			String... fields) {
 		return getMultiple(
-				QueryBuilder.builder().skip(skip).limit(limit).sortBy(sortBy).criteria(criteria).fields(true, fields));
-	}
-
-	@Override
-	public List<Type> getMultiple(Integer skip, Integer limit, String sortBy, Pair<String, Object> criteria,
-			boolean defaultIgnore, String... fields) {
-		return getMultiple(QueryBuilder.builder().skip(skip).limit(limit).sortBy(sortBy).fields(true, fields)
-				.criteria(criteria).defaultIgnore(defaultIgnore));
+				QueryBuilder.builder().skip(skip).limit(limit).sortBy(sortBy).fields(true, fields).criteria(criteria));
 	}
 
 	@Override
@@ -100,12 +90,8 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
 		if (builder.getFields() != null) {
 			String[] fields = builder.getFields();
 			if (builder.isInclude()) {
-				if (builder.isDefaultIgnore() && ignoredFieldsMulti.length > 0)
-					fields = setMinus(fields, ignoredFieldsMulti);
 				q.retrievedFields(true, fields);
 			} else {
-				if (builder.isDefaultIgnore() && ignoredFieldsMulti.length > 0)
-					fields = setPlus(fields, ignoredFieldsMulti);
 				q.retrievedFields(false, fields);
 			}
 		} else if (ignoredFieldsMulti.length > 0) {
@@ -173,24 +159,4 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
 		return new MongoService<Type, IdType>(mongo, clazz);
 	}
 
-	private String[] setMinus(String[] a, String[] b) {
-		if (a != null && b != null) {
-			Set<String> sa = new HashSet<>(Arrays.asList(a));
-			Set<String> sb = new HashSet<>(Arrays.asList(b));
-			sa.removeAll(sb);
-			return sa.toArray(new String[sa.size()]);
-		}
-		return a;
-	}
-
-	private String[] setPlus(String[] a, String[] b) {
-		if (a != null && b != null) {
-			Set<String> sa = new HashSet<>(Arrays.asList(a));
-			Set<String> sb = new HashSet<>(Arrays.asList(b));
-			sa.addAll(sb);
-			return sa.toArray(new String[sa.size()]);
-		}
-		return a;
-	}
-
 }

vipra-util/src/main/java/de/vipra/util/service/Service.java

@@ -43,12 +43,6 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
 	List<Type> getMultiple(Integer skip, Integer limit, String sortBy, Pair<String, Object> criteria, String... fields)
 			throws E;
 
-	/**
-	 * @see {@link Service#getMultiple(QueryBuilder)}
-	 */
-	List<Type> getMultiple(Integer skip, Integer limit, String sortBy, Pair<String, Object> criteria,
-			boolean noDefaultIgnore, String... fields) throws E;
-
 	/**
 	 * Returns multiple entities from the database.
 	 * 
@@ -147,7 +141,6 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
 		private List<Pair<String, Object>> criteria;
 		private String[] fields;
 		private boolean include;
-		private boolean defaultIgnore = true;
 
 		private QueryBuilder() {}
 
@@ -242,19 +235,6 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
 			return this;
 		}
 
-		/**
-		 * Whether to use default ignoring fields or not. Some fields are
-		 * ignored by default, as per defined in the models.
-		 * 
-		 * @param defaultIgnore
-		 *            true to use default ignoring
-		 * @return QueryBuilder instance
-		 */
-		public QueryBuilder defaultIgnore(boolean defaultIgnore) {
-			this.defaultIgnore = defaultIgnore;
-			return this;
-		}
-
 		public Integer getSkip() {
 			return skip;
 		}
@@ -279,10 +259,6 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
 			return fields;
 		}
 
-		public boolean isDefaultIgnore() {
-			return defaultIgnore;
-		}
-
 	}
 
 }