Skip to content
Snippets Groups Projects
Commit 1664c2fd authored by Eike Cochu's avatar Eike Cochu
Browse files

problem with topic insertion

parent a9b2fe6e
No related branches found
No related tags found
No related merge requests found
......@@ -101,11 +101,8 @@ public class JGibbAnalyzer extends Analyzer {
// discovered (line does not start with a tab). Cut off
// after k words are gathered.
String nextLine;
int words = 0;
while ((nextLine = nextLine()) != null) {
if (nextLine.startsWith("\t")) {
if (words > Constants.K_TOPIC_WORDS)
continue;
String[] parts = nextLine.trim().split("\\s+");
try {
Word word = wordMap.get(parts[0]);
......@@ -113,7 +110,6 @@ public class JGibbAnalyzer extends Analyzer {
Constants.LIKELINESS_PRECISION);
TopicWord topicWord = new TopicWord(word, likeliness);
topicWords.add(topicWord);
words++;
} catch (NumberFormatException e) {
log.error("could not parse number in line: " + nextLine);
} catch (ArrayIndexOutOfBoundsException e) {
......
......@@ -107,7 +107,7 @@ public class ModelingCommand implements Command {
while (indexIter.hasNext() && topicRefsListIter.hasNext()) {
// get article from database
String id = indexIter.next();
ArticleFull article = dbArticles.getSingle(MongoUtils.objectId(id));
ArticleFull article = dbArticles.getSingle(MongoUtils.objectId(id), "processedText");
if (article == null) {
log.error("no article found in db for id " + id);
continue;
......
<li><a>Advanced</a></li>
\ No newline at end of file
......@@ -38,7 +38,7 @@
</thead>
<tbody>
<tr ng-repeat="word in ::topic.words">
<td><a ui-sref="words.show({id:word.word})" ng-bind="word.word"></a></td>
<td><a ui-sref="words.show({id:word.id})" ng-bind="word.id"></a></td>
<td ng-bind="word.likeliness"></td>
</tr>
</tbody>
......
......@@ -55,7 +55,6 @@
<li ui-sref-active="active"><a ui-sref="topics.index">Topics</a></li>
<li ui-sref-active="active"><a ui-sref="words.index">Words</a></li>
</ul>
<ul class="nav navbar-nav navbar-right" ui-view="menu"></ul>
</div><!-- /.navbar-collapse -->
</div><!-- /.container-fluid -->
</nav>
......
......@@ -76,6 +76,7 @@
edges = [],
articleColor = '#BBC9D2',
topicColor = '#DBB234',
wordColor = '#B6C7BD',
container = $element.find("#visgraph")[0];
$scope.articleColor = articleColor;
......@@ -134,6 +135,41 @@
}
};
};
var wordNode = function(word) {
return {
id: ++id,
title: word.word,
label: word.word,
type: 'word',
word: word.word,
color: {
background: wordColor,
highlight: { background: wordColor }
}
};
};
var constructor = function(result, key, nodeFunction) {
if(result.data && result.data[key]) {
var data = result.data[key],
newNodes = [],
newEdges = [];
for(var i = 0; i < data.length; i++) {
var current = data[i];
if(ids.hasOwnProperty(current.id)) {
if(!$scope.nodes.get(ids[current.id]).loaded)
newEdges.push({from:ids[current.id], to:node.id});
} else {
newNodes.push(nodeFunction(current));
newEdges.push({from:id, to:node.id});
ids[current.id] = id;
}
}
if(newNodes.length) $scope.nodes.add(newNodes);
if(newEdges.length) $scope.edges.add(newEdges);
}
};
// on node select
var selectTimeout;
......@@ -145,46 +181,17 @@
if(node.type === 'article') {
// node is article, load article to get topics
ArticleFactory.get({id:node.article}, function(res) {
if(res.data && res.data.topics) {
var topics = res.data.topics,
newNodes = [],
newEdges = [];
for(var i = 0; i < topics.length; i++) {
var topic = topics[i].topic;
if(ids.hasOwnProperty(topic.id)) {
if(!$scope.nodes.get(ids[topic.id]).loaded)
newEdges.push({from:node.id, to:ids[topic.id]});
} else {
newNodes.push(topicNode(topic));
newEdges.push({from:node.id, to:id});
ids[topic.id] = id;
}
}
if(newNodes.length) $scope.nodes.add(newNodes);
if(newEdges.length) $scope.edges.add(newEdges);
}
constructor(res, 'topics', topicNode);
});
} else {
// node is topic, load topic to get articles
} else if(node.type === 'topic') {
// node is topic, load topic to get words
TopicFactory.get({id:node.topic}, function(res) {
if(res.data && res.data.articles) {
var articles = res.data.articles,
newNodes = [],
newEdges = [];
for(var i = 0; i < articles.length; i++) {
var article = articles[i];
if(ids.hasOwnProperty(article.id)) {
if(!$scope.nodes.get(ids[article.id]).loaded)
newEdges.push({from:ids[article.id], to:node.id});
} else {
newNodes.push(articleNode(article));
newEdges.push({from:id, to:node.id});
ids[article.id] = id;
}
}
if(newNodes.length) $scope.nodes.add(newNodes);
if(newEdges.length) $scope.edges.add(newEdges);
}
constructor(res, 'words', wordNode);
});
} else if(node.type === 'word') {
// node is word, load word to get topics
WordFactory.get({id:node.word}, function(res) {
constructor(res, 'topics', topicNode);
});
}
node.loaded = true;
......@@ -199,8 +206,10 @@
var node = $scope.nodes.get(props.nodes[0]);
if(node.type === 'article')
$state.transitionTo('articles.show', {id:node.article});
else
else if(node.type === 'topic')
$state.transitionTo('topics.show', {id:node.topic});
else if(node.type === 'word')
$state.transitionTo('words.show', {id:node.word});
};
// watch for changes to model
......@@ -210,8 +219,10 @@
// root node
if($scope.visType === 'articles')
nodes.push(articleNode($scope.ngModel));
else
else if($scope.visType === 'topics')
nodes.push(topicNode($scope.ngModel));
else if($scope.visType === 'words')
nodes.push(wordNode($scope.ngModel));
ids[$scope.ngModel.id] = id;
// add nodes and edges
......
......@@ -47,7 +47,7 @@ public class ArticleFull extends FileModel<ObjectId> implements Serializable {
private String text;
@ElasticIndex("text")
@QueryIgnore(multi = true)
@QueryIgnore(all = true)
private String processedText;
private String url;
......
......@@ -17,7 +17,7 @@ public class TopicWord implements Comparable<TopicWord>, Serializable {
@JsonIgnore
private Word word;
@JsonProperty("word")
@JsonProperty("id")
private String wordString;
private Double likeliness;
......
......@@ -2,10 +2,7 @@ package de.vipra.util.service;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.mongodb.morphia.Datastore;
import org.mongodb.morphia.query.Query;
......@@ -54,7 +51,7 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
if (cache == null || (fields != null && fields.length > 0) || !cache.contains(id)) {
Query<Type> q = datastore.createQuery(clazz).field("_id").equal(id);
if (fields != null && fields.length > 0)
q.retrievedFields(true, setMinus(fields, ignoredFieldsSingle));
q.retrievedFields(true, fields);
else if (ignoredFieldsSingle.length > 0)
q.retrievedFields(false, ignoredFieldsSingle);
t = q.get();
......@@ -75,14 +72,7 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
public List<Type> getMultiple(Integer skip, Integer limit, String sortBy, Pair<String, Object> criteria,
String... fields) {
return getMultiple(
QueryBuilder.builder().skip(skip).limit(limit).sortBy(sortBy).criteria(criteria).fields(true, fields));
}
@Override
public List<Type> getMultiple(Integer skip, Integer limit, String sortBy, Pair<String, Object> criteria,
boolean defaultIgnore, String... fields) {
return getMultiple(QueryBuilder.builder().skip(skip).limit(limit).sortBy(sortBy).fields(true, fields)
.criteria(criteria).defaultIgnore(defaultIgnore));
QueryBuilder.builder().skip(skip).limit(limit).sortBy(sortBy).fields(true, fields).criteria(criteria));
}
@Override
......@@ -100,12 +90,8 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
if (builder.getFields() != null) {
String[] fields = builder.getFields();
if (builder.isInclude()) {
if (builder.isDefaultIgnore() && ignoredFieldsMulti.length > 0)
fields = setMinus(fields, ignoredFieldsMulti);
q.retrievedFields(true, fields);
} else {
if (builder.isDefaultIgnore() && ignoredFieldsMulti.length > 0)
fields = setPlus(fields, ignoredFieldsMulti);
q.retrievedFields(false, fields);
}
} else if (ignoredFieldsMulti.length > 0) {
......@@ -173,24 +159,4 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
return new MongoService<Type, IdType>(mongo, clazz);
}
private String[] setMinus(String[] a, String[] b) {
if (a != null && b != null) {
Set<String> sa = new HashSet<>(Arrays.asList(a));
Set<String> sb = new HashSet<>(Arrays.asList(b));
sa.removeAll(sb);
return sa.toArray(new String[sa.size()]);
}
return a;
}
private String[] setPlus(String[] a, String[] b) {
if (a != null && b != null) {
Set<String> sa = new HashSet<>(Arrays.asList(a));
Set<String> sb = new HashSet<>(Arrays.asList(b));
sa.addAll(sb);
return sa.toArray(new String[sa.size()]);
}
return a;
}
}
......@@ -43,12 +43,6 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
List<Type> getMultiple(Integer skip, Integer limit, String sortBy, Pair<String, Object> criteria, String... fields)
throws E;
/**
* @see {@link Service#getMultiple(QueryBuilder)}
*/
List<Type> getMultiple(Integer skip, Integer limit, String sortBy, Pair<String, Object> criteria,
boolean noDefaultIgnore, String... fields) throws E;
/**
* Returns multiple entities from the database.
*
......@@ -147,7 +141,6 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
private List<Pair<String, Object>> criteria;
private String[] fields;
private boolean include;
private boolean defaultIgnore = true;
private QueryBuilder() {}
......@@ -242,19 +235,6 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
return this;
}
/**
* Whether to use default ignoring fields or not. Some fields are
* ignored by default, as per defined in the models.
*
* @param defaultIgnore
* true to use default ignoring
* @return QueryBuilder instance
*/
public QueryBuilder defaultIgnore(boolean defaultIgnore) {
this.defaultIgnore = defaultIgnore;
return this;
}
public Integer getSkip() {
return skip;
}
......@@ -279,10 +259,6 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
return fields;
}
public boolean isDefaultIgnore() {
return defaultIgnore;
}
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment