From 6242da02d98e826986fd93a6184e954ca5b3169d Mon Sep 17 00:00:00 2001
From: Eike Cochu <eike@cochu.com>
Date: Tue, 19 Jan 2016 23:52:44 +0100
Subject: [PATCH] updated
added dependency to redirect corenlp output to slf4j
added corenlp preprocessor with lemmatizer, set to default
split topic in topic and topicfull for faster single article query
added wordmap for word creation on import
added word entity to store words in the database
topic names are now automatically generated from the top 4 topic words
removed uniquewords/-count from article stats, not much use
added timer labeled lap functionality
added catch exception error logging in cmd main
removed termfrequency, was used for unique words in article stats
removed ping resource from rest service
added queryignore annotation to ignore fields on query by service
---
ma-impl.sublime-workspace | 56 ++------
vipra-cmd/pom.xml | 5 +
.../src/main/java/de/vipra/cmd/Main.java | 2 +
.../de/vipra/cmd/lda/JGibbLDAAnalyzer.java | 38 +++--
.../java/de/vipra/cmd/lda/LDAAnalyzer.java | 28 +++-
.../de/vipra/cmd/model/ProcessedArticle.java | 1 +
.../de/vipra/cmd/option/ClearCommand.java | 10 +-
.../de/vipra/cmd/option/ImportCommand.java | 45 +++---
.../de/vipra/cmd/option/StatsCommand.java | 6 +-
.../de/vipra/cmd/text/CoreNLPProcessor.java | 4 +-
.../java/de/vipra/cmd/text/Processor.java | 1 -
vipra-rest/pom.xml | 17 +--
.../rest/provider/ObjectMapperProvider.java | 13 +-
.../vipra/rest/resource/ArticleResource.java | 16 +--
.../de/vipra/rest/resource/PingResource.java | 18 ---
.../de/vipra/rest/resource/TopicResource.java | 45 +++---
.../rest/serializer/GenericDeserializer.java | 5 +-
.../rest/serializer/GenericSerializer.java | 18 +--
.../rest/serializer/ObjectIdDeserializer.java | 21 +++
.../rest/serializer/ObjectIdSerializer.java | 20 +++
.../de/vipra/rest/service/ArticleService.java | 27 ----
.../de/vipra/rest/service/TopicService.java | 24 ----
vipra-ui/app/templates/articles/show.hbs | 38 +----
vipra-ui/app/templates/topics/index.hbs | 2 +-
vipra-ui/app/templates/topics/show/index.hbs | 10 +-
.../main/java/de/vipra/util/Constants.java | 4 +-
.../main/java/de/vipra/util/ListUtils.java | 8 ++
.../main/java/de/vipra/util/StringUtils.java | 10 ++
.../src/main/java/de/vipra/util/Timer.java | 26 ++++
.../java/de/vipra/util/an/JsonIgnore.java | 6 +-
.../java/de/vipra/util/an/QueryIgnore.java | 16 +++
.../java/de/vipra/util/model/Article.java | 56 ++++----
.../de/vipra/util/model/ArticleStats.java | 55 +------
.../main/java/de/vipra/util/model/Model.java | 3 +-
.../de/vipra/util/model/TermFrequency.java | 45 ------
.../main/java/de/vipra/util/model/Topic.java | 88 +-----------
.../java/de/vipra/util/model/TopicFull.java | 135 ++++++++++++++++++
.../java/de/vipra/util/model/TopicRef.java | 18 ++-
.../java/de/vipra/util/model/TopicWord.java | 13 +-
.../main/java/de/vipra/util/model/Word.java | 59 ++++++++
.../java/de/vipra/util/model/WordMap.java | 99 +++++++++++++
.../vipra/util/service/DatabaseService.java | 54 ++++++-
.../java/de/vipra/util/service/Service.java | 12 +-
43 files changed, 680 insertions(+), 497 deletions(-)
delete mode 100644 vipra-rest/src/main/java/de/vipra/rest/resource/PingResource.java
create mode 100644 vipra-rest/src/main/java/de/vipra/rest/serializer/ObjectIdDeserializer.java
create mode 100644 vipra-rest/src/main/java/de/vipra/rest/serializer/ObjectIdSerializer.java
delete mode 100644 vipra-rest/src/main/java/de/vipra/rest/service/ArticleService.java
delete mode 100644 vipra-rest/src/main/java/de/vipra/rest/service/TopicService.java
create mode 100644 vipra-util/src/main/java/de/vipra/util/an/QueryIgnore.java
delete mode 100644 vipra-util/src/main/java/de/vipra/util/model/TermFrequency.java
create mode 100644 vipra-util/src/main/java/de/vipra/util/model/TopicFull.java
create mode 100644 vipra-util/src/main/java/de/vipra/util/model/Word.java
create mode 100644 vipra-util/src/main/java/de/vipra/util/model/WordMap.java
diff --git a/ma-impl.sublime-workspace b/ma-impl.sublime-workspace
index 0a9c9a9b..1178f378 100644
--- a/ma-impl.sublime-workspace
+++ b/ma-impl.sublime-workspace
@@ -275,15 +275,6 @@
},
"buffers":
[
- {
- "contents": "stemming to lemmatization (corenlp benutzen)\ntop n words for topic name\nbuch: natural language processing with java\ndynamic lda\n\nVISUALISIERUNG!",
- "settings":
- {
- "buffer_size": 144,
- "line_ending": "Unix",
- "name": "stemming to lemmatization (corenlp benutzen)"
- }
- }
],
"build_system": "",
"build_system_choices":
@@ -463,6 +454,7 @@
"expanded_folders":
[
"/home/eike/repos/master/ma-impl",
+ "/home/eike/repos/master/ma-impl/vipra-cmd",
"/home/eike/repos/master/ma-impl/vipra-ui",
"/home/eike/repos/master/ma-impl/vipra-ui/app",
"/home/eike/repos/master/ma-impl/vipra-ui/app/adapters",
@@ -471,15 +463,21 @@
"/home/eike/repos/master/ma-impl/vipra-ui/app/routes/topics",
"/home/eike/repos/master/ma-impl/vipra-ui/app/templates",
"/home/eike/repos/master/ma-impl/vipra-ui/app/templates/articles",
- "/home/eike/repos/master/ma-impl/vipra-ui/app/templates/components"
+ "/home/eike/repos/master/ma-impl/vipra-ui/app/templates/components",
+ "/home/eike/repos/master/ma-impl/vipra-ui/app/templates/topics",
+ "/home/eike/repos/master/ma-impl/vipra-ui/app/templates/topics/show"
],
"file_history":
[
+ "/home/eike/repos/master/ma-impl/vipra-ui/app/components/topic-link.js",
+ "/home/eike/repos/master/ma-impl/vipra-ui/app/templates/articles/show.hbs",
+ "/home/eike/.local/share/vipra/jgibb/jgibb.twords",
+ "/home/eike/.local/share/vipra/jgibb/jgibb.tassign",
+ "/home/eike/repos/master/ma-impl/vipra-ui/app/templates/topics/index.hbs",
+ "/home/eike/repos/master/ma-impl/vipra-ui/app/templates/topics/show/index.hbs",
"/home/eike/Downloads/FRITZ.Box 7490 113.06.30_17.01.16_2147.export",
"/home/eike/repos/master/ma-impl/vm/data/test-1.json",
"/home/eike/repos/master/ma-impl/vm/data/test-2.json",
- "/home/eike/.local/share/vipra/jgibb/jgibb.twords",
- "/home/eike/.local/share/vipra/jgibb/jgibb.tassign",
"/home/eike/repos/master/ma-impl/vipra-ui/app/routes/topics/index.js",
"/home/eike/repos/master/ma-impl/vipra-ui/app/templates/articles/index.hbs",
"/home/eike/repos/master/ma-impl/vipra-ui/app/adapters/application.js",
@@ -494,18 +492,14 @@
"/home/eike/repos/master/ma-impl/vipra-ui/app/models/topic.js",
"/home/eike/repos/master/ma-impl/vipra-ui/app/routes/topics/show/edit.js",
"/home/eike/repos/master/ma-impl/vipra-ui/app/templates/components/topics-list.hbs",
- "/home/eike/repos/master/ma-impl/vipra-ui/app/templates/topics/index.hbs",
"/home/eike/repos/master/ma-impl/vipra-ui/app/router.js",
- "/home/eike/repos/master/ma-impl/vipra-ui/app/templates/topics/show/index.hbs",
"/home/eike/repos/master/ma-impl/vipra-ui/app/routes/topics/show.js",
"/home/eike/repos/master/ma-impl/vipra-ui/app/templates/topics/show/edit.hbs",
"/home/eike/repos/master/ma-impl/vipra-ui/app/templates/topics/show.hbs",
"/home/eike/repos/master/ma-impl/vipra-ui/app/helpers/topic-share.js",
"/home/eike/repos/master/ma-impl/vipra-ui/app/styles/app.css",
- "/home/eike/repos/master/ma-impl/vipra-ui/app/components/topic-link.js",
"/home/eike/repos/master/ma-impl/vipra-ui/app/templates/components/topic-link.hbs",
"/home/eike/repos/master/ma-impl/vipra-ui/app/templates/topics/edit.hbs",
- "/home/eike/repos/master/ma-impl/vipra-ui/app/templates/articles/show.hbs",
"/home/eike/repos/master/ma-impl/vipra-ui/app/routes/articles/show.js",
"/home/eike/repos/master/ma-impl/vipra-ui/app/templates/articles/edit.hbs",
"/home/eike/repos/master/ma-impl/vipra-ui/app/routes/topics/edit.js",
@@ -926,38 +920,8 @@
"groups":
[
{
- "selected": 0,
"sheets":
[
- {
- "buffer": 0,
- "semi_transient": false,
- "settings":
- {
- "buffer_size": 144,
- "regions":
- {
- },
- "selection":
- [
- [
- 99,
- 99
- ]
- ],
- "settings":
- {
- "auto_name": "stemming to lemmatization (corenlp benutzen)",
- "default_dir": "/home/eike/repos/master/ma-impl",
- "syntax": "Packages/Text/Plain text.tmLanguage"
- },
- "translation.x": 0.0,
- "translation.y": 0.0,
- "zoom_level": 1.0
- },
- "stack_index": 0,
- "type": "text"
- }
]
}
],
diff --git a/vipra-cmd/pom.xml b/vipra-cmd/pom.xml
index 5aed56ed..eb06c628 100644
--- a/vipra-cmd/pom.xml
+++ b/vipra-cmd/pom.xml
@@ -86,6 +86,11 @@
<artifactId>log4j-slf4j-impl</artifactId>
<version>${log4jVersion}</version>
</dependency>
+ <dependency>
+ <groupId>uk.org.lidalia</groupId>
+ <artifactId>sysout-over-slf4j</artifactId>
+ <version>1.0.2</version>
+ </dependency>
<!-- MongoDB Database Adapter -->
<dependency>
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/Main.java b/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
index 4c488805..4a08e5fc 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
@@ -30,6 +30,7 @@ import de.vipra.util.ConsoleUtils;
import de.vipra.util.ConsoleUtils.Choice;
import de.vipra.util.StringUtils;
import de.vipra.util.Timer;
+import uk.org.lidalia.sysoutslf4j.context.SysOutOverSLF4J;
public class Main {
@@ -38,6 +39,7 @@ public class Main {
static {
MorphiaLoggerFactory.registerLogger(SLF4JLoggerImplFactory.class);
+ SysOutOverSLF4J.sendSystemOutAndErrToSLF4J();
}
public static void main(String[] args) {
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbLDAAnalyzer.java b/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbLDAAnalyzer.java
index 684f0f65..df84fbe2 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbLDAAnalyzer.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbLDAAnalyzer.java
@@ -17,9 +17,11 @@ import de.vipra.util.Config;
import de.vipra.util.ConvertStream;
import de.vipra.util.StringUtils;
import de.vipra.util.ex.ConfigException;
-import de.vipra.util.model.Topic;
+import de.vipra.util.model.TopicFull;
import de.vipra.util.model.TopicRef;
import de.vipra.util.model.TopicWord;
+import de.vipra.util.model.Word;
+import de.vipra.util.model.WordMap;
import jgibblda.Estimator;
import jgibblda.Inferencer;
import jgibblda.LDACmdOption;
@@ -33,13 +35,14 @@ public class JGibbLDAAnalyzer extends LDAAnalyzer {
private File modelDir;
private File modelFile;
private LDACmdOption options;
+ private WordMap wordMap;
protected JGibbLDAAnalyzer() {
super("JGibb Analyzer");
}
@Override
- public void init(Config config) throws LDAAnalyzerException {
+ public void init(Config config, WordMap wordMap) throws LDAAnalyzerException {
options = new LDACmdOption();
try {
@@ -57,6 +60,8 @@ public class JGibbLDAAnalyzer extends LDAAnalyzer {
options.dfile = modelFile.getName();
options.modelName = "jgibb";
+
+ this.wordMap = wordMap;
}
private void estimate() {
@@ -81,28 +86,40 @@ public class JGibbLDAAnalyzer extends LDAAnalyzer {
}
@Override
- public ConvertStream<Topic> getTopicDefinitions() throws LDAAnalyzerException {
+ public ConvertStream<TopicFull> getTopicDefinitions() throws LDAAnalyzerException {
File twords = new File(modelDir, "jgibb.twords");
try {
- return new ConvertStream<Topic>(twords) {
+ return new ConvertStream<TopicFull>(twords) {
@Override
- public Topic convert(String line) {
- Topic topicDef = new Topic();
+ public TopicFull convert(String line) {
+ TopicFull topicDef = new TopicFull();
List<TopicWord> topicWords = new ArrayList<>();
+
+ // get index of topic
+ // the index will be used as a temporary id until the topic
+ // is created in the database
Integer index = StringUtils.getFirstNumber(line);
if (index == null) {
log.error("could not extract topic index from line: " + line);
} else {
topicDef.setIndex(index);
}
+
+ // get all lines that follow until the next topic is
+ // discovered (line does not start with a tab)
String nextLine;
while ((nextLine = nextLine()) != null) {
if (nextLine.startsWith("\t")) {
String[] parts = nextLine.trim().split("\\s+");
try {
- topicWords.add(new TopicWord(parts[0], Double.parseDouble(parts[1])));
+ Word word = wordMap.get(parts[0]);
+ double likeliness = Double.parseDouble(parts[1]);
+ TopicWord topicWord = new TopicWord(word, likeliness);
+ topicWords.add(topicWord);
} catch (NumberFormatException e) {
log.error("could not parse number in line: " + nextLine);
+ } catch (ArrayIndexOutOfBoundsException e) {
+ log.error("ill formatted topic definition in line: " + nextLine);
}
} else {
buffer(nextLine);
@@ -110,8 +127,8 @@ public class JGibbLDAAnalyzer extends LDAAnalyzer {
}
}
Collections.sort(topicWords);
- topicDef.setWords(topicWords);
- topicDef.setName(topicDef.getNameFromWords());
+ topicDef.setTopicWords(topicWords);
+ topicDef.setName(TopicFull.getNameFromWords(topicWords));
return topicDef;
}
};
@@ -131,6 +148,9 @@ public class JGibbLDAAnalyzer extends LDAAnalyzer {
Map<String, Integer> countMap = new HashMap<>();
String[] wordList = line.split("\\s+");
for (String word : wordList) {
+ // the file uses a simple integer as topic ids. Use this
+ // id as a temporary id until the topic is created in
+ // the database
String topic = word.split(":")[1];
Integer count = countMap.get(topic);
countMap.put(topic, count == null ? 1 : count + 1);
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/lda/LDAAnalyzer.java b/vipra-cmd/src/main/java/de/vipra/cmd/lda/LDAAnalyzer.java
index dc3e1131..6a34f1ce 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/lda/LDAAnalyzer.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/lda/LDAAnalyzer.java
@@ -7,8 +7,9 @@ import de.vipra.util.Config;
import de.vipra.util.Config.Key;
import de.vipra.util.Constants;
import de.vipra.util.ConvertStream;
-import de.vipra.util.model.Topic;
+import de.vipra.util.model.TopicFull;
import de.vipra.util.model.TopicRef;
+import de.vipra.util.model.WordMap;
public abstract class LDAAnalyzer {
@@ -22,15 +23,32 @@ public abstract class LDAAnalyzer {
return name;
}
- public abstract void init(Config config) throws LDAAnalyzerException;
+ public abstract void init(Config config, WordMap wordMap) throws LDAAnalyzerException;
public abstract void analyze() throws LDAAnalyzerException;
- public abstract ConvertStream<Topic> getTopicDefinitions() throws LDAAnalyzerException;
+ /**
+ * Returns a converting stream of topics, read from the topic definition
+ * file. Usually, a topic definition consists of a list of words, that are
+ * assigned to that topic with a certain likeliness.
+ *
+ * @return topic definition stream
+ * @throws LDAAnalyzerException
+ */
+ public abstract ConvertStream<TopicFull> getTopicDefinitions() throws LDAAnalyzerException;
+ /**
+ * Returns a converting stream of lists of topic references. Normally, topic
+ * modeling outputs topics for each word of each document. These references
+ * are returned by this function.
+ *
+ * @return stream of lists of topic references per document (ordered by
+ * index)
+ * @throws LDAAnalyzerException
+ */
public abstract ConvertStream<List<TopicRef>> getTopics() throws LDAAnalyzerException;
- public static LDAAnalyzer getAnalyzer(Config config) throws LDAAnalyzerException {
+ public static LDAAnalyzer getAnalyzer(Config config, WordMap wordMap) throws LDAAnalyzerException {
LDAAnalyzer analyzer = null;
switch (Constants.Analyzer.fromString(config.getString(Key.ANALYZER))) {
case JGIBB:
@@ -39,7 +57,7 @@ public abstract class LDAAnalyzer {
analyzer = new JGibbLDAAnalyzer();
break;
}
- analyzer.init(config);
+ analyzer.init(config, wordMap);
return analyzer;
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/model/ProcessedArticle.java b/vipra-cmd/src/main/java/de/vipra/cmd/model/ProcessedArticle.java
index 897bcc0a..aea42f74 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/model/ProcessedArticle.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/model/ProcessedArticle.java
@@ -6,6 +6,7 @@ import org.mongodb.morphia.annotations.Transient;
import de.vipra.cmd.text.ProcessedText;
+@SuppressWarnings("serial")
@Entity(value = "articles", noClassnameStored = true)
public class ProcessedArticle extends de.vipra.util.model.Article {
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java
index 178ec404..d271eabc 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java
@@ -13,7 +13,8 @@ import de.vipra.cmd.model.ProcessedArticle;
import de.vipra.util.Config;
import de.vipra.util.ConsoleUtils;
import de.vipra.util.ex.ConfigException;
-import de.vipra.util.model.Topic;
+import de.vipra.util.model.TopicFull;
+import de.vipra.util.model.Word;
import de.vipra.util.service.DatabaseService;
public class ClearCommand implements Command {
@@ -24,7 +25,8 @@ public class ClearCommand implements Command {
private boolean defaults;
private Config config;
private DatabaseService<ProcessedArticle> dbArticles;
- private DatabaseService<Topic> dbTopics;
+ private DatabaseService<TopicFull> dbTopics;
+ private DatabaseService<Word> dbWords;
public ClearCommand(boolean defaults) {
this.defaults = defaults;
@@ -34,7 +36,8 @@ public class ClearCommand implements Command {
try {
config = Config.getConfig();
dbArticles = DatabaseService.getDatabaseService(config, ProcessedArticle.class);
- dbTopics = DatabaseService.getDatabaseService(config, Topic.class);
+ dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class);
+ dbWords = DatabaseService.getDatabaseService(config, Word.class);
} catch (Exception e) {
throw new ClearException(e);
}
@@ -42,6 +45,7 @@ public class ClearCommand implements Command {
out.info("clearing database");
dbArticles.drop();
dbTopics.drop();
+ dbWords.drop();
out.info("clearing filebase");
File dataDir = config.getDataDirectory();
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
index 2f850325..1a54d02d 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
@@ -33,8 +33,10 @@ import de.vipra.util.StringUtils;
import de.vipra.util.Timer;
import de.vipra.util.ex.DatabaseException;
import de.vipra.util.model.ArticleStats;
-import de.vipra.util.model.Topic;
+import de.vipra.util.model.TopicFull;
import de.vipra.util.model.TopicRef;
+import de.vipra.util.model.Word;
+import de.vipra.util.model.WordMap;
import de.vipra.util.service.DatabaseService;
public class ImportCommand implements Command {
@@ -46,9 +48,11 @@ public class ImportCommand implements Command {
private JSONParser parser = new JSONParser();
private Config config;
private DatabaseService<ProcessedArticle> dbArticles;
- private DatabaseService<Topic> dbTopics;
+ private DatabaseService<TopicFull> dbTopics;
+ private DatabaseService<Word> dbWords;
private Filebase filebase;
private Processor preprocessor;
+ private WordMap wordMap;
private LDAAnalyzer analyzer;
/**
@@ -106,7 +110,7 @@ public class ImportCommand implements Command {
try {
// preprocess text and generate text statistics
ProcessedText processedText = preprocessor.preprocess(article.getText());
- ArticleStats articleStats = ArticleStats.generateFromText(processedText.getText());
+ ArticleStats articleStats = ArticleStats.generateFromText(processedText.getText(), wordMap);
// add article to mongodb
article.setProcessedText(processedText);
@@ -166,14 +170,15 @@ public class ImportCommand implements Command {
* @throws DatabaseException
*/
private Map<String, String> saveTopicDefinitions() throws LDAAnalyzerException, DatabaseException {
- ConvertStream<Topic> topics = analyzer.getTopicDefinitions();
+ ConvertStream<TopicFull> topics = analyzer.getTopicDefinitions();
Map<String, String> topicIndexMap = new HashMap<>();
// recreate topics in database
+ // create one topic at a time for less memory usage
dbTopics.drop();
- for (Topic topic : topics) {
- Topic newTopic = dbTopics.createSingle(topic);
- topicIndexMap.put(Integer.toString(newTopic.getIndex()), newTopic.getId().toString());
+ for (TopicFull topic : topics) {
+ dbTopics.createSingle(topic);
+ topicIndexMap.put(Integer.toString(topic.getIndex()), topic.getId().toString());
}
return topicIndexMap;
@@ -223,10 +228,12 @@ public class ImportCommand implements Command {
try {
config = Config.getConfig();
dbArticles = DatabaseService.getDatabaseService(config, ProcessedArticle.class);
- dbTopics = DatabaseService.getDatabaseService(config, Topic.class);
+ dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class);
+ dbWords = DatabaseService.getDatabaseService(config, Word.class);
filebase = Filebase.getFilebase(config);
preprocessor = Processor.getPreprocessor(config);
- analyzer = LDAAnalyzer.getAnalyzer(config);
+ wordMap = new WordMap(dbWords);
+ analyzer = LDAAnalyzer.getAnalyzer(config, wordMap);
out.info("using data directory: " + config.getDataDirectory().getAbsolutePath());
out.info("using preprocessor: " + preprocessor.getName());
@@ -238,26 +245,32 @@ public class ImportCommand implements Command {
// import files into database and filebase
out.info("file import");
long imported = importFiles(files);
- long durImport = timer.lap();
+ timer.lap("import");
// write filebase
out.info("writing file index");
filebase.close();
- timer.lap();
+ timer.lap("filebase write");
// do topic modeling
out.info("topic modeling");
analyzer.analyze();
- long durAnalyze = timer.lap();
+ timer.lap("topic modeling");
// save topic model
- out.info("saving topic models");
+ out.info("saving topic definitions");
Map<String, String> topicIndexMap = saveTopicDefinitions();
+ timer.lap("saving topics");
+
+ // save topic refs
+ out.info("saving document topics");
saveTopicsPerDocument(topicIndexMap);
+ timer.lap("saving topic refs");
- out.info("imported " + imported + " " + (imported == 1 ? "article" : "articles"));
- out.info("import: " + StringUtils.timeString(durImport) + ", analyze: "
- + StringUtils.timeString(durAnalyze));
+ out.info("imported " + imported + " new " + StringUtils.quantity(imported, "article"));
+ long newWords = wordMap.getNewWords();
+ out.info("imported " + newWords + " new " + StringUtils.quantity(newWords, "word"));
+ out.info(timer.toString());
} catch (Exception e) {
throw new ExecutionException(e);
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/StatsCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/StatsCommand.java
index 774fd432..4793f14f 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/StatsCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/StatsCommand.java
@@ -12,7 +12,7 @@ import de.vipra.cmd.file.Filebase;
import de.vipra.util.Config;
import de.vipra.util.StringUtils;
import de.vipra.util.ex.ConfigException;
-import de.vipra.util.model.Topic;
+import de.vipra.util.model.TopicFull;
import de.vipra.util.service.DatabaseService;
public class StatsCommand implements Command {
@@ -22,7 +22,7 @@ public class StatsCommand implements Command {
private Config config;
private Filebase filebase;
- private DatabaseService<Topic> dbTopics;
+ private DatabaseService<TopicFull> dbTopics;
private void stats() {
File modelFile = filebase.getModelFile();
@@ -37,7 +37,7 @@ public class StatsCommand implements Command {
try {
config = Config.getConfig();
filebase = Filebase.getFilebase(config);
- dbTopics = DatabaseService.getDatabaseService(config, Topic.class);
+ dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class);
stats();
} catch (IOException | ConfigException | FilebaseException e) {
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/text/CoreNLPProcessor.java b/vipra-cmd/src/main/java/de/vipra/cmd/text/CoreNLPProcessor.java
index d0534bca..afc32c13 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/text/CoreNLPProcessor.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/text/CoreNLPProcessor.java
@@ -4,6 +4,7 @@ import java.util.List;
import java.util.Properties;
import de.vipra.cmd.ex.PreprocessorException;
+import de.vipra.util.Constants;
import edu.stanford.nlp.ling.CoreAnnotations.SentencesAnnotation;
import edu.stanford.nlp.ling.CoreAnnotations.TokensAnnotation;
import edu.stanford.nlp.ling.CoreLabel;
@@ -41,7 +42,8 @@ public class CoreNLPProcessor extends Processor {
sb.append(word.word()).append(" ");
}
}
- return new ProcessedText(sb.toString().trim());
+ String text = sb.toString().trim().replaceAll(Constants.CHARS_DISALLOWED, "").replaceAll("\\s+", " ");
+ return new ProcessedText(text);
}
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/text/Processor.java b/vipra-cmd/src/main/java/de/vipra/cmd/text/Processor.java
index fcf6521d..7b3974e7 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/text/Processor.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/text/Processor.java
@@ -1,6 +1,5 @@
package de.vipra.cmd.text;
-import java.util.Arrays;
import java.util.List;
import de.vipra.cmd.ex.PreprocessorException;
diff --git a/vipra-rest/pom.xml b/vipra-rest/pom.xml
index 8b81b7ee..ff073cf9 100644
--- a/vipra-rest/pom.xml
+++ b/vipra-rest/pom.xml
@@ -34,22 +34,7 @@
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
- <version>${jerseyVersion}</version>
- </dependency>
- <dependency>
- <groupId>org.glassfish.jersey.test-framework</groupId>
- <artifactId>jersey-test-framework-core</artifactId>
- <version>${jerseyVersion}</version>
- </dependency>
- <dependency>
- <groupId>org.glassfish.jersey.test-framework.providers</groupId>
- <artifactId>jersey-test-framework-provider-simple</artifactId>
- <version>${jerseyVersion}</version>
- </dependency>
- <dependency>
- <groupId>com.github.fge</groupId>
- <artifactId>json-patch</artifactId>
- <version>1.9</version>
+ <version>2.22.1</version>
</dependency>
<!-- Servlet API -->
diff --git a/vipra-rest/src/main/java/de/vipra/rest/provider/ObjectMapperProvider.java b/vipra-rest/src/main/java/de/vipra/rest/provider/ObjectMapperProvider.java
index 9289222c..e7f5577a 100644
--- a/vipra-rest/src/main/java/de/vipra/rest/provider/ObjectMapperProvider.java
+++ b/vipra-rest/src/main/java/de/vipra/rest/provider/ObjectMapperProvider.java
@@ -5,6 +5,7 @@ import java.text.SimpleDateFormat;
import javax.ws.rs.ext.ContextResolver;
import javax.ws.rs.ext.Provider;
+import org.bson.types.ObjectId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -15,9 +16,11 @@ import com.fasterxml.jackson.databind.module.SimpleModule;
import de.vipra.rest.serializer.GenericDeserializer;
import de.vipra.rest.serializer.GenericSerializer;
+import de.vipra.rest.serializer.ObjectIdDeserializer;
+import de.vipra.rest.serializer.ObjectIdSerializer;
import de.vipra.util.Constants;
import de.vipra.util.model.Article;
-import de.vipra.util.model.Topic;
+import de.vipra.util.model.TopicFull;
@Provider
public class ObjectMapperProvider implements ContextResolver<ObjectMapper> {
@@ -39,8 +42,12 @@ public class ObjectMapperProvider implements ContextResolver<ObjectMapper> {
SimpleModule module = new SimpleModule();
module.addSerializer(Article.class, new GenericSerializer<Article>(Article.class));
module.addDeserializer(Article.class, new GenericDeserializer<Article>(Article.class));
- module.addSerializer(Topic.class, new GenericSerializer<Topic>(Topic.class));
- module.addDeserializer(Topic.class, new GenericDeserializer<Topic>(Topic.class));
+
+ module.addSerializer(TopicFull.class, new GenericSerializer<TopicFull>(TopicFull.class));
+ module.addDeserializer(TopicFull.class, new GenericDeserializer<TopicFull>(TopicFull.class));
+
+ module.addSerializer(ObjectId.class, new ObjectIdSerializer());
+ module.addDeserializer(ObjectId.class, new ObjectIdDeserializer());
final ObjectMapper mapper = new ObjectMapper();
mapper.enable(SerializationFeature.INDENT_OUTPUT);
diff --git a/vipra-rest/src/main/java/de/vipra/rest/resource/ArticleResource.java b/vipra-rest/src/main/java/de/vipra/rest/resource/ArticleResource.java
index e3c5d06a..bf182eaa 100644
--- a/vipra-rest/src/main/java/de/vipra/rest/resource/ArticleResource.java
+++ b/vipra-rest/src/main/java/de/vipra/rest/resource/ArticleResource.java
@@ -27,12 +27,11 @@ import de.vipra.rest.Messages;
import de.vipra.rest.PATCH;
import de.vipra.rest.model.APIError;
import de.vipra.rest.model.Wrapper;
-import de.vipra.rest.service.ArticleService;
import de.vipra.util.Config;
-import de.vipra.util.Mongo;
import de.vipra.util.ex.ConfigException;
import de.vipra.util.ex.DatabaseException;
import de.vipra.util.model.Article;
+import de.vipra.util.service.DatabaseService;
@Path("articles")
public class ArticleResource {
@@ -40,20 +39,19 @@ public class ArticleResource {
@Context
UriInfo uri;
- Cache<String, Article> articleCache;
-
- final ArticleService service;
+ final Cache<String, Article> articleCache;
+ final DatabaseService<Article> service;
public ArticleResource(@Context ServletContext servletContext) throws ConfigException, IOException {
Config config = Config.getConfig();
- Mongo mongo = Mongo.getInstance(config);
- service = new ArticleService(mongo);
+ service = DatabaseService.getDatabaseService(config, Article.class);
CacheManager manager = (CacheManager) servletContext.getAttribute("cachemanager");
- articleCache = manager.getCache("articlecache", String.class, Article.class);
+ Cache<String, Article> articleCache = manager.getCache("articlecache", String.class, Article.class);
if (articleCache == null)
articleCache = manager.createCache("articlecache",
CacheConfigurationBuilder.newCacheConfigurationBuilder().buildConfig(String.class, Article.class));
+ this.articleCache = articleCache;
}
@GET
@@ -61,7 +59,7 @@ public class ArticleResource {
public Response getArticles(@QueryParam("skip") @DefaultValue("0") int skip,
@QueryParam("limit") @DefaultValue("0") int limit,
@QueryParam("sort") @DefaultValue("date") String sortBy) {
- List<Article> articles = service.getMultiple(uri.getAbsolutePath(), skip, limit, sortBy);
+ List<Article> articles = service.getMultiple(skip, limit, sortBy);
Wrapper<List<Article>> res = new Wrapper<>(articles);
return Response.ok().entity(res).tag(res.tag()).build();
}
diff --git a/vipra-rest/src/main/java/de/vipra/rest/resource/PingResource.java b/vipra-rest/src/main/java/de/vipra/rest/resource/PingResource.java
deleted file mode 100644
index 028c1caf..00000000
--- a/vipra-rest/src/main/java/de/vipra/rest/resource/PingResource.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package de.vipra.rest.resource;
-
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-@Path("ping")
-public class PingResource {
-
- @GET
- @Produces(MediaType.TEXT_PLAIN)
- public Response ping() {
- return Response.ok().entity("running").build();
- }
-
-}
diff --git a/vipra-rest/src/main/java/de/vipra/rest/resource/TopicResource.java b/vipra-rest/src/main/java/de/vipra/rest/resource/TopicResource.java
index df56b8c3..44cc2964 100644
--- a/vipra-rest/src/main/java/de/vipra/rest/resource/TopicResource.java
+++ b/vipra-rest/src/main/java/de/vipra/rest/resource/TopicResource.java
@@ -25,13 +25,11 @@ import de.vipra.rest.Messages;
import de.vipra.rest.PATCH;
import de.vipra.rest.model.APIError;
import de.vipra.rest.model.Wrapper;
-import de.vipra.rest.service.TopicService;
import de.vipra.util.Config;
-import de.vipra.util.Mongo;
import de.vipra.util.ex.ConfigException;
import de.vipra.util.ex.DatabaseException;
-import de.vipra.util.model.Article;
-import de.vipra.util.model.Topic;
+import de.vipra.util.model.TopicFull;
+import de.vipra.util.service.DatabaseService;
@Path("topics")
public class TopicResource {
@@ -39,28 +37,27 @@ public class TopicResource {
@Context
UriInfo uri;
- Cache<String, Topic> topicCache;
-
- TopicService service;
+ final Cache<String, TopicFull> topicCache;
+ final DatabaseService<TopicFull> service;
public TopicResource(@Context ServletContext servletContext) throws ConfigException, IOException {
Config config = Config.getConfig();
- Mongo mongo = Mongo.getInstance(config);
- service = new TopicService(mongo);
+ service = DatabaseService.getDatabaseService(config, TopicFull.class);
CacheManager manager = (CacheManager) servletContext.getAttribute("cachemanager");
- topicCache = manager.getCache("topiccache", String.class, Topic.class);
+ Cache<String, TopicFull> topicCache = manager.getCache("topiccache", String.class, TopicFull.class);
if (topicCache == null)
- topicCache = manager.createCache("topiccache",
- CacheConfigurationBuilder.newCacheConfigurationBuilder().buildConfig(String.class, Topic.class));
+ topicCache = manager.createCache("topiccache", CacheConfigurationBuilder.newCacheConfigurationBuilder()
+ .buildConfig(String.class, TopicFull.class));
+ this.topicCache = topicCache;
}
@GET
@Produces(APIMediaType.APPLICATION_JSONAPI)
public Response getTopics(@QueryParam("skip") @DefaultValue("0") int skip,
@QueryParam("limit") @DefaultValue("0") int limit) {
- List<Topic> topics = service.getMultiple(uri.getAbsolutePath(), skip, limit, null);
- Wrapper<List<Topic>> res = new Wrapper<>(topics);
+ List<TopicFull> topics = service.getMultiple(skip, limit, null);
+ Wrapper<List<TopicFull>> res = new Wrapper<>(topics);
return Response.ok().entity(res).tag(res.tag()).build();
}
@@ -69,14 +66,14 @@ public class TopicResource {
@Consumes(APIMediaType.APPLICATION_JSONAPI)
@Path("{id}")
public Response getTopic(@PathParam("id") String id) {
- Wrapper<Topic> res = new Wrapper<>();
+ Wrapper<TopicFull> res = new Wrapper<>();
if (id == null || id.trim().length() == 0) {
res.addError(new APIError(Response.Status.BAD_REQUEST, "ID is empty",
String.format(Messages.BAD_REQUEST, "id cannot be empty")));
return Response.status(Response.Status.BAD_REQUEST).entity(res).build();
}
- Topic topic = getSingle(id);
+ TopicFull topic = getSingle(id);
if (topic != null) {
res.setData(topic);
@@ -92,9 +89,9 @@ public class TopicResource {
@Consumes(APIMediaType.APPLICATION_JSONAPI)
@Produces(APIMediaType.APPLICATION_JSONAPI)
@Path("{id}")
- public Response replaceTopic(@PathParam("id") String id, Wrapper<Topic> wrapper) {
- Topic topic = wrapper.getData();
- Wrapper<Topic> res = new Wrapper<>();
+ public Response replaceTopic(@PathParam("id") String id, Wrapper<TopicFull> wrapper) {
+ TopicFull topic = wrapper.getData();
+ Wrapper<TopicFull> res = new Wrapper<>();
try {
service.updateSingle(topic);
topicCache.put(id, topic);
@@ -111,15 +108,15 @@ public class TopicResource {
@Consumes(APIMediaType.APPLICATION_JSONAPI)
@Produces(APIMediaType.APPLICATION_JSONAPI)
@Path("{id}")
- public Response updateTopic(@PathParam("id") String id, Wrapper<Topic> wrapper) {
- Topic newTopic = wrapper.getData();
- Topic topic = getSingle(id);
+ public Response updateTopic(@PathParam("id") String id, Wrapper<TopicFull> wrapper) {
+ TopicFull newTopic = wrapper.getData();
+ TopicFull topic = getSingle(id);
// TODO implement
return null;
}
- private Topic getSingle(String id) {
- Topic topic = topicCache.get(id);
+ private TopicFull getSingle(String id) {
+ TopicFull topic = topicCache.get(id);
if (topic == null) {
topic = service.getSingle(id);
if (topic != null)
diff --git a/vipra-rest/src/main/java/de/vipra/rest/serializer/GenericDeserializer.java b/vipra-rest/src/main/java/de/vipra/rest/serializer/GenericDeserializer.java
index 710073d2..ff9ea7d9 100644
--- a/vipra-rest/src/main/java/de/vipra/rest/serializer/GenericDeserializer.java
+++ b/vipra-rest/src/main/java/de/vipra/rest/serializer/GenericDeserializer.java
@@ -8,6 +8,7 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
@@ -17,7 +18,6 @@ import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonDeserializer;
import de.vipra.util.an.JsonField;
-import de.vipra.util.an.JsonIgnore;
import de.vipra.util.an.JsonWrap;
import de.vipra.util.model.Model;
@@ -32,7 +32,8 @@ public class GenericDeserializer<T extends Model> extends JsonDeserializer<T> {
Field[] fields = clazz.getDeclaredFields();
for (Field field : fields) {
- if (Modifier.isPrivate(field.getModifiers())) {
+ int modifiers = field.getModifiers();
+ if (Modifier.isPrivate(modifiers) && !Modifier.isStatic(modifiers)) {
field.setAccessible(true);
JsonIgnore ji = field.getDeclaredAnnotation(JsonIgnore.class);
diff --git a/vipra-rest/src/main/java/de/vipra/rest/serializer/GenericSerializer.java b/vipra-rest/src/main/java/de/vipra/rest/serializer/GenericSerializer.java
index 85311f4a..b7c896c0 100644
--- a/vipra-rest/src/main/java/de/vipra/rest/serializer/GenericSerializer.java
+++ b/vipra-rest/src/main/java/de/vipra/rest/serializer/GenericSerializer.java
@@ -11,9 +11,6 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import org.bson.types.ObjectId;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -41,11 +38,14 @@ public class GenericSerializer<T extends Model> extends JsonSerializer<T> {
Field[] fields = clazz.getDeclaredFields();
for (Field field : fields) {
- if (Modifier.isPrivate(field.getModifiers())) {
+ int modifiers = field.getModifiers();
+ if (Modifier.isPrivate(modifiers) && !Modifier.isStatic(modifiers)) {
field.setAccessible(true);
- JsonIgnore ji = field.getDeclaredAnnotation(JsonIgnore.class);
- if (ji != null && ji.value())
+ com.fasterxml.jackson.annotation.JsonIgnore ji1 = field
+ .getDeclaredAnnotation(com.fasterxml.jackson.annotation.JsonIgnore.class);
+ de.vipra.util.an.JsonIgnore ji2 = field.getDeclaredAnnotation(de.vipra.util.an.JsonIgnore.class);
+ if ((ji1 != null && ji1.value()) || (ji2 != null && ji2.value()))
continue;
String name = field.getName();
@@ -91,12 +91,8 @@ public class GenericSerializer<T extends Model> extends JsonSerializer<T> {
e.printStackTrace();
}
- if (v != null) {
- if (v instanceof ObjectId)
- v = ((ObjectId) v).toString();
-
+ if (v != null)
pathAdd(map, entry.getKey(), v);
- }
}
serializers.defaultSerializeValue(map, gen);
diff --git a/vipra-rest/src/main/java/de/vipra/rest/serializer/ObjectIdDeserializer.java b/vipra-rest/src/main/java/de/vipra/rest/serializer/ObjectIdDeserializer.java
new file mode 100644
index 00000000..4a6f5994
--- /dev/null
+++ b/vipra-rest/src/main/java/de/vipra/rest/serializer/ObjectIdDeserializer.java
@@ -0,0 +1,21 @@
+package de.vipra.rest.serializer;
+
+import java.io.IOException;
+
+import org.bson.types.ObjectId;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+
+import de.vipra.util.MongoUtils;
+
+public class ObjectIdDeserializer extends JsonDeserializer<ObjectId> {
+
+ @Override
+ public ObjectId deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
+ return MongoUtils.objectId(p.getValueAsString());
+ }
+
+}
diff --git a/vipra-rest/src/main/java/de/vipra/rest/serializer/ObjectIdSerializer.java b/vipra-rest/src/main/java/de/vipra/rest/serializer/ObjectIdSerializer.java
new file mode 100644
index 00000000..b6653911
--- /dev/null
+++ b/vipra-rest/src/main/java/de/vipra/rest/serializer/ObjectIdSerializer.java
@@ -0,0 +1,20 @@
+package de.vipra.rest.serializer;
+
+import java.io.IOException;
+
+import org.bson.types.ObjectId;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+
+public class ObjectIdSerializer extends JsonSerializer<ObjectId> {
+
+ @Override
+ public void serialize(ObjectId value, JsonGenerator gen, SerializerProvider serializers)
+ throws IOException, JsonProcessingException {
+ gen.writeString(value.toString());
+ }
+
+}
diff --git a/vipra-rest/src/main/java/de/vipra/rest/service/ArticleService.java b/vipra-rest/src/main/java/de/vipra/rest/service/ArticleService.java
deleted file mode 100644
index b53aaad4..00000000
--- a/vipra-rest/src/main/java/de/vipra/rest/service/ArticleService.java
+++ /dev/null
@@ -1,27 +0,0 @@
-package de.vipra.rest.service;
-
-import java.net.URI;
-import java.util.List;
-
-import de.vipra.util.Mongo;
-import de.vipra.util.model.Article;
-import de.vipra.util.service.DatabaseService;
-
-public class ArticleService extends DatabaseService<Article> {
-
- public ArticleService(Mongo mongo) {
- super(mongo, de.vipra.util.model.Article.class);
- }
-
- public List<Article> getMultiple(URI base, int skip, int limit, String sortBy) {
- List<Article> articles = super.getMultiple(skip, limit, sortBy);
- for (Article article : articles) {
- // delete data for listing
- article.setText(null);
- article.setStats(null);
- article.setTopics(null);
- }
- return articles;
- }
-
-}
diff --git a/vipra-rest/src/main/java/de/vipra/rest/service/TopicService.java b/vipra-rest/src/main/java/de/vipra/rest/service/TopicService.java
deleted file mode 100644
index 4ec6eb60..00000000
--- a/vipra-rest/src/main/java/de/vipra/rest/service/TopicService.java
+++ /dev/null
@@ -1,24 +0,0 @@
-package de.vipra.rest.service;
-
-import java.net.URI;
-import java.util.List;
-
-import de.vipra.util.Mongo;
-import de.vipra.util.model.Topic;
-import de.vipra.util.service.DatabaseService;
-
-public class TopicService extends DatabaseService<Topic> {
-
- public TopicService(Mongo mongo) {
- super(mongo, de.vipra.util.model.Topic.class);
- }
-
- public List<Topic> getMultiple(URI base, int skip, int limit, String sortBy) {
- List<Topic> topics = super.getMultiple(skip, limit, sortBy);
- for (Topic topic : topics) {
- topic.setWords(null);
- }
- return topics;
- }
-
-}
diff --git a/vipra-ui/app/templates/articles/show.hbs b/vipra-ui/app/templates/articles/show.hbs
index 9ee5c2bc..bcddff84 100644
--- a/vipra-ui/app/templates/articles/show.hbs
+++ b/vipra-ui/app/templates/articles/show.hbs
@@ -7,49 +7,21 @@
<dd>{{model.article.date}}</dd>
<dt>URL</dt>
<dd><a href="{{model.article.url}}">{{model.article.url}}</a></dd>
+ <dt>Word count</dt>
+ <dd>{{model.article.stats.wordCount}}</dd>
</dl>
<h3>Topics</h3>
-{{#each model.article.topics as |topic|}}
- {{#topic-link topic=topic}} ({{topic-share topic.count model.article.stats.wordCount}}%){{/topic-link}}
+{{#each model.article.topics as |topicRef|}}
+ [{{#topic-link topic=topicRef.topic}} ({{topic-share topicRef.count model.article.stats.wordCount}}%){{/topic-link}}]
{{/each}}
<h3>Statistics</h3>
<table>
<tbody>
- <tr>
- <td>Word count</td>
- <td>{{model.article.stats.wordCount}}</td>
- </tr>
- <tr>
- <td>Unique word count</td>
- <td>{{model.article.stats.uniqueWordCount}}</td>
- </tr>
- <tr>
- <td>Most frequent words</td>
- <td>
- <table>
- <thead>
- <tr>
- <th>Word</th>
- <th>Count</th>
- <th>NTF</th>
- </tr>
- </thead>
- <tbody>
- {{#each model.statsData as |stats|}}
- <tr>
- <td>{{stats.word}}</td>
- <td>{{stats.count}}</td>
- <td>{{stats.norm}}</td>
- </tr>
- {{/each}}
- </tbody>
- </table>
- </td>
- </tr>
+
</tbody>
</table>
diff --git a/vipra-ui/app/templates/topics/index.hbs b/vipra-ui/app/templates/topics/index.hbs
index f7cabdd7..a8a4eece 100644
--- a/vipra-ui/app/templates/topics/index.hbs
+++ b/vipra-ui/app/templates/topics/index.hbs
@@ -1,5 +1,5 @@
<h2>Found topics</h2>
-{{debounced-input placeholder='Filter' size='50' valueBinding='filter' debounce='150'}}
+{{debounced-input placeholder='Filter' size='50' value=filter debounce='150'}}
{{topics-list items=model.topics filter=filter}}
\ No newline at end of file
diff --git a/vipra-ui/app/templates/topics/show/index.hbs b/vipra-ui/app/templates/topics/show/index.hbs
index 0a8dcf51..e7f21034 100644
--- a/vipra-ui/app/templates/topics/show/index.hbs
+++ b/vipra-ui/app/templates/topics/show/index.hbs
@@ -1,11 +1,3 @@
{{#link-to 'topics.show.edit'}}Edit{{/link-to}}
-<h2>{{model.topic._name}}</h2>
-
-<h3>Words</h3>
-
-{{#each model.topic.words as |word|}}
- <span class="word" title="Likeliness: {{word.likeliness}}">{{word.word}}</span>
-{{/each}}
-
-<h3>Articles</h3>
\ No newline at end of file
+<h2>{{model.topic._name}}</h2>
\ No newline at end of file
diff --git a/vipra-util/src/main/java/de/vipra/util/Constants.java b/vipra-util/src/main/java/de/vipra/util/Constants.java
index c5eea30e..22072d5d 100644
--- a/vipra-util/src/main/java/de/vipra/util/Constants.java
+++ b/vipra-util/src/main/java/de/vipra/util/Constants.java
@@ -41,7 +41,7 @@ public class Constants {
* expression is used to strip text of characters that should not be
* processed.
*/
- public static final String CHARS_DISALLOWED = "[^a-zA-Z0-9]";
+ public static final String CHARS_DISALLOWED = "[^a-zA-Z0-9 ]";
/**
* The number of words to be used to generate a topic name. The top n words
@@ -127,7 +127,7 @@ public class Constants {
CUSTOM("custom"),
CORENLP("corenlp"),
LUCENE("lucene"),
- DEFAULT(LUCENE);
+ DEFAULT(CORENLP);
public final String name;
diff --git a/vipra-util/src/main/java/de/vipra/util/ListUtils.java b/vipra-util/src/main/java/de/vipra/util/ListUtils.java
index bddd22ab..8daf6a8a 100644
--- a/vipra-util/src/main/java/de/vipra/util/ListUtils.java
+++ b/vipra-util/src/main/java/de/vipra/util/ListUtils.java
@@ -1,5 +1,6 @@
package de.vipra.util;
+import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
@@ -21,4 +22,11 @@ public class ListUtils {
return strings;
}
+ public static <T> List<T> toList(Iterable<T> it) {
+ List<T> list = new ArrayList<>();
+ for (T t : it)
+ list.add(t);
+ return list;
+ }
+
}
diff --git a/vipra-util/src/main/java/de/vipra/util/StringUtils.java b/vipra-util/src/main/java/de/vipra/util/StringUtils.java
index 566ade74..adeda301 100644
--- a/vipra-util/src/main/java/de/vipra/util/StringUtils.java
+++ b/vipra-util/src/main/java/de/vipra/util/StringUtils.java
@@ -124,4 +124,14 @@ public class StringUtils {
return null;
}
+ public static String quantity(Number qty, String singular, String plural) {
+ if (qty.intValue() == 1)
+ return singular;
+ return plural;
+ }
+
+ public static String quantity(Number qty, String singular) {
+ return quantity(qty, singular, singular + "s");
+ }
+
}
diff --git a/vipra-util/src/main/java/de/vipra/util/Timer.java b/vipra-util/src/main/java/de/vipra/util/Timer.java
index 9ca70c51..1ec369f9 100644
--- a/vipra-util/src/main/java/de/vipra/util/Timer.java
+++ b/vipra-util/src/main/java/de/vipra/util/Timer.java
@@ -1,11 +1,17 @@
package de.vipra.util;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
public class Timer {
private long start;
+ private Map<String, Long> laps;
public long start() {
start = System.nanoTime();
+ laps = new LinkedHashMap<>();
return start;
}
@@ -19,4 +25,24 @@ public class Timer {
return lap;
}
+ public long lap(String name) {
+ long lap = lap();
+ laps.put(name, lap);
+ return lap;
+ }
+
+ public String toString() {
+ String out = null;
+ if (laps != null && laps.size() > 0) {
+ StringBuilder sb = new StringBuilder();
+ for (Entry<String, Long> e : laps.entrySet()) {
+ sb.append(", ").append(e.getKey()).append(": ").append(StringUtils.timeString(e.getValue()));
+ }
+ out = sb.toString().substring(2);
+ } else {
+ out = super.toString();
+ }
+ return out;
+ }
+
}
diff --git a/vipra-util/src/main/java/de/vipra/util/an/JsonIgnore.java b/vipra-util/src/main/java/de/vipra/util/an/JsonIgnore.java
index bd89d6f1..70e5b17b 100644
--- a/vipra-util/src/main/java/de/vipra/util/an/JsonIgnore.java
+++ b/vipra-util/src/main/java/de/vipra/util/an/JsonIgnore.java
@@ -6,9 +6,9 @@ import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Retention(RetentionPolicy.RUNTIME)
-@Target(ElementType.TYPE)
+@Target(ElementType.FIELD)
public @interface JsonIgnore {
- public boolean value() default false;
+ public boolean value() default true;
-}
+}
\ No newline at end of file
diff --git a/vipra-util/src/main/java/de/vipra/util/an/QueryIgnore.java b/vipra-util/src/main/java/de/vipra/util/an/QueryIgnore.java
new file mode 100644
index 00000000..422439e7
--- /dev/null
+++ b/vipra-util/src/main/java/de/vipra/util/an/QueryIgnore.java
@@ -0,0 +1,16 @@
+package de.vipra.util.an;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.FIELD)
+public @interface QueryIgnore {
+
+ public boolean single() default false;
+
+ public boolean multi() default false;
+
+}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/Article.java b/vipra-util/src/main/java/de/vipra/util/model/Article.java
index 17531569..72da3502 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/Article.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/Article.java
@@ -18,43 +18,43 @@ import de.vipra.util.Constants;
import de.vipra.util.FileUtils;
import de.vipra.util.MongoUtils;
import de.vipra.util.StringUtils;
-import de.vipra.util.an.JsonField;
+import de.vipra.util.an.JsonWrap;
+import de.vipra.util.an.QueryIgnore;
+@SuppressWarnings("serial")
@Entity(value = "articles", noClassnameStored = true)
public class Article extends Model implements Serializable {
- private static final long serialVersionUID = -3357348905924854240L;
-
@Id
private ObjectId id;
- @JsonField("attributes.title")
+ @JsonWrap("attributes")
private String title;
- @JsonField("attributes.text")
+ @JsonWrap("attributes")
+ @QueryIgnore(multi = true)
private String text;
- @JsonField("attributes.url")
+ @JsonWrap("attributes")
private String url;
- @JsonField("attributes.date")
+ @JsonWrap("attributes")
private Date date;
- @JsonField("attributes.complete")
- private boolean complete;
-
@Embedded
- @JsonField("attributes.stats")
- private ArticleStats stats;
+ @JsonWrap("attributes")
+ @QueryIgnore(multi = true)
+ private List<TopicRef> topics;
@Embedded
- @JsonField("attributes.topics")
- private List<TopicRef> topics;
+ @JsonWrap("attributes")
+ @QueryIgnore(multi = true)
+ private ArticleStats stats;
- @JsonField("attributes.created")
+ @JsonWrap("attributes")
private Date created = new Date();
- @JsonField("attributes.modified")
+ @JsonWrap("attributes")
private Date modified;
public ObjectId getId() {
@@ -101,22 +101,6 @@ public class Article extends Model implements Serializable {
this.date = date;
}
- public boolean isComplete() {
- return complete;
- }
-
- public void setComplete(boolean complete) {
- this.complete = complete;
- }
-
- public ArticleStats getStats() {
- return stats;
- }
-
- public void setStats(ArticleStats stats) {
- this.stats = stats;
- }
-
public void setDate(String date) {
SimpleDateFormat df = new SimpleDateFormat(Constants.DATETIME_FORMAT);
try {
@@ -132,6 +116,14 @@ public class Article extends Model implements Serializable {
this.topics = topics;
}
+ public ArticleStats getStats() {
+ return stats;
+ }
+
+ public void setStats(ArticleStats stats) {
+ this.stats = stats;
+ }
+
public Date getCreated() {
return created;
}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/ArticleStats.java b/vipra-util/src/main/java/de/vipra/util/model/ArticleStats.java
index 579d5621..06a3b1e9 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/ArticleStats.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/ArticleStats.java
@@ -1,11 +1,8 @@
package de.vipra.util.model;
import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
import org.bson.types.ObjectId;
-import org.mongodb.morphia.annotations.Embedded;
import org.mongodb.morphia.annotations.Entity;
import org.mongodb.morphia.annotations.Id;
@@ -13,13 +10,10 @@ import org.mongodb.morphia.annotations.Id;
public class ArticleStats implements Serializable {
private static final long serialVersionUID = -4712841724990200627L;
-
+
@Id
private ObjectId id;
private long wordCount;
- private long uniqueWordCount;
- @Embedded
- private Map<String, TermFrequency> uniqueWords;
public ObjectId getId() {
return id;
@@ -37,57 +31,16 @@ public class ArticleStats implements Serializable {
this.wordCount = wordCount;
}
- public long getUniqueWordCount() {
- return uniqueWordCount;
- }
-
- public void setUniqueWordCount(long uniqueWordCount) {
- this.uniqueWordCount = uniqueWordCount;
- }
-
- public Map<String, TermFrequency> getUniqueWords() {
- return uniqueWords;
- }
-
- public void setUniqueWords(Map<String, TermFrequency> uniqueWords) {
- this.uniqueWords = uniqueWords;
- }
-
- public static ArticleStats generateFromText(final String text) {
+ public static ArticleStats generateFromText(final String text, final WordMap wordMap) {
ArticleStats stats = new ArticleStats();
String[] words = text.split("\\s+");
stats.setWordCount(words.length);
- Map<String, TermFrequency> uniqueWords = new HashMap<>();
- long maxFrequency = 0;
-
- // loop and count unique words
- // also remember maximum frequency
- for (String word : words) {
- TermFrequency tf = uniqueWords.get(word);
- if (tf == null) {
- tf = new TermFrequency();
- }
- tf.incrementTermFrequency();
- if (tf.getTermFrequency() > maxFrequency) {
- maxFrequency = tf.getTermFrequency();
- }
- uniqueWords.put(word, tf);
- }
-
- // normalize frequencies
- for (Map.Entry<String, TermFrequency> entry : uniqueWords.entrySet()) {
- entry.getValue().normalizeTermFrequency(maxFrequency);
- }
-
- stats.setUniqueWordCount(uniqueWords.size());
- stats.setUniqueWords(uniqueWords);
return stats;
}
@Override
public String toString() {
- return ArticleStats.class.getSimpleName() + "[id:" + id + ", wordCount:" + wordCount + ", uniqueWordCount:"
- + uniqueWordCount + "]";
+ return ArticleStats.class.getSimpleName() + "[id:" + id + ", wordCount:" + wordCount + "]";
}
-}
+}
\ No newline at end of file
diff --git a/vipra-util/src/main/java/de/vipra/util/model/Model.java b/vipra-util/src/main/java/de/vipra/util/model/Model.java
index 70baf225..8fa5aab1 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/Model.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/Model.java
@@ -11,10 +11,9 @@ import org.bson.types.ObjectId;
import de.vipra.util.Constants;
+@SuppressWarnings("serial")
public abstract class Model implements Serializable {
- private static final long serialVersionUID = -1991594352707918633L;
-
public URI uri(URI base) {
try {
return new URI(base.toString() + "/" + getId().toString());
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TermFrequency.java b/vipra-util/src/main/java/de/vipra/util/model/TermFrequency.java
deleted file mode 100644
index 75099a27..00000000
--- a/vipra-util/src/main/java/de/vipra/util/model/TermFrequency.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package de.vipra.util.model;
-
-import java.io.Serializable;
-
-import org.mongodb.morphia.annotations.Embedded;
-
-@Embedded
-public class TermFrequency implements Serializable {
-
- private static final long serialVersionUID = 4042573510472738071L;
-
- private long termFrequency = 0;
- private double normalizedTermFrequency = 0;
-
- public long getTermFrequency() {
- return termFrequency;
- }
-
- public void setTermFrequency(long termFrequency) {
- this.termFrequency = termFrequency;
- }
-
- public double getNormalizedTermFrequency() {
- return normalizedTermFrequency;
- }
-
- public void setNormalizedTermFrequency(double normalizedTermFrequency) {
- this.normalizedTermFrequency = normalizedTermFrequency;
- }
-
- public void normalizeTermFrequency(double max) {
- setNormalizedTermFrequency(getTermFrequency() / max);
- }
-
- public void incrementTermFrequency() {
- setTermFrequency(getTermFrequency() + 1);
- }
-
- @Override
- public String toString() {
- return TermFrequency.class.getSimpleName() + "[termFrequency:" + termFrequency + ", normalizedTermFrequency:"
- + normalizedTermFrequency + "]";
- }
-
-}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/Topic.java b/vipra-util/src/main/java/de/vipra/util/model/Topic.java
index 1d9488cd..1e1d7a47 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/Topic.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/Topic.java
@@ -3,52 +3,22 @@ package de.vipra.util.model;
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
import org.bson.types.ObjectId;
-import org.mongodb.morphia.annotations.Embedded;
import org.mongodb.morphia.annotations.Entity;
import org.mongodb.morphia.annotations.Id;
-import org.mongodb.morphia.annotations.PrePersist;
-import de.vipra.util.Constants;
import de.vipra.util.MongoUtils;
-import de.vipra.util.StringUtils;
-import de.vipra.util.an.JsonWrap;
import de.vipra.util.ex.NotImplementedException;
+@SuppressWarnings("serial")
@Entity(value = "topics", noClassnameStored = true)
public class Topic extends Model implements Serializable {
- private static final long serialVersionUID = 7121629487498450992L;
-
@Id
private ObjectId id;
-
- @JsonWrap("attributes")
- private int index;
-
- @JsonWrap("attributes")
private String name;
- @Embedded
- @JsonWrap("attributes")
- private List<TopicWord> words;
-
- @JsonWrap("attributes")
- private Date created = new Date();
-
- @JsonWrap("attributes")
- private Date modified;
-
- public Topic() {}
-
- public Topic(List<TopicWord> words) {
- this.words = words;
- }
-
public ObjectId getId() {
return id;
}
@@ -61,14 +31,6 @@ public class Topic extends Model implements Serializable {
this.id = MongoUtils.objectId(id);
}
- public int getIndex() {
- return index;
- }
-
- public void setIndex(int index) {
- this.index = index;
- }
-
public String getName() {
return name;
}
@@ -77,43 +39,6 @@ public class Topic extends Model implements Serializable {
this.name = name;
}
- public List<TopicWord> getWords() {
- return words;
- }
-
- public void setWords(List<TopicWord> words) {
- this.words = words;
- }
-
- public Date getCreated() {
- return created;
- }
-
- public void setCreated(Date created) {
- this.created = created;
- }
-
- public Date getModified() {
- return modified;
- }
-
- public void setModified(Date modified) {
- this.modified = modified;
- }
-
- public String getNameFromWords() {
- String name = null;
- if (words != null && words.size() > 0) {
- int size = Math.min(Constants.AUTO_TOPIC_WORDS, words.size());
- List<String> topWords = new ArrayList<>(size);
- for (int i = 0; i < size; i++) {
- topWords.add(words.get(i).getWord());
- }
- name = StringUtils.join(topWords);
- }
- return name;
- }
-
@Override
public void fromFile(File file) throws IOException {
throw new NotImplementedException();
@@ -124,15 +49,4 @@ public class Topic extends Model implements Serializable {
throw new NotImplementedException();
}
- @Override
- public String toString() {
- return Topic.class.getSimpleName() + "[id:" + id + ", name:" + name + ", created:" + created + ", modified:"
- + modified + "]";
- }
-
- @PrePersist
- public void prePersist() {
- this.modified = new Date();
- }
-
}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java b/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java
new file mode 100644
index 00000000..5af0a958
--- /dev/null
+++ b/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java
@@ -0,0 +1,135 @@
+package de.vipra.util.model;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import org.bson.types.ObjectId;
+import org.mongodb.morphia.annotations.Embedded;
+import org.mongodb.morphia.annotations.Entity;
+import org.mongodb.morphia.annotations.Id;
+import org.mongodb.morphia.annotations.PrePersist;
+
+import de.vipra.util.Constants;
+import de.vipra.util.MongoUtils;
+import de.vipra.util.StringUtils;
+import de.vipra.util.an.JsonType;
+import de.vipra.util.an.JsonWrap;
+import de.vipra.util.an.QueryIgnore;
+import de.vipra.util.ex.NotImplementedException;
+
+@SuppressWarnings("serial")
+@JsonType("topic")
+@Entity(value = "topics", noClassnameStored = true)
+public class TopicFull extends Model implements Serializable {
+
+ @Id
+ private ObjectId id;
+
+ @JsonWrap("attributes")
+ private String name;
+
+ @JsonWrap("attributes")
+ private int index;
+
+ @Embedded
+ @JsonWrap("attributes")
+ @QueryIgnore(multi = true)
+ private List<TopicWord> topicWords;
+
+ @JsonWrap("attributes")
+ private Date created = new Date();
+
+ @JsonWrap("attributes")
+ private Date modified;
+
+ public ObjectId getId() {
+ return id;
+ }
+
+ public void setId(ObjectId id) {
+ this.id = id;
+ }
+
+ public void setId(String id) {
+ this.id = MongoUtils.objectId(id);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public int getIndex() {
+ return index;
+ }
+
+ public void setIndex(int index) {
+ this.index = index;
+ }
+
+ public List<TopicWord> getTopicWords() {
+ return topicWords;
+ }
+
+ public void setTopicWords(List<TopicWord> topicWords) {
+ this.topicWords = topicWords;
+ }
+
+ public Date getCreated() {
+ return created;
+ }
+
+ public void setCreated(Date created) {
+ this.created = created;
+ }
+
+ public Date getModified() {
+ return modified;
+ }
+
+ public void setModified(Date modified) {
+ this.modified = modified;
+ }
+
+ @Override
+ public void fromFile(File file) throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String toFileString() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String toString() {
+ return TopicFull.class.getSimpleName() + "[id:" + getId() + ", name:" + getName() + ", created:" + created
+ + ", modified:" + modified + "]";
+ }
+
+ @PrePersist
+ public void prePersist() {
+ this.modified = new Date();
+ }
+
+ public static String getNameFromWords(List<TopicWord> words) {
+ String name = null;
+ if (words != null && words.size() > 0) {
+ int size = Math.min(Constants.AUTO_TOPIC_WORDS, words.size());
+ List<String> topWords = new ArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ topWords.add(words.get(i).getWord().getWord());
+ }
+ name = StringUtils.join(topWords);
+ }
+ return name;
+ }
+
+}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TopicRef.java b/vipra-util/src/main/java/de/vipra/util/model/TopicRef.java
index e6b6df22..0380a0da 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/TopicRef.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/TopicRef.java
@@ -4,14 +4,18 @@ import java.io.Serializable;
import org.mongodb.morphia.annotations.Embedded;
import org.mongodb.morphia.annotations.Reference;
+import org.mongodb.morphia.annotations.Transient;
+import de.vipra.util.an.JsonIgnore;
+
+@SuppressWarnings("serial")
@Embedded
public class TopicRef implements Comparable<TopicRef>, Serializable {
- private static final long serialVersionUID = 3301635858822787398L;
-
+ @Transient
+ @JsonIgnore
private String topicId;
- @Reference(lazy = true)
+ @Reference
private Topic topic;
private int count;
@@ -33,6 +37,14 @@ public class TopicRef implements Comparable<TopicRef>, Serializable {
this.count = count;
}
+ public Topic getTopic() {
+ return topic;
+ }
+
+ public void setTopic(Topic topic) {
+ this.topic = topic;
+ }
+
@Override
public int compareTo(TopicRef arg0) {
return count - arg0.getCount();
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TopicWord.java b/vipra-util/src/main/java/de/vipra/util/model/TopicWord.java
index 8a1f1f20..5bf9a5cb 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/TopicWord.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/TopicWord.java
@@ -3,27 +3,28 @@ package de.vipra.util.model;
import java.io.Serializable;
import org.mongodb.morphia.annotations.Embedded;
+import org.mongodb.morphia.annotations.Reference;
+@SuppressWarnings("serial")
@Embedded
public class TopicWord implements Comparable<TopicWord>, Serializable {
- private static final long serialVersionUID = -5409441821591159243L;
-
- private String word;
+ @Reference
+ private Word word;
private double likeliness;
public TopicWord() {}
- public TopicWord(String word, double likeliness) {
+ public TopicWord(Word word, double likeliness) {
this.word = word;
this.likeliness = likeliness;
}
- public String getWord() {
+ public Word getWord() {
return word;
}
- public void setWord(String word) {
+ public void setWord(Word word) {
this.word = word;
}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/Word.java b/vipra-util/src/main/java/de/vipra/util/model/Word.java
new file mode 100644
index 00000000..2effbd0d
--- /dev/null
+++ b/vipra-util/src/main/java/de/vipra/util/model/Word.java
@@ -0,0 +1,59 @@
+package de.vipra.util.model;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.bson.types.ObjectId;
+import org.mongodb.morphia.annotations.Entity;
+import org.mongodb.morphia.annotations.Id;
+
+import de.vipra.util.MongoUtils;
+import de.vipra.util.ex.NotImplementedException;
+
+@SuppressWarnings("serial")
+@Entity(value = "words", noClassnameStored = true)
+public class Word extends Model implements Serializable {
+
+ @Id
+ private ObjectId id;
+ private String word;
+
+ public Word() {}
+
+ public Word(String word) {
+ this.word = word;
+ }
+
+ public ObjectId getId() {
+ return id;
+ }
+
+ public void setId(ObjectId id) {
+ this.id = id;
+ }
+
+ public String getWord() {
+ return word;
+ }
+
+ public void setWord(String word) {
+ this.word = word;
+ }
+
+ @Override
+ public void setId(String id) {
+ this.id = MongoUtils.objectId(id);
+ }
+
+ @Override
+ public void fromFile(File file) throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String toFileString() {
+ throw new NotImplementedException();
+ }
+
+}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/WordMap.java b/vipra-util/src/main/java/de/vipra/util/model/WordMap.java
new file mode 100644
index 00000000..76351d34
--- /dev/null
+++ b/vipra-util/src/main/java/de/vipra/util/model/WordMap.java
@@ -0,0 +1,99 @@
+package de.vipra.util.model;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import de.vipra.util.ex.DatabaseException;
+import de.vipra.util.service.DatabaseService;
+
+public class WordMap extends HashMap<String, Word> {
+
+ private static final long serialVersionUID = 8321873837437524923L;
+ public static final Logger log = LoggerFactory.getLogger(WordMap.class);
+
+ private final DatabaseService<Word> dbWords;
+ private boolean createNow = true;
+ private long newWords = 0;
+
+ public WordMap(DatabaseService<Word> dbWords) {
+ this.dbWords = dbWords;
+ List<Word> words = dbWords.getAll();
+ for (Word word : words)
+ put(word.getWord().toLowerCase(), word);
+ }
+
+ @Override
+ public Word get(Object w) {
+ String strWord = w.toString();
+ Word word = super.get(strWord.toLowerCase());
+ if (word == null) {
+ word = new Word(strWord);
+ createWord(word);
+ }
+ return word;
+ }
+
+ @Override
+ public Word put(String strWord, Word word) {
+ Word currentWord = get(strWord);
+ if (currentWord == null) {
+ if (word == null)
+ word = new Word(strWord);
+ createWord(word);
+ put(strWord, word);
+ currentWord = word;
+ } else {
+ currentWord.setWord(word.getWord());
+ try {
+ dbWords.updateSingle(currentWord);
+ } catch (DatabaseException e) {
+ log.error("could not update word in database", e);
+ throw new RuntimeException(e);
+ }
+ }
+ return currentWord;
+ }
+
+ private Word createWord(Word word) {
+ if (createNow) {
+ try {
+ dbWords.createSingle(word);
+ newWords++;
+ } catch (DatabaseException e) {
+ log.error("could not create word in database", e);
+ throw new RuntimeException(e);
+ }
+ }
+ return word;
+ }
+
+ public Word put(String strWord) {
+ return put(strWord, null);
+ }
+
+ public void create() throws DatabaseException {
+ List<Word> newWords = new ArrayList<>();
+ for (Entry<String, Word> e : this.entrySet())
+ if (e.getValue().getId() == null)
+ newWords.add(e.getValue());
+ dbWords.createMultiple(newWords);
+ this.newWords += newWords.size();
+ }
+
+ public boolean isCreateNow() {
+ return createNow;
+ }
+
+ public void setCreateNow(boolean createNow) {
+ this.createNow = createNow;
+ }
+
+ public long getNewWords() {
+ return newWords;
+ }
+
+}
diff --git a/vipra-util/src/main/java/de/vipra/util/service/DatabaseService.java b/vipra-util/src/main/java/de/vipra/util/service/DatabaseService.java
index 6172a5e2..ed3f222a 100644
--- a/vipra-util/src/main/java/de/vipra/util/service/DatabaseService.java
+++ b/vipra-util/src/main/java/de/vipra/util/service/DatabaseService.java
@@ -1,13 +1,18 @@
package de.vipra.util.service;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
import java.util.List;
+import org.bson.types.ObjectId;
import org.mongodb.morphia.Datastore;
import org.mongodb.morphia.query.Query;
import de.vipra.util.Config;
+import de.vipra.util.ListUtils;
import de.vipra.util.Mongo;
import de.vipra.util.MongoUtils;
+import de.vipra.util.an.QueryIgnore;
import de.vipra.util.ex.ConfigException;
import de.vipra.util.ex.DatabaseException;
import de.vipra.util.model.Model;
@@ -16,31 +21,70 @@ public class DatabaseService<T extends Model> implements Service<T, DatabaseExce
private final Datastore datastore;
private final Class<T> clazz;
+ private final String[] ignoredFieldsSingle;
+ private final String[] ignoredFieldsMulti;
public DatabaseService(Mongo mongo, Class<T> clazz) {
this.datastore = mongo.getDatastore();
this.clazz = clazz;
+
+ List<String> ignoreSingle = new ArrayList<>();
+ List<String> ignoreMulti = new ArrayList<>();
+ Field[] fields = clazz.getDeclaredFields();
+ for (Field field : fields) {
+ QueryIgnore qi = field.getDeclaredAnnotation(QueryIgnore.class);
+ if (qi != null) {
+ if (qi.single())
+ ignoreSingle.add(field.getName());
+ if (qi.multi())
+ ignoreMulti.add(field.getName());
+ }
+ }
+ this.ignoredFieldsSingle = ignoreSingle.toArray(new String[ignoreSingle.size()]);
+ this.ignoredFieldsMulti = ignoreMulti.toArray(new String[ignoreMulti.size()]);
}
@Override
public T getSingle(String id) {
- return datastore.get(clazz, MongoUtils.objectId(id));
+ Query<T> q = datastore.createQuery(clazz).field("_id").equal(new ObjectId(id));
+ if (ignoredFieldsSingle.length > 0)
+ q.retrievedFields(false, ignoredFieldsSingle);
+ return q.get();
}
- public List<T> getMultiple(int skip, int limit, String sortBy) {
- Query<T> q = datastore.createQuery(clazz).offset(skip).limit(limit);
+ @Override
+ public List<T> getMultiple(Integer skip, Integer limit, String sortBy) {
+ Query<T> q = datastore.createQuery(clazz);
+ if (skip != null)
+ q.offset(skip);
+ if (limit != null)
+ q.limit(limit);
if (sortBy != null)
- q = q.order(sortBy);
+ q.order(sortBy);
+ if (ignoredFieldsMulti.length > 0)
+ q.retrievedFields(false, ignoredFieldsMulti);
List<T> list = q.asList();
return list;
}
+ @Override
+ public List<T> getAll() {
+ return getMultiple(null, null, null);
+ }
+
@Override
public T createSingle(T t) throws DatabaseException {
datastore.save(t);
return t;
}
+ @Override
+ public List<T> createMultiple(Iterable<T> t) throws DatabaseException {
+ List<T> list = ListUtils.toList(t);
+ datastore.save(list);
+ return list;
+ }
+
@Override
public long deleteSingle(String id) throws DatabaseException {
return datastore.delete(MongoUtils.objectId(id)).getN();
@@ -51,10 +95,12 @@ public class DatabaseService<T extends Model> implements Service<T, DatabaseExce
datastore.save(t);
}
+ @Override
public void drop() {
datastore.getCollection(clazz).drop();
}
+ @Override
public long count() {
return datastore.getCount(clazz);
}
diff --git a/vipra-util/src/main/java/de/vipra/util/service/Service.java b/vipra-util/src/main/java/de/vipra/util/service/Service.java
index f3a4129c..659a5795 100644
--- a/vipra-util/src/main/java/de/vipra/util/service/Service.java
+++ b/vipra-util/src/main/java/de/vipra/util/service/Service.java
@@ -1,17 +1,27 @@
package de.vipra.util.service;
+import java.util.List;
+
import de.vipra.util.model.Model;
public interface Service<T extends Model, E extends Exception> {
T getSingle(String id) throws E;
+ List<T> getMultiple(Integer skip, Integer limit, String sortBy) throws E;
+
+ List<T> getAll() throws E;
+
T createSingle(T t) throws E;
+ List<T> createMultiple(Iterable<T> t) throws E;
+
long deleteSingle(String id) throws E;
void updateSingle(T t) throws E;
- void drop();
+ void drop() throws E;
+
+ long count() throws E;
}
--
GitLab