diff --git a/ma-impl.sublime-workspace b/ma-impl.sublime-workspace
index 0a9c9a9b0229e6cdc8a4c459bc0da93f5ffd99b5..1178f37887c3d459c6e40efa5159186a480d8a98 100644
--- a/ma-impl.sublime-workspace
+++ b/ma-impl.sublime-workspace
@@ -275,15 +275,6 @@
},
"buffers":
[
- {
- "contents": "stemming to lemmatization (corenlp benutzen)\ntop n words for topic name\nbuch: natural language processing with java\ndynamic lda\n\nVISUALISIERUNG!",
- "settings":
- {
- "buffer_size": 144,
- "line_ending": "Unix",
- "name": "stemming to lemmatization (corenlp benutzen)"
- }
- }
],
"build_system": "",
"build_system_choices":
@@ -463,6 +454,7 @@
"expanded_folders":
[
"/home/eike/repos/master/ma-impl",
+ "/home/eike/repos/master/ma-impl/vipra-cmd",
"/home/eike/repos/master/ma-impl/vipra-ui",
"/home/eike/repos/master/ma-impl/vipra-ui/app",
"/home/eike/repos/master/ma-impl/vipra-ui/app/adapters",
@@ -471,15 +463,21 @@
"/home/eike/repos/master/ma-impl/vipra-ui/app/routes/topics",
"/home/eike/repos/master/ma-impl/vipra-ui/app/templates",
"/home/eike/repos/master/ma-impl/vipra-ui/app/templates/articles",
- "/home/eike/repos/master/ma-impl/vipra-ui/app/templates/components"
+ "/home/eike/repos/master/ma-impl/vipra-ui/app/templates/components",
+ "/home/eike/repos/master/ma-impl/vipra-ui/app/templates/topics",
+ "/home/eike/repos/master/ma-impl/vipra-ui/app/templates/topics/show"
],
"file_history":
[
+ "/home/eike/repos/master/ma-impl/vipra-ui/app/components/topic-link.js",
+ "/home/eike/repos/master/ma-impl/vipra-ui/app/templates/articles/show.hbs",
+ "/home/eike/.local/share/vipra/jgibb/jgibb.twords",
+ "/home/eike/.local/share/vipra/jgibb/jgibb.tassign",
+ "/home/eike/repos/master/ma-impl/vipra-ui/app/templates/topics/index.hbs",
+ "/home/eike/repos/master/ma-impl/vipra-ui/app/templates/topics/show/index.hbs",
"/home/eike/Downloads/FRITZ.Box 7490 113.06.30_17.01.16_2147.export",
"/home/eike/repos/master/ma-impl/vm/data/test-1.json",
"/home/eike/repos/master/ma-impl/vm/data/test-2.json",
- "/home/eike/.local/share/vipra/jgibb/jgibb.twords",
- "/home/eike/.local/share/vipra/jgibb/jgibb.tassign",
"/home/eike/repos/master/ma-impl/vipra-ui/app/routes/topics/index.js",
"/home/eike/repos/master/ma-impl/vipra-ui/app/templates/articles/index.hbs",
"/home/eike/repos/master/ma-impl/vipra-ui/app/adapters/application.js",
@@ -494,18 +492,14 @@
"/home/eike/repos/master/ma-impl/vipra-ui/app/models/topic.js",
"/home/eike/repos/master/ma-impl/vipra-ui/app/routes/topics/show/edit.js",
"/home/eike/repos/master/ma-impl/vipra-ui/app/templates/components/topics-list.hbs",
- "/home/eike/repos/master/ma-impl/vipra-ui/app/templates/topics/index.hbs",
"/home/eike/repos/master/ma-impl/vipra-ui/app/router.js",
- "/home/eike/repos/master/ma-impl/vipra-ui/app/templates/topics/show/index.hbs",
"/home/eike/repos/master/ma-impl/vipra-ui/app/routes/topics/show.js",
"/home/eike/repos/master/ma-impl/vipra-ui/app/templates/topics/show/edit.hbs",
"/home/eike/repos/master/ma-impl/vipra-ui/app/templates/topics/show.hbs",
"/home/eike/repos/master/ma-impl/vipra-ui/app/helpers/topic-share.js",
"/home/eike/repos/master/ma-impl/vipra-ui/app/styles/app.css",
- "/home/eike/repos/master/ma-impl/vipra-ui/app/components/topic-link.js",
"/home/eike/repos/master/ma-impl/vipra-ui/app/templates/components/topic-link.hbs",
"/home/eike/repos/master/ma-impl/vipra-ui/app/templates/topics/edit.hbs",
- "/home/eike/repos/master/ma-impl/vipra-ui/app/templates/articles/show.hbs",
"/home/eike/repos/master/ma-impl/vipra-ui/app/routes/articles/show.js",
"/home/eike/repos/master/ma-impl/vipra-ui/app/templates/articles/edit.hbs",
"/home/eike/repos/master/ma-impl/vipra-ui/app/routes/topics/edit.js",
@@ -926,38 +920,8 @@
"groups":
[
{
- "selected": 0,
"sheets":
[
- {
- "buffer": 0,
- "semi_transient": false,
- "settings":
- {
- "buffer_size": 144,
- "regions":
- {
- },
- "selection":
- [
- [
- 99,
- 99
- ]
- ],
- "settings":
- {
- "auto_name": "stemming to lemmatization (corenlp benutzen)",
- "default_dir": "/home/eike/repos/master/ma-impl",
- "syntax": "Packages/Text/Plain text.tmLanguage"
- },
- "translation.x": 0.0,
- "translation.y": 0.0,
- "zoom_level": 1.0
- },
- "stack_index": 0,
- "type": "text"
- }
]
}
],
diff --git a/vipra-cmd/pom.xml b/vipra-cmd/pom.xml
index 5aed56ed1592d2fc7f23fb6262bfb56bdbc0963e..eb06c6280c715624504631972ef393625d0854d6 100644
--- a/vipra-cmd/pom.xml
+++ b/vipra-cmd/pom.xml
@@ -86,6 +86,11 @@
<artifactId>log4j-slf4j-impl</artifactId>
<version>${log4jVersion}</version>
</dependency>
+ <dependency>
+ <groupId>uk.org.lidalia</groupId>
+ <artifactId>sysout-over-slf4j</artifactId>
+ <version>1.0.2</version>
+ </dependency>
<!-- MongoDB Database Adapter -->
<dependency>
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/Main.java b/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
index 4c488805545eb165f9bb8e73868370a9b7e6e88e..4a08e5fc1fcec0670305dc15f258b436e2ebfc87 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
@@ -30,6 +30,7 @@ import de.vipra.util.ConsoleUtils;
import de.vipra.util.ConsoleUtils.Choice;
import de.vipra.util.StringUtils;
import de.vipra.util.Timer;
+import uk.org.lidalia.sysoutslf4j.context.SysOutOverSLF4J;
public class Main {
@@ -38,6 +39,7 @@ public class Main {
static {
MorphiaLoggerFactory.registerLogger(SLF4JLoggerImplFactory.class);
+ SysOutOverSLF4J.sendSystemOutAndErrToSLF4J();
}
public static void main(String[] args) {
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbLDAAnalyzer.java b/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbLDAAnalyzer.java
index 684f0f65e536fbc0ee5662b5a4d50eb67fb5d710..df84fbe297193d17a2660f7c8fadbb6069aa540f 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbLDAAnalyzer.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbLDAAnalyzer.java
@@ -17,9 +17,11 @@ import de.vipra.util.Config;
import de.vipra.util.ConvertStream;
import de.vipra.util.StringUtils;
import de.vipra.util.ex.ConfigException;
-import de.vipra.util.model.Topic;
+import de.vipra.util.model.TopicFull;
import de.vipra.util.model.TopicRef;
import de.vipra.util.model.TopicWord;
+import de.vipra.util.model.Word;
+import de.vipra.util.model.WordMap;
import jgibblda.Estimator;
import jgibblda.Inferencer;
import jgibblda.LDACmdOption;
@@ -33,13 +35,14 @@ public class JGibbLDAAnalyzer extends LDAAnalyzer {
private File modelDir;
private File modelFile;
private LDACmdOption options;
+ private WordMap wordMap;
protected JGibbLDAAnalyzer() {
super("JGibb Analyzer");
}
@Override
- public void init(Config config) throws LDAAnalyzerException {
+ public void init(Config config, WordMap wordMap) throws LDAAnalyzerException {
options = new LDACmdOption();
try {
@@ -57,6 +60,8 @@ public class JGibbLDAAnalyzer extends LDAAnalyzer {
options.dfile = modelFile.getName();
options.modelName = "jgibb";
+
+ this.wordMap = wordMap;
}
private void estimate() {
@@ -81,28 +86,40 @@ public class JGibbLDAAnalyzer extends LDAAnalyzer {
}
@Override
- public ConvertStream<Topic> getTopicDefinitions() throws LDAAnalyzerException {
+ public ConvertStream<TopicFull> getTopicDefinitions() throws LDAAnalyzerException {
File twords = new File(modelDir, "jgibb.twords");
try {
- return new ConvertStream<Topic>(twords) {
+ return new ConvertStream<TopicFull>(twords) {
@Override
- public Topic convert(String line) {
- Topic topicDef = new Topic();
+ public TopicFull convert(String line) {
+ TopicFull topicDef = new TopicFull();
List<TopicWord> topicWords = new ArrayList<>();
+
+ // get index of topic
+ // the index will be used as a temporary id until the topic
+ // is created in the database
Integer index = StringUtils.getFirstNumber(line);
if (index == null) {
log.error("could not extract topic index from line: " + line);
} else {
topicDef.setIndex(index);
}
+
+ // get all lines that follow until the next topic is
+ // discovered (line does not start with a tab)
String nextLine;
while ((nextLine = nextLine()) != null) {
if (nextLine.startsWith("\t")) {
String[] parts = nextLine.trim().split("\\s+");
try {
- topicWords.add(new TopicWord(parts[0], Double.parseDouble(parts[1])));
+ Word word = wordMap.get(parts[0]);
+ double likeliness = Double.parseDouble(parts[1]);
+ TopicWord topicWord = new TopicWord(word, likeliness);
+ topicWords.add(topicWord);
} catch (NumberFormatException e) {
log.error("could not parse number in line: " + nextLine);
+ } catch (ArrayIndexOutOfBoundsException e) {
+ log.error("ill formatted topic definition in line: " + nextLine);
}
} else {
buffer(nextLine);
@@ -110,8 +127,8 @@ public class JGibbLDAAnalyzer extends LDAAnalyzer {
}
}
Collections.sort(topicWords);
- topicDef.setWords(topicWords);
- topicDef.setName(topicDef.getNameFromWords());
+ topicDef.setTopicWords(topicWords);
+ topicDef.setName(TopicFull.getNameFromWords(topicWords));
return topicDef;
}
};
@@ -131,6 +148,9 @@ public class JGibbLDAAnalyzer extends LDAAnalyzer {
Map<String, Integer> countMap = new HashMap<>();
String[] wordList = line.split("\\s+");
for (String word : wordList) {
+ // the file uses a simple integer as topic ids. Use this
+ // id as a temporary id until the topic is created in
+ // the database
String topic = word.split(":")[1];
Integer count = countMap.get(topic);
countMap.put(topic, count == null ? 1 : count + 1);
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/lda/LDAAnalyzer.java b/vipra-cmd/src/main/java/de/vipra/cmd/lda/LDAAnalyzer.java
index dc3e11319d6591d1172d194055219ef05f090a7a..6a34f1ceec90eda810685d7e38b067906723bba4 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/lda/LDAAnalyzer.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/lda/LDAAnalyzer.java
@@ -7,8 +7,9 @@ import de.vipra.util.Config;
import de.vipra.util.Config.Key;
import de.vipra.util.Constants;
import de.vipra.util.ConvertStream;
-import de.vipra.util.model.Topic;
+import de.vipra.util.model.TopicFull;
import de.vipra.util.model.TopicRef;
+import de.vipra.util.model.WordMap;
public abstract class LDAAnalyzer {
@@ -22,15 +23,32 @@ public abstract class LDAAnalyzer {
return name;
}
- public abstract void init(Config config) throws LDAAnalyzerException;
+ public abstract void init(Config config, WordMap wordMap) throws LDAAnalyzerException;
public abstract void analyze() throws LDAAnalyzerException;
- public abstract ConvertStream<Topic> getTopicDefinitions() throws LDAAnalyzerException;
+ /**
+ * Returns a converting stream of topics, read from the topic definition
+ * file. Usually, a topic definition consists of a list of words, that are
+ * assigned to that topic with a certain likeliness.
+ *
+ * @return topic definition stream
+ * @throws LDAAnalyzerException
+ */
+ public abstract ConvertStream<TopicFull> getTopicDefinitions() throws LDAAnalyzerException;
+ /**
+ * Returns a converting stream of lists of topic references. Normally, topic
+ * modeling outputs topics for each word of each document. These references
+ * are returned by this function.
+ *
+ * @return stream of lists of topic references per document (ordered by
+ * index)
+ * @throws LDAAnalyzerException
+ */
public abstract ConvertStream<List<TopicRef>> getTopics() throws LDAAnalyzerException;
- public static LDAAnalyzer getAnalyzer(Config config) throws LDAAnalyzerException {
+ public static LDAAnalyzer getAnalyzer(Config config, WordMap wordMap) throws LDAAnalyzerException {
LDAAnalyzer analyzer = null;
switch (Constants.Analyzer.fromString(config.getString(Key.ANALYZER))) {
case JGIBB:
@@ -39,7 +57,7 @@ public abstract class LDAAnalyzer {
analyzer = new JGibbLDAAnalyzer();
break;
}
- analyzer.init(config);
+ analyzer.init(config, wordMap);
return analyzer;
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/model/ProcessedArticle.java b/vipra-cmd/src/main/java/de/vipra/cmd/model/ProcessedArticle.java
index 897bcc0ada11c858abfd05c44dcf07c4b9f84025..aea42f749f606fc774e903e6d683296c770d91cb 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/model/ProcessedArticle.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/model/ProcessedArticle.java
@@ -6,6 +6,7 @@ import org.mongodb.morphia.annotations.Transient;
import de.vipra.cmd.text.ProcessedText;
+@SuppressWarnings("serial")
@Entity(value = "articles", noClassnameStored = true)
public class ProcessedArticle extends de.vipra.util.model.Article {
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java
index 178ec40448a8b5c801f1034fbcb62a2c3e246995..d271eabc64c75a79c23e36812ad7e02b68ee4072 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java
@@ -13,7 +13,8 @@ import de.vipra.cmd.model.ProcessedArticle;
import de.vipra.util.Config;
import de.vipra.util.ConsoleUtils;
import de.vipra.util.ex.ConfigException;
-import de.vipra.util.model.Topic;
+import de.vipra.util.model.TopicFull;
+import de.vipra.util.model.Word;
import de.vipra.util.service.DatabaseService;
public class ClearCommand implements Command {
@@ -24,7 +25,8 @@ public class ClearCommand implements Command {
private boolean defaults;
private Config config;
private DatabaseService<ProcessedArticle> dbArticles;
- private DatabaseService<Topic> dbTopics;
+ private DatabaseService<TopicFull> dbTopics;
+ private DatabaseService<Word> dbWords;
public ClearCommand(boolean defaults) {
this.defaults = defaults;
@@ -34,7 +36,8 @@ public class ClearCommand implements Command {
try {
config = Config.getConfig();
dbArticles = DatabaseService.getDatabaseService(config, ProcessedArticle.class);
- dbTopics = DatabaseService.getDatabaseService(config, Topic.class);
+ dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class);
+ dbWords = DatabaseService.getDatabaseService(config, Word.class);
} catch (Exception e) {
throw new ClearException(e);
}
@@ -42,6 +45,7 @@ public class ClearCommand implements Command {
out.info("clearing database");
dbArticles.drop();
dbTopics.drop();
+ dbWords.drop();
out.info("clearing filebase");
File dataDir = config.getDataDirectory();
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
index 2f85032520412e00a193a14b93a3913f2d73de07..1a54d02dc32420a22cad46339522e56c90855786 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
@@ -33,8 +33,10 @@ import de.vipra.util.StringUtils;
import de.vipra.util.Timer;
import de.vipra.util.ex.DatabaseException;
import de.vipra.util.model.ArticleStats;
-import de.vipra.util.model.Topic;
+import de.vipra.util.model.TopicFull;
import de.vipra.util.model.TopicRef;
+import de.vipra.util.model.Word;
+import de.vipra.util.model.WordMap;
import de.vipra.util.service.DatabaseService;
public class ImportCommand implements Command {
@@ -46,9 +48,11 @@ public class ImportCommand implements Command {
private JSONParser parser = new JSONParser();
private Config config;
private DatabaseService<ProcessedArticle> dbArticles;
- private DatabaseService<Topic> dbTopics;
+ private DatabaseService<TopicFull> dbTopics;
+ private DatabaseService<Word> dbWords;
private Filebase filebase;
private Processor preprocessor;
+ private WordMap wordMap;
private LDAAnalyzer analyzer;
/**
@@ -106,7 +110,7 @@ public class ImportCommand implements Command {
try {
// preprocess text and generate text statistics
ProcessedText processedText = preprocessor.preprocess(article.getText());
- ArticleStats articleStats = ArticleStats.generateFromText(processedText.getText());
+ ArticleStats articleStats = ArticleStats.generateFromText(processedText.getText(), wordMap);
// add article to mongodb
article.setProcessedText(processedText);
@@ -166,14 +170,15 @@ public class ImportCommand implements Command {
* @throws DatabaseException
*/
private Map<String, String> saveTopicDefinitions() throws LDAAnalyzerException, DatabaseException {
- ConvertStream<Topic> topics = analyzer.getTopicDefinitions();
+ ConvertStream<TopicFull> topics = analyzer.getTopicDefinitions();
Map<String, String> topicIndexMap = new HashMap<>();
// recreate topics in database
+ // create one topic at a time for less memory usage
dbTopics.drop();
- for (Topic topic : topics) {
- Topic newTopic = dbTopics.createSingle(topic);
- topicIndexMap.put(Integer.toString(newTopic.getIndex()), newTopic.getId().toString());
+ for (TopicFull topic : topics) {
+ dbTopics.createSingle(topic);
+ topicIndexMap.put(Integer.toString(topic.getIndex()), topic.getId().toString());
}
return topicIndexMap;
@@ -223,10 +228,12 @@ public class ImportCommand implements Command {
try {
config = Config.getConfig();
dbArticles = DatabaseService.getDatabaseService(config, ProcessedArticle.class);
- dbTopics = DatabaseService.getDatabaseService(config, Topic.class);
+ dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class);
+ dbWords = DatabaseService.getDatabaseService(config, Word.class);
filebase = Filebase.getFilebase(config);
preprocessor = Processor.getPreprocessor(config);
- analyzer = LDAAnalyzer.getAnalyzer(config);
+ wordMap = new WordMap(dbWords);
+ analyzer = LDAAnalyzer.getAnalyzer(config, wordMap);
out.info("using data directory: " + config.getDataDirectory().getAbsolutePath());
out.info("using preprocessor: " + preprocessor.getName());
@@ -238,26 +245,32 @@ public class ImportCommand implements Command {
// import files into database and filebase
out.info("file import");
long imported = importFiles(files);
- long durImport = timer.lap();
+ timer.lap("import");
// write filebase
out.info("writing file index");
filebase.close();
- timer.lap();
+ timer.lap("filebase write");
// do topic modeling
out.info("topic modeling");
analyzer.analyze();
- long durAnalyze = timer.lap();
+ timer.lap("topic modeling");
// save topic model
- out.info("saving topic models");
+ out.info("saving topic definitions");
Map<String, String> topicIndexMap = saveTopicDefinitions();
+ timer.lap("saving topics");
+
+ // save topic refs
+ out.info("saving document topics");
saveTopicsPerDocument(topicIndexMap);
+ timer.lap("saving topic refs");
- out.info("imported " + imported + " " + (imported == 1 ? "article" : "articles"));
- out.info("import: " + StringUtils.timeString(durImport) + ", analyze: "
- + StringUtils.timeString(durAnalyze));
+ out.info("imported " + imported + " new " + StringUtils.quantity(imported, "article"));
+ long newWords = wordMap.getNewWords();
+ out.info("imported " + newWords + " new " + StringUtils.quantity(newWords, "word"));
+ out.info(timer.toString());
} catch (Exception e) {
throw new ExecutionException(e);
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/StatsCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/StatsCommand.java
index 774fd43291690fbb0d065166f1ec7f43be745e03..4793f14f216a68e9b6816010c471b8d34a091d6e 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/StatsCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/StatsCommand.java
@@ -12,7 +12,7 @@ import de.vipra.cmd.file.Filebase;
import de.vipra.util.Config;
import de.vipra.util.StringUtils;
import de.vipra.util.ex.ConfigException;
-import de.vipra.util.model.Topic;
+import de.vipra.util.model.TopicFull;
import de.vipra.util.service.DatabaseService;
public class StatsCommand implements Command {
@@ -22,7 +22,7 @@ public class StatsCommand implements Command {
private Config config;
private Filebase filebase;
- private DatabaseService<Topic> dbTopics;
+ private DatabaseService<TopicFull> dbTopics;
private void stats() {
File modelFile = filebase.getModelFile();
@@ -37,7 +37,7 @@ public class StatsCommand implements Command {
try {
config = Config.getConfig();
filebase = Filebase.getFilebase(config);
- dbTopics = DatabaseService.getDatabaseService(config, Topic.class);
+ dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class);
stats();
} catch (IOException | ConfigException | FilebaseException e) {
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/text/CoreNLPProcessor.java b/vipra-cmd/src/main/java/de/vipra/cmd/text/CoreNLPProcessor.java
index d0534bca8a96cb1943cb90951155fcb20a169a83..afc32c130db9ab84115c5be5218d529b8ba4bf25 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/text/CoreNLPProcessor.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/text/CoreNLPProcessor.java
@@ -4,6 +4,7 @@ import java.util.List;
import java.util.Properties;
import de.vipra.cmd.ex.PreprocessorException;
+import de.vipra.util.Constants;
import edu.stanford.nlp.ling.CoreAnnotations.SentencesAnnotation;
import edu.stanford.nlp.ling.CoreAnnotations.TokensAnnotation;
import edu.stanford.nlp.ling.CoreLabel;
@@ -41,7 +42,8 @@ public class CoreNLPProcessor extends Processor {
sb.append(word.word()).append(" ");
}
}
- return new ProcessedText(sb.toString().trim());
+ String text = sb.toString().trim().replaceAll(Constants.CHARS_DISALLOWED, "").replaceAll("\\s+", " ");
+ return new ProcessedText(text);
}
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/text/Processor.java b/vipra-cmd/src/main/java/de/vipra/cmd/text/Processor.java
index fcf6521d6cf66f89724d0e972971f7a94bc1e57e..7b3974e7bf798aa4153843e8bc5fb55ea0a2c488 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/text/Processor.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/text/Processor.java
@@ -1,6 +1,5 @@
package de.vipra.cmd.text;
-import java.util.Arrays;
import java.util.List;
import de.vipra.cmd.ex.PreprocessorException;
diff --git a/vipra-rest/pom.xml b/vipra-rest/pom.xml
index 8b81b7ee714fb55b749b29339048e3261b561fb8..ff073cf971f5db8c048fa8280b9a875cc5d7bc43 100644
--- a/vipra-rest/pom.xml
+++ b/vipra-rest/pom.xml
@@ -34,22 +34,7 @@
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
- <version>${jerseyVersion}</version>
- </dependency>
- <dependency>
- <groupId>org.glassfish.jersey.test-framework</groupId>
- <artifactId>jersey-test-framework-core</artifactId>
- <version>${jerseyVersion}</version>
- </dependency>
- <dependency>
- <groupId>org.glassfish.jersey.test-framework.providers</groupId>
- <artifactId>jersey-test-framework-provider-simple</artifactId>
- <version>${jerseyVersion}</version>
- </dependency>
- <dependency>
- <groupId>com.github.fge</groupId>
- <artifactId>json-patch</artifactId>
- <version>1.9</version>
+ <version>2.22.1</version>
</dependency>
<!-- Servlet API -->
diff --git a/vipra-rest/src/main/java/de/vipra/rest/provider/ObjectMapperProvider.java b/vipra-rest/src/main/java/de/vipra/rest/provider/ObjectMapperProvider.java
index 9289222c737362b445d2e4318b2686381e424c40..e7f5577aaca3739c57e2515e223ad87e444e8cde 100644
--- a/vipra-rest/src/main/java/de/vipra/rest/provider/ObjectMapperProvider.java
+++ b/vipra-rest/src/main/java/de/vipra/rest/provider/ObjectMapperProvider.java
@@ -5,6 +5,7 @@ import java.text.SimpleDateFormat;
import javax.ws.rs.ext.ContextResolver;
import javax.ws.rs.ext.Provider;
+import org.bson.types.ObjectId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -15,9 +16,11 @@ import com.fasterxml.jackson.databind.module.SimpleModule;
import de.vipra.rest.serializer.GenericDeserializer;
import de.vipra.rest.serializer.GenericSerializer;
+import de.vipra.rest.serializer.ObjectIdDeserializer;
+import de.vipra.rest.serializer.ObjectIdSerializer;
import de.vipra.util.Constants;
import de.vipra.util.model.Article;
-import de.vipra.util.model.Topic;
+import de.vipra.util.model.TopicFull;
@Provider
public class ObjectMapperProvider implements ContextResolver<ObjectMapper> {
@@ -39,8 +42,12 @@ public class ObjectMapperProvider implements ContextResolver<ObjectMapper> {
SimpleModule module = new SimpleModule();
module.addSerializer(Article.class, new GenericSerializer<Article>(Article.class));
module.addDeserializer(Article.class, new GenericDeserializer<Article>(Article.class));
- module.addSerializer(Topic.class, new GenericSerializer<Topic>(Topic.class));
- module.addDeserializer(Topic.class, new GenericDeserializer<Topic>(Topic.class));
+
+ module.addSerializer(TopicFull.class, new GenericSerializer<TopicFull>(TopicFull.class));
+ module.addDeserializer(TopicFull.class, new GenericDeserializer<TopicFull>(TopicFull.class));
+
+ module.addSerializer(ObjectId.class, new ObjectIdSerializer());
+ module.addDeserializer(ObjectId.class, new ObjectIdDeserializer());
final ObjectMapper mapper = new ObjectMapper();
mapper.enable(SerializationFeature.INDENT_OUTPUT);
diff --git a/vipra-rest/src/main/java/de/vipra/rest/resource/ArticleResource.java b/vipra-rest/src/main/java/de/vipra/rest/resource/ArticleResource.java
index e3c5d06aa49607cc2c3cf7990dc7f7e33b0e88dd..bf182eaacc59fe0f56f75fa70703944608d1c954 100644
--- a/vipra-rest/src/main/java/de/vipra/rest/resource/ArticleResource.java
+++ b/vipra-rest/src/main/java/de/vipra/rest/resource/ArticleResource.java
@@ -27,12 +27,11 @@ import de.vipra.rest.Messages;
import de.vipra.rest.PATCH;
import de.vipra.rest.model.APIError;
import de.vipra.rest.model.Wrapper;
-import de.vipra.rest.service.ArticleService;
import de.vipra.util.Config;
-import de.vipra.util.Mongo;
import de.vipra.util.ex.ConfigException;
import de.vipra.util.ex.DatabaseException;
import de.vipra.util.model.Article;
+import de.vipra.util.service.DatabaseService;
@Path("articles")
public class ArticleResource {
@@ -40,20 +39,19 @@ public class ArticleResource {
@Context
UriInfo uri;
- Cache<String, Article> articleCache;
-
- final ArticleService service;
+ final Cache<String, Article> articleCache;
+ final DatabaseService<Article> service;
public ArticleResource(@Context ServletContext servletContext) throws ConfigException, IOException {
Config config = Config.getConfig();
- Mongo mongo = Mongo.getInstance(config);
- service = new ArticleService(mongo);
+ service = DatabaseService.getDatabaseService(config, Article.class);
CacheManager manager = (CacheManager) servletContext.getAttribute("cachemanager");
- articleCache = manager.getCache("articlecache", String.class, Article.class);
+ Cache<String, Article> articleCache = manager.getCache("articlecache", String.class, Article.class);
if (articleCache == null)
articleCache = manager.createCache("articlecache",
CacheConfigurationBuilder.newCacheConfigurationBuilder().buildConfig(String.class, Article.class));
+ this.articleCache = articleCache;
}
@GET
@@ -61,7 +59,7 @@ public class ArticleResource {
public Response getArticles(@QueryParam("skip") @DefaultValue("0") int skip,
@QueryParam("limit") @DefaultValue("0") int limit,
@QueryParam("sort") @DefaultValue("date") String sortBy) {
- List<Article> articles = service.getMultiple(uri.getAbsolutePath(), skip, limit, sortBy);
+ List<Article> articles = service.getMultiple(skip, limit, sortBy);
Wrapper<List<Article>> res = new Wrapper<>(articles);
return Response.ok().entity(res).tag(res.tag()).build();
}
diff --git a/vipra-rest/src/main/java/de/vipra/rest/resource/PingResource.java b/vipra-rest/src/main/java/de/vipra/rest/resource/PingResource.java
deleted file mode 100644
index 028c1cafc565077b56471b27eed997bb7c5ad2c4..0000000000000000000000000000000000000000
--- a/vipra-rest/src/main/java/de/vipra/rest/resource/PingResource.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package de.vipra.rest.resource;
-
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-@Path("ping")
-public class PingResource {
-
- @GET
- @Produces(MediaType.TEXT_PLAIN)
- public Response ping() {
- return Response.ok().entity("running").build();
- }
-
-}
diff --git a/vipra-rest/src/main/java/de/vipra/rest/resource/TopicResource.java b/vipra-rest/src/main/java/de/vipra/rest/resource/TopicResource.java
index df56b8c3b2273caaf7eb884ba7c8bee123998b3a..44cc2964da7a70befbeba57c8bc4b269fa9543ce 100644
--- a/vipra-rest/src/main/java/de/vipra/rest/resource/TopicResource.java
+++ b/vipra-rest/src/main/java/de/vipra/rest/resource/TopicResource.java
@@ -25,13 +25,11 @@ import de.vipra.rest.Messages;
import de.vipra.rest.PATCH;
import de.vipra.rest.model.APIError;
import de.vipra.rest.model.Wrapper;
-import de.vipra.rest.service.TopicService;
import de.vipra.util.Config;
-import de.vipra.util.Mongo;
import de.vipra.util.ex.ConfigException;
import de.vipra.util.ex.DatabaseException;
-import de.vipra.util.model.Article;
-import de.vipra.util.model.Topic;
+import de.vipra.util.model.TopicFull;
+import de.vipra.util.service.DatabaseService;
@Path("topics")
public class TopicResource {
@@ -39,28 +37,27 @@ public class TopicResource {
@Context
UriInfo uri;
- Cache<String, Topic> topicCache;
-
- TopicService service;
+ final Cache<String, TopicFull> topicCache;
+ final DatabaseService<TopicFull> service;
public TopicResource(@Context ServletContext servletContext) throws ConfigException, IOException {
Config config = Config.getConfig();
- Mongo mongo = Mongo.getInstance(config);
- service = new TopicService(mongo);
+ service = DatabaseService.getDatabaseService(config, TopicFull.class);
CacheManager manager = (CacheManager) servletContext.getAttribute("cachemanager");
- topicCache = manager.getCache("topiccache", String.class, Topic.class);
+ Cache<String, TopicFull> topicCache = manager.getCache("topiccache", String.class, TopicFull.class);
if (topicCache == null)
- topicCache = manager.createCache("topiccache",
- CacheConfigurationBuilder.newCacheConfigurationBuilder().buildConfig(String.class, Topic.class));
+ topicCache = manager.createCache("topiccache", CacheConfigurationBuilder.newCacheConfigurationBuilder()
+ .buildConfig(String.class, TopicFull.class));
+ this.topicCache = topicCache;
}
@GET
@Produces(APIMediaType.APPLICATION_JSONAPI)
public Response getTopics(@QueryParam("skip") @DefaultValue("0") int skip,
@QueryParam("limit") @DefaultValue("0") int limit) {
- List<Topic> topics = service.getMultiple(uri.getAbsolutePath(), skip, limit, null);
- Wrapper<List<Topic>> res = new Wrapper<>(topics);
+ List<TopicFull> topics = service.getMultiple(skip, limit, null);
+ Wrapper<List<TopicFull>> res = new Wrapper<>(topics);
return Response.ok().entity(res).tag(res.tag()).build();
}
@@ -69,14 +66,14 @@ public class TopicResource {
@Consumes(APIMediaType.APPLICATION_JSONAPI)
@Path("{id}")
public Response getTopic(@PathParam("id") String id) {
- Wrapper<Topic> res = new Wrapper<>();
+ Wrapper<TopicFull> res = new Wrapper<>();
if (id == null || id.trim().length() == 0) {
res.addError(new APIError(Response.Status.BAD_REQUEST, "ID is empty",
String.format(Messages.BAD_REQUEST, "id cannot be empty")));
return Response.status(Response.Status.BAD_REQUEST).entity(res).build();
}
- Topic topic = getSingle(id);
+ TopicFull topic = getSingle(id);
if (topic != null) {
res.setData(topic);
@@ -92,9 +89,9 @@ public class TopicResource {
@Consumes(APIMediaType.APPLICATION_JSONAPI)
@Produces(APIMediaType.APPLICATION_JSONAPI)
@Path("{id}")
- public Response replaceTopic(@PathParam("id") String id, Wrapper<Topic> wrapper) {
- Topic topic = wrapper.getData();
- Wrapper<Topic> res = new Wrapper<>();
+ public Response replaceTopic(@PathParam("id") String id, Wrapper<TopicFull> wrapper) {
+ TopicFull topic = wrapper.getData();
+ Wrapper<TopicFull> res = new Wrapper<>();
try {
service.updateSingle(topic);
topicCache.put(id, topic);
@@ -111,15 +108,15 @@ public class TopicResource {
@Consumes(APIMediaType.APPLICATION_JSONAPI)
@Produces(APIMediaType.APPLICATION_JSONAPI)
@Path("{id}")
- public Response updateTopic(@PathParam("id") String id, Wrapper<Topic> wrapper) {
- Topic newTopic = wrapper.getData();
- Topic topic = getSingle(id);
+ public Response updateTopic(@PathParam("id") String id, Wrapper<TopicFull> wrapper) {
+ TopicFull newTopic = wrapper.getData();
+ TopicFull topic = getSingle(id);
// TODO implement
return null;
}
- private Topic getSingle(String id) {
- Topic topic = topicCache.get(id);
+ private TopicFull getSingle(String id) {
+ TopicFull topic = topicCache.get(id);
if (topic == null) {
topic = service.getSingle(id);
if (topic != null)
diff --git a/vipra-rest/src/main/java/de/vipra/rest/serializer/GenericDeserializer.java b/vipra-rest/src/main/java/de/vipra/rest/serializer/GenericDeserializer.java
index 710073d2687bba690a903960188d9dfd147cfad6..ff9ea7d98237212ef1c0b82d7cffdef0a520781e 100644
--- a/vipra-rest/src/main/java/de/vipra/rest/serializer/GenericDeserializer.java
+++ b/vipra-rest/src/main/java/de/vipra/rest/serializer/GenericDeserializer.java
@@ -8,6 +8,7 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
@@ -17,7 +18,6 @@ import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonDeserializer;
import de.vipra.util.an.JsonField;
-import de.vipra.util.an.JsonIgnore;
import de.vipra.util.an.JsonWrap;
import de.vipra.util.model.Model;
@@ -32,7 +32,8 @@ public class GenericDeserializer<T extends Model> extends JsonDeserializer<T> {
Field[] fields = clazz.getDeclaredFields();
for (Field field : fields) {
- if (Modifier.isPrivate(field.getModifiers())) {
+ int modifiers = field.getModifiers();
+ if (Modifier.isPrivate(modifiers) && !Modifier.isStatic(modifiers)) {
field.setAccessible(true);
JsonIgnore ji = field.getDeclaredAnnotation(JsonIgnore.class);
diff --git a/vipra-rest/src/main/java/de/vipra/rest/serializer/GenericSerializer.java b/vipra-rest/src/main/java/de/vipra/rest/serializer/GenericSerializer.java
index 85311f4a35b009239972ebd794fdbfe7153e81e1..b7c896c0e33cb600c4ac2895fefdc12b55ea7a29 100644
--- a/vipra-rest/src/main/java/de/vipra/rest/serializer/GenericSerializer.java
+++ b/vipra-rest/src/main/java/de/vipra/rest/serializer/GenericSerializer.java
@@ -11,9 +11,6 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import org.bson.types.ObjectId;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -41,11 +38,14 @@ public class GenericSerializer<T extends Model> extends JsonSerializer<T> {
Field[] fields = clazz.getDeclaredFields();
for (Field field : fields) {
- if (Modifier.isPrivate(field.getModifiers())) {
+ int modifiers = field.getModifiers();
+ if (Modifier.isPrivate(modifiers) && !Modifier.isStatic(modifiers)) {
field.setAccessible(true);
- JsonIgnore ji = field.getDeclaredAnnotation(JsonIgnore.class);
- if (ji != null && ji.value())
+ com.fasterxml.jackson.annotation.JsonIgnore ji1 = field
+ .getDeclaredAnnotation(com.fasterxml.jackson.annotation.JsonIgnore.class);
+ de.vipra.util.an.JsonIgnore ji2 = field.getDeclaredAnnotation(de.vipra.util.an.JsonIgnore.class);
+ if ((ji1 != null && ji1.value()) || (ji2 != null && ji2.value()))
continue;
String name = field.getName();
@@ -91,12 +91,8 @@ public class GenericSerializer<T extends Model> extends JsonSerializer<T> {
e.printStackTrace();
}
- if (v != null) {
- if (v instanceof ObjectId)
- v = ((ObjectId) v).toString();
-
+ if (v != null)
pathAdd(map, entry.getKey(), v);
- }
}
serializers.defaultSerializeValue(map, gen);
diff --git a/vipra-rest/src/main/java/de/vipra/rest/serializer/ObjectIdDeserializer.java b/vipra-rest/src/main/java/de/vipra/rest/serializer/ObjectIdDeserializer.java
new file mode 100644
index 0000000000000000000000000000000000000000..4a6f5994ef36e243e63d642c73a76a12781ede36
--- /dev/null
+++ b/vipra-rest/src/main/java/de/vipra/rest/serializer/ObjectIdDeserializer.java
@@ -0,0 +1,21 @@
+package de.vipra.rest.serializer;
+
+import java.io.IOException;
+
+import org.bson.types.ObjectId;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+
+import de.vipra.util.MongoUtils;
+
+public class ObjectIdDeserializer extends JsonDeserializer<ObjectId> {
+
+ @Override
+ public ObjectId deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
+ return MongoUtils.objectId(p.getValueAsString());
+ }
+
+}
diff --git a/vipra-rest/src/main/java/de/vipra/rest/serializer/ObjectIdSerializer.java b/vipra-rest/src/main/java/de/vipra/rest/serializer/ObjectIdSerializer.java
new file mode 100644
index 0000000000000000000000000000000000000000..b6653911281c5436f63fbbc4cb5af93e633905cd
--- /dev/null
+++ b/vipra-rest/src/main/java/de/vipra/rest/serializer/ObjectIdSerializer.java
@@ -0,0 +1,20 @@
+package de.vipra.rest.serializer;
+
+import java.io.IOException;
+
+import org.bson.types.ObjectId;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+
+public class ObjectIdSerializer extends JsonSerializer<ObjectId> {
+
+ @Override
+ public void serialize(ObjectId value, JsonGenerator gen, SerializerProvider serializers)
+ throws IOException, JsonProcessingException {
+ gen.writeString(value.toString());
+ }
+
+}
diff --git a/vipra-rest/src/main/java/de/vipra/rest/service/ArticleService.java b/vipra-rest/src/main/java/de/vipra/rest/service/ArticleService.java
deleted file mode 100644
index b53aaad4e5c95f216f835ca6117c22415c127f30..0000000000000000000000000000000000000000
--- a/vipra-rest/src/main/java/de/vipra/rest/service/ArticleService.java
+++ /dev/null
@@ -1,27 +0,0 @@
-package de.vipra.rest.service;
-
-import java.net.URI;
-import java.util.List;
-
-import de.vipra.util.Mongo;
-import de.vipra.util.model.Article;
-import de.vipra.util.service.DatabaseService;
-
-public class ArticleService extends DatabaseService<Article> {
-
- public ArticleService(Mongo mongo) {
- super(mongo, de.vipra.util.model.Article.class);
- }
-
- public List<Article> getMultiple(URI base, int skip, int limit, String sortBy) {
- List<Article> articles = super.getMultiple(skip, limit, sortBy);
- for (Article article : articles) {
- // delete data for listing
- article.setText(null);
- article.setStats(null);
- article.setTopics(null);
- }
- return articles;
- }
-
-}
diff --git a/vipra-rest/src/main/java/de/vipra/rest/service/TopicService.java b/vipra-rest/src/main/java/de/vipra/rest/service/TopicService.java
deleted file mode 100644
index 4ec6eb604d23f49d37282bfb6524bcca1ee46285..0000000000000000000000000000000000000000
--- a/vipra-rest/src/main/java/de/vipra/rest/service/TopicService.java
+++ /dev/null
@@ -1,24 +0,0 @@
-package de.vipra.rest.service;
-
-import java.net.URI;
-import java.util.List;
-
-import de.vipra.util.Mongo;
-import de.vipra.util.model.Topic;
-import de.vipra.util.service.DatabaseService;
-
-public class TopicService extends DatabaseService<Topic> {
-
- public TopicService(Mongo mongo) {
- super(mongo, de.vipra.util.model.Topic.class);
- }
-
- public List<Topic> getMultiple(URI base, int skip, int limit, String sortBy) {
- List<Topic> topics = super.getMultiple(skip, limit, sortBy);
- for (Topic topic : topics) {
- topic.setWords(null);
- }
- return topics;
- }
-
-}
diff --git a/vipra-ui/app/templates/articles/show.hbs b/vipra-ui/app/templates/articles/show.hbs
index 9ee5c2bc361cde649cfa35427a519bb54788a600..bcddff844c6ad60c7bd715799cc4b4ed9b1489c2 100644
--- a/vipra-ui/app/templates/articles/show.hbs
+++ b/vipra-ui/app/templates/articles/show.hbs
@@ -7,49 +7,21 @@
<dd>{{model.article.date}}</dd>
<dt>URL</dt>
<dd><a href="{{model.article.url}}">{{model.article.url}}</a></dd>
+ <dt>Word count</dt>
+ <dd>{{model.article.stats.wordCount}}</dd>
</dl>
<h3>Topics</h3>
-{{#each model.article.topics as |topic|}}
- {{#topic-link topic=topic}} ({{topic-share topic.count model.article.stats.wordCount}}%){{/topic-link}}
+{{#each model.article.topics as |topicRef|}}
+ [{{#topic-link topic=topicRef.topic}} ({{topic-share topicRef.count model.article.stats.wordCount}}%){{/topic-link}}]
{{/each}}
<h3>Statistics</h3>
<table>
<tbody>
- <tr>
- <td>Word count</td>
- <td>{{model.article.stats.wordCount}}</td>
- </tr>
- <tr>
- <td>Unique word count</td>
- <td>{{model.article.stats.uniqueWordCount}}</td>
- </tr>
- <tr>
- <td>Most frequent words</td>
- <td>
- <table>
- <thead>
- <tr>
- <th>Word</th>
- <th>Count</th>
- <th>NTF</th>
- </tr>
- </thead>
- <tbody>
- {{#each model.statsData as |stats|}}
- <tr>
- <td>{{stats.word}}</td>
- <td>{{stats.count}}</td>
- <td>{{stats.norm}}</td>
- </tr>
- {{/each}}
- </tbody>
- </table>
- </td>
- </tr>
+
</tbody>
</table>
diff --git a/vipra-ui/app/templates/topics/index.hbs b/vipra-ui/app/templates/topics/index.hbs
index f7cabdd7052be50106761da749d418fda9acf5f6..a8a4eece3ceafbe0649445c4f7f031a0e6a26227 100644
--- a/vipra-ui/app/templates/topics/index.hbs
+++ b/vipra-ui/app/templates/topics/index.hbs
@@ -1,5 +1,5 @@
<h2>Found topics</h2>
-{{debounced-input placeholder='Filter' size='50' valueBinding='filter' debounce='150'}}
+{{debounced-input placeholder='Filter' size='50' value=filter debounce='150'}}
{{topics-list items=model.topics filter=filter}}
\ No newline at end of file
diff --git a/vipra-ui/app/templates/topics/show/index.hbs b/vipra-ui/app/templates/topics/show/index.hbs
index 0a8dcf51f60d093085587ee44c815954a09f5d6f..e7f210342630bac1cc8bf90b340116afe2218b87 100644
--- a/vipra-ui/app/templates/topics/show/index.hbs
+++ b/vipra-ui/app/templates/topics/show/index.hbs
@@ -1,11 +1,3 @@
{{#link-to 'topics.show.edit'}}Edit{{/link-to}}
-<h2>{{model.topic._name}}</h2>
-
-<h3>Words</h3>
-
-{{#each model.topic.words as |word|}}
- <span class="word" title="Likeliness: {{word.likeliness}}">{{word.word}}</span>
-{{/each}}
-
-<h3>Articles</h3>
\ No newline at end of file
+<h2>{{model.topic._name}}</h2>
\ No newline at end of file
diff --git a/vipra-util/src/main/java/de/vipra/util/Constants.java b/vipra-util/src/main/java/de/vipra/util/Constants.java
index c5eea30ef8a97ba510884f3d40ba5f676078ab3e..22072d5de9984be43d0047df0209f1bd6230cce9 100644
--- a/vipra-util/src/main/java/de/vipra/util/Constants.java
+++ b/vipra-util/src/main/java/de/vipra/util/Constants.java
@@ -41,7 +41,7 @@ public class Constants {
* expression is used to strip text of characters that should not be
* processed.
*/
- public static final String CHARS_DISALLOWED = "[^a-zA-Z0-9]";
+ public static final String CHARS_DISALLOWED = "[^a-zA-Z0-9 ]";
/**
* The number of words to be used to generate a topic name. The top n words
@@ -127,7 +127,7 @@ public class Constants {
CUSTOM("custom"),
CORENLP("corenlp"),
LUCENE("lucene"),
- DEFAULT(LUCENE);
+ DEFAULT(CORENLP);
public final String name;
diff --git a/vipra-util/src/main/java/de/vipra/util/ListUtils.java b/vipra-util/src/main/java/de/vipra/util/ListUtils.java
index bddd22abffc7bc8bba3df0f3fe6ab383cb71af75..8daf6a8a59d8b8f9ecee92a336ef7696a2a30970 100644
--- a/vipra-util/src/main/java/de/vipra/util/ListUtils.java
+++ b/vipra-util/src/main/java/de/vipra/util/ListUtils.java
@@ -1,5 +1,6 @@
package de.vipra.util;
+import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
@@ -21,4 +22,11 @@ public class ListUtils {
return strings;
}
+ public static <T> List<T> toList(Iterable<T> it) {
+ List<T> list = new ArrayList<>();
+ for (T t : it)
+ list.add(t);
+ return list;
+ }
+
}
diff --git a/vipra-util/src/main/java/de/vipra/util/StringUtils.java b/vipra-util/src/main/java/de/vipra/util/StringUtils.java
index 566ade74a16f1c2e16a8389c8d8f9ee09ab385b4..adeda301024f9a75363d7a010e84b8f183e6222a 100644
--- a/vipra-util/src/main/java/de/vipra/util/StringUtils.java
+++ b/vipra-util/src/main/java/de/vipra/util/StringUtils.java
@@ -124,4 +124,14 @@ public class StringUtils {
return null;
}
+ public static String quantity(Number qty, String singular, String plural) {
+ if (qty.intValue() == 1)
+ return singular;
+ return plural;
+ }
+
+ public static String quantity(Number qty, String singular) {
+ return quantity(qty, singular, singular + "s");
+ }
+
}
diff --git a/vipra-util/src/main/java/de/vipra/util/Timer.java b/vipra-util/src/main/java/de/vipra/util/Timer.java
index 9ca70c5188281fe10cfd4ac7b05eced4b4fcdbb4..1ec369f9f0b7ba0a748e490aec8a1ef7467e52cd 100644
--- a/vipra-util/src/main/java/de/vipra/util/Timer.java
+++ b/vipra-util/src/main/java/de/vipra/util/Timer.java
@@ -1,11 +1,17 @@
package de.vipra.util;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
public class Timer {
private long start;
+ private Map<String, Long> laps;
public long start() {
start = System.nanoTime();
+ laps = new LinkedHashMap<>();
return start;
}
@@ -19,4 +25,24 @@ public class Timer {
return lap;
}
+ public long lap(String name) {
+ long lap = lap();
+ laps.put(name, lap);
+ return lap;
+ }
+
+ public String toString() {
+ String out = null;
+ if (laps != null && laps.size() > 0) {
+ StringBuilder sb = new StringBuilder();
+ for (Entry<String, Long> e : laps.entrySet()) {
+ sb.append(", ").append(e.getKey()).append(": ").append(StringUtils.timeString(e.getValue()));
+ }
+ out = sb.toString().substring(2);
+ } else {
+ out = super.toString();
+ }
+ return out;
+ }
+
}
diff --git a/vipra-util/src/main/java/de/vipra/util/an/JsonIgnore.java b/vipra-util/src/main/java/de/vipra/util/an/JsonIgnore.java
index bd89d6f11dc38327f62c74b6963ddf54e2be9249..70e5b17b880d5f318747978acc587cf8a61b64b5 100644
--- a/vipra-util/src/main/java/de/vipra/util/an/JsonIgnore.java
+++ b/vipra-util/src/main/java/de/vipra/util/an/JsonIgnore.java
@@ -6,9 +6,9 @@ import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Retention(RetentionPolicy.RUNTIME)
-@Target(ElementType.TYPE)
+@Target(ElementType.FIELD)
public @interface JsonIgnore {
- public boolean value() default false;
+ public boolean value() default true;
-}
+}
\ No newline at end of file
diff --git a/vipra-util/src/main/java/de/vipra/util/an/QueryIgnore.java b/vipra-util/src/main/java/de/vipra/util/an/QueryIgnore.java
new file mode 100644
index 0000000000000000000000000000000000000000..422439e7d892fb45e675ce6eec0115310e34e446
--- /dev/null
+++ b/vipra-util/src/main/java/de/vipra/util/an/QueryIgnore.java
@@ -0,0 +1,16 @@
+package de.vipra.util.an;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.FIELD)
+public @interface QueryIgnore {
+
+ public boolean single() default false;
+
+ public boolean multi() default false;
+
+}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/Article.java b/vipra-util/src/main/java/de/vipra/util/model/Article.java
index 17531569ba14608ae5d199653a8b0ec11fa3c418..72da35029f52ed7f53ba6db80a6059714751dba8 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/Article.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/Article.java
@@ -18,43 +18,43 @@ import de.vipra.util.Constants;
import de.vipra.util.FileUtils;
import de.vipra.util.MongoUtils;
import de.vipra.util.StringUtils;
-import de.vipra.util.an.JsonField;
+import de.vipra.util.an.JsonWrap;
+import de.vipra.util.an.QueryIgnore;
+@SuppressWarnings("serial")
@Entity(value = "articles", noClassnameStored = true)
public class Article extends Model implements Serializable {
- private static final long serialVersionUID = -3357348905924854240L;
-
@Id
private ObjectId id;
- @JsonField("attributes.title")
+ @JsonWrap("attributes")
private String title;
- @JsonField("attributes.text")
+ @JsonWrap("attributes")
+ @QueryIgnore(multi = true)
private String text;
- @JsonField("attributes.url")
+ @JsonWrap("attributes")
private String url;
- @JsonField("attributes.date")
+ @JsonWrap("attributes")
private Date date;
- @JsonField("attributes.complete")
- private boolean complete;
-
@Embedded
- @JsonField("attributes.stats")
- private ArticleStats stats;
+ @JsonWrap("attributes")
+ @QueryIgnore(multi = true)
+ private List<TopicRef> topics;
@Embedded
- @JsonField("attributes.topics")
- private List<TopicRef> topics;
+ @JsonWrap("attributes")
+ @QueryIgnore(multi = true)
+ private ArticleStats stats;
- @JsonField("attributes.created")
+ @JsonWrap("attributes")
private Date created = new Date();
- @JsonField("attributes.modified")
+ @JsonWrap("attributes")
private Date modified;
public ObjectId getId() {
@@ -101,22 +101,6 @@ public class Article extends Model implements Serializable {
this.date = date;
}
- public boolean isComplete() {
- return complete;
- }
-
- public void setComplete(boolean complete) {
- this.complete = complete;
- }
-
- public ArticleStats getStats() {
- return stats;
- }
-
- public void setStats(ArticleStats stats) {
- this.stats = stats;
- }
-
public void setDate(String date) {
SimpleDateFormat df = new SimpleDateFormat(Constants.DATETIME_FORMAT);
try {
@@ -132,6 +116,14 @@ public class Article extends Model implements Serializable {
this.topics = topics;
}
+ public ArticleStats getStats() {
+ return stats;
+ }
+
+ public void setStats(ArticleStats stats) {
+ this.stats = stats;
+ }
+
public Date getCreated() {
return created;
}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/ArticleStats.java b/vipra-util/src/main/java/de/vipra/util/model/ArticleStats.java
index 579d5621a7498824adf6ea19ebf9acbc8c749e8a..06a3b1e9f9069fba467b892a8d9809d70cf83401 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/ArticleStats.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/ArticleStats.java
@@ -1,11 +1,8 @@
package de.vipra.util.model;
import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
import org.bson.types.ObjectId;
-import org.mongodb.morphia.annotations.Embedded;
import org.mongodb.morphia.annotations.Entity;
import org.mongodb.morphia.annotations.Id;
@@ -13,13 +10,10 @@ import org.mongodb.morphia.annotations.Id;
public class ArticleStats implements Serializable {
private static final long serialVersionUID = -4712841724990200627L;
-
+
@Id
private ObjectId id;
private long wordCount;
- private long uniqueWordCount;
- @Embedded
- private Map<String, TermFrequency> uniqueWords;
public ObjectId getId() {
return id;
@@ -37,57 +31,16 @@ public class ArticleStats implements Serializable {
this.wordCount = wordCount;
}
- public long getUniqueWordCount() {
- return uniqueWordCount;
- }
-
- public void setUniqueWordCount(long uniqueWordCount) {
- this.uniqueWordCount = uniqueWordCount;
- }
-
- public Map<String, TermFrequency> getUniqueWords() {
- return uniqueWords;
- }
-
- public void setUniqueWords(Map<String, TermFrequency> uniqueWords) {
- this.uniqueWords = uniqueWords;
- }
-
- public static ArticleStats generateFromText(final String text) {
+ public static ArticleStats generateFromText(final String text, final WordMap wordMap) {
ArticleStats stats = new ArticleStats();
String[] words = text.split("\\s+");
stats.setWordCount(words.length);
- Map<String, TermFrequency> uniqueWords = new HashMap<>();
- long maxFrequency = 0;
-
- // loop and count unique words
- // also remember maximum frequency
- for (String word : words) {
- TermFrequency tf = uniqueWords.get(word);
- if (tf == null) {
- tf = new TermFrequency();
- }
- tf.incrementTermFrequency();
- if (tf.getTermFrequency() > maxFrequency) {
- maxFrequency = tf.getTermFrequency();
- }
- uniqueWords.put(word, tf);
- }
-
- // normalize frequencies
- for (Map.Entry<String, TermFrequency> entry : uniqueWords.entrySet()) {
- entry.getValue().normalizeTermFrequency(maxFrequency);
- }
-
- stats.setUniqueWordCount(uniqueWords.size());
- stats.setUniqueWords(uniqueWords);
return stats;
}
@Override
public String toString() {
- return ArticleStats.class.getSimpleName() + "[id:" + id + ", wordCount:" + wordCount + ", uniqueWordCount:"
- + uniqueWordCount + "]";
+ return ArticleStats.class.getSimpleName() + "[id:" + id + ", wordCount:" + wordCount + "]";
}
-}
+}
\ No newline at end of file
diff --git a/vipra-util/src/main/java/de/vipra/util/model/Model.java b/vipra-util/src/main/java/de/vipra/util/model/Model.java
index 70baf2257b93c900ef71832336926302149075bd..8fa5aab18b609ce00af81ade8e02059ae472610e 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/Model.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/Model.java
@@ -11,10 +11,9 @@ import org.bson.types.ObjectId;
import de.vipra.util.Constants;
+@SuppressWarnings("serial")
public abstract class Model implements Serializable {
- private static final long serialVersionUID = -1991594352707918633L;
-
public URI uri(URI base) {
try {
return new URI(base.toString() + "/" + getId().toString());
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TermFrequency.java b/vipra-util/src/main/java/de/vipra/util/model/TermFrequency.java
deleted file mode 100644
index 75099a27a65c8fe9e7bbb79fd8302e8809359c0f..0000000000000000000000000000000000000000
--- a/vipra-util/src/main/java/de/vipra/util/model/TermFrequency.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package de.vipra.util.model;
-
-import java.io.Serializable;
-
-import org.mongodb.morphia.annotations.Embedded;
-
-@Embedded
-public class TermFrequency implements Serializable {
-
- private static final long serialVersionUID = 4042573510472738071L;
-
- private long termFrequency = 0;
- private double normalizedTermFrequency = 0;
-
- public long getTermFrequency() {
- return termFrequency;
- }
-
- public void setTermFrequency(long termFrequency) {
- this.termFrequency = termFrequency;
- }
-
- public double getNormalizedTermFrequency() {
- return normalizedTermFrequency;
- }
-
- public void setNormalizedTermFrequency(double normalizedTermFrequency) {
- this.normalizedTermFrequency = normalizedTermFrequency;
- }
-
- public void normalizeTermFrequency(double max) {
- setNormalizedTermFrequency(getTermFrequency() / max);
- }
-
- public void incrementTermFrequency() {
- setTermFrequency(getTermFrequency() + 1);
- }
-
- @Override
- public String toString() {
- return TermFrequency.class.getSimpleName() + "[termFrequency:" + termFrequency + ", normalizedTermFrequency:"
- + normalizedTermFrequency + "]";
- }
-
-}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/Topic.java b/vipra-util/src/main/java/de/vipra/util/model/Topic.java
index 1d9488cd2e2235e8979d82a6fe304e1c1da5aa5b..1e1d7a47af4967da703cde59b19124835c851a6d 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/Topic.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/Topic.java
@@ -3,52 +3,22 @@ package de.vipra.util.model;
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
import org.bson.types.ObjectId;
-import org.mongodb.morphia.annotations.Embedded;
import org.mongodb.morphia.annotations.Entity;
import org.mongodb.morphia.annotations.Id;
-import org.mongodb.morphia.annotations.PrePersist;
-import de.vipra.util.Constants;
import de.vipra.util.MongoUtils;
-import de.vipra.util.StringUtils;
-import de.vipra.util.an.JsonWrap;
import de.vipra.util.ex.NotImplementedException;
+@SuppressWarnings("serial")
@Entity(value = "topics", noClassnameStored = true)
public class Topic extends Model implements Serializable {
- private static final long serialVersionUID = 7121629487498450992L;
-
@Id
private ObjectId id;
-
- @JsonWrap("attributes")
- private int index;
-
- @JsonWrap("attributes")
private String name;
- @Embedded
- @JsonWrap("attributes")
- private List<TopicWord> words;
-
- @JsonWrap("attributes")
- private Date created = new Date();
-
- @JsonWrap("attributes")
- private Date modified;
-
- public Topic() {}
-
- public Topic(List<TopicWord> words) {
- this.words = words;
- }
-
public ObjectId getId() {
return id;
}
@@ -61,14 +31,6 @@ public class Topic extends Model implements Serializable {
this.id = MongoUtils.objectId(id);
}
- public int getIndex() {
- return index;
- }
-
- public void setIndex(int index) {
- this.index = index;
- }
-
public String getName() {
return name;
}
@@ -77,43 +39,6 @@ public class Topic extends Model implements Serializable {
this.name = name;
}
- public List<TopicWord> getWords() {
- return words;
- }
-
- public void setWords(List<TopicWord> words) {
- this.words = words;
- }
-
- public Date getCreated() {
- return created;
- }
-
- public void setCreated(Date created) {
- this.created = created;
- }
-
- public Date getModified() {
- return modified;
- }
-
- public void setModified(Date modified) {
- this.modified = modified;
- }
-
- public String getNameFromWords() {
- String name = null;
- if (words != null && words.size() > 0) {
- int size = Math.min(Constants.AUTO_TOPIC_WORDS, words.size());
- List<String> topWords = new ArrayList<>(size);
- for (int i = 0; i < size; i++) {
- topWords.add(words.get(i).getWord());
- }
- name = StringUtils.join(topWords);
- }
- return name;
- }
-
@Override
public void fromFile(File file) throws IOException {
throw new NotImplementedException();
@@ -124,15 +49,4 @@ public class Topic extends Model implements Serializable {
throw new NotImplementedException();
}
- @Override
- public String toString() {
- return Topic.class.getSimpleName() + "[id:" + id + ", name:" + name + ", created:" + created + ", modified:"
- + modified + "]";
- }
-
- @PrePersist
- public void prePersist() {
- this.modified = new Date();
- }
-
}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java b/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java
new file mode 100644
index 0000000000000000000000000000000000000000..5af0a958ee8598d8f87bed4b5cbb6a3213d064ac
--- /dev/null
+++ b/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java
@@ -0,0 +1,135 @@
+package de.vipra.util.model;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import org.bson.types.ObjectId;
+import org.mongodb.morphia.annotations.Embedded;
+import org.mongodb.morphia.annotations.Entity;
+import org.mongodb.morphia.annotations.Id;
+import org.mongodb.morphia.annotations.PrePersist;
+
+import de.vipra.util.Constants;
+import de.vipra.util.MongoUtils;
+import de.vipra.util.StringUtils;
+import de.vipra.util.an.JsonType;
+import de.vipra.util.an.JsonWrap;
+import de.vipra.util.an.QueryIgnore;
+import de.vipra.util.ex.NotImplementedException;
+
+@SuppressWarnings("serial")
+@JsonType("topic")
+@Entity(value = "topics", noClassnameStored = true)
+public class TopicFull extends Model implements Serializable {
+
+ @Id
+ private ObjectId id;
+
+ @JsonWrap("attributes")
+ private String name;
+
+ @JsonWrap("attributes")
+ private int index;
+
+ @Embedded
+ @JsonWrap("attributes")
+ @QueryIgnore(multi = true)
+ private List<TopicWord> topicWords;
+
+ @JsonWrap("attributes")
+ private Date created = new Date();
+
+ @JsonWrap("attributes")
+ private Date modified;
+
+ public ObjectId getId() {
+ return id;
+ }
+
+ public void setId(ObjectId id) {
+ this.id = id;
+ }
+
+ public void setId(String id) {
+ this.id = MongoUtils.objectId(id);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public int getIndex() {
+ return index;
+ }
+
+ public void setIndex(int index) {
+ this.index = index;
+ }
+
+ public List<TopicWord> getTopicWords() {
+ return topicWords;
+ }
+
+ public void setTopicWords(List<TopicWord> topicWords) {
+ this.topicWords = topicWords;
+ }
+
+ public Date getCreated() {
+ return created;
+ }
+
+ public void setCreated(Date created) {
+ this.created = created;
+ }
+
+ public Date getModified() {
+ return modified;
+ }
+
+ public void setModified(Date modified) {
+ this.modified = modified;
+ }
+
+ @Override
+ public void fromFile(File file) throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String toFileString() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String toString() {
+ return TopicFull.class.getSimpleName() + "[id:" + getId() + ", name:" + getName() + ", created:" + created
+ + ", modified:" + modified + "]";
+ }
+
+ @PrePersist
+ public void prePersist() {
+ this.modified = new Date();
+ }
+
+ public static String getNameFromWords(List<TopicWord> words) {
+ String name = null;
+ if (words != null && words.size() > 0) {
+ int size = Math.min(Constants.AUTO_TOPIC_WORDS, words.size());
+ List<String> topWords = new ArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ topWords.add(words.get(i).getWord().getWord());
+ }
+ name = StringUtils.join(topWords);
+ }
+ return name;
+ }
+
+}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TopicRef.java b/vipra-util/src/main/java/de/vipra/util/model/TopicRef.java
index e6b6df22bf13665dbc7b82990bbf34b58002fda1..0380a0dac47eb9e0375ceb7ec47cf70a9454d9f4 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/TopicRef.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/TopicRef.java
@@ -4,14 +4,18 @@ import java.io.Serializable;
import org.mongodb.morphia.annotations.Embedded;
import org.mongodb.morphia.annotations.Reference;
+import org.mongodb.morphia.annotations.Transient;
+import de.vipra.util.an.JsonIgnore;
+
+@SuppressWarnings("serial")
@Embedded
public class TopicRef implements Comparable<TopicRef>, Serializable {
- private static final long serialVersionUID = 3301635858822787398L;
-
+ @Transient
+ @JsonIgnore
private String topicId;
- @Reference(lazy = true)
+ @Reference
private Topic topic;
private int count;
@@ -33,6 +37,14 @@ public class TopicRef implements Comparable<TopicRef>, Serializable {
this.count = count;
}
+ public Topic getTopic() {
+ return topic;
+ }
+
+ public void setTopic(Topic topic) {
+ this.topic = topic;
+ }
+
@Override
public int compareTo(TopicRef arg0) {
return count - arg0.getCount();
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TopicWord.java b/vipra-util/src/main/java/de/vipra/util/model/TopicWord.java
index 8a1f1f20c20d1f6f950e578b460102d27aaf2a72..5bf9a5cbcea3b5af21eee0060a7c34cdfcac08c8 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/TopicWord.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/TopicWord.java
@@ -3,27 +3,28 @@ package de.vipra.util.model;
import java.io.Serializable;
import org.mongodb.morphia.annotations.Embedded;
+import org.mongodb.morphia.annotations.Reference;
+@SuppressWarnings("serial")
@Embedded
public class TopicWord implements Comparable<TopicWord>, Serializable {
- private static final long serialVersionUID = -5409441821591159243L;
-
- private String word;
+ @Reference
+ private Word word;
private double likeliness;
public TopicWord() {}
- public TopicWord(String word, double likeliness) {
+ public TopicWord(Word word, double likeliness) {
this.word = word;
this.likeliness = likeliness;
}
- public String getWord() {
+ public Word getWord() {
return word;
}
- public void setWord(String word) {
+ public void setWord(Word word) {
this.word = word;
}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/Word.java b/vipra-util/src/main/java/de/vipra/util/model/Word.java
new file mode 100644
index 0000000000000000000000000000000000000000..2effbd0d5bae56c6ff4ba584bb195b52beed9526
--- /dev/null
+++ b/vipra-util/src/main/java/de/vipra/util/model/Word.java
@@ -0,0 +1,59 @@
+package de.vipra.util.model;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.bson.types.ObjectId;
+import org.mongodb.morphia.annotations.Entity;
+import org.mongodb.morphia.annotations.Id;
+
+import de.vipra.util.MongoUtils;
+import de.vipra.util.ex.NotImplementedException;
+
+@SuppressWarnings("serial")
+@Entity(value = "words", noClassnameStored = true)
+public class Word extends Model implements Serializable {
+
+ @Id
+ private ObjectId id;
+ private String word;
+
+ public Word() {}
+
+ public Word(String word) {
+ this.word = word;
+ }
+
+ public ObjectId getId() {
+ return id;
+ }
+
+ public void setId(ObjectId id) {
+ this.id = id;
+ }
+
+ public String getWord() {
+ return word;
+ }
+
+ public void setWord(String word) {
+ this.word = word;
+ }
+
+ @Override
+ public void setId(String id) {
+ this.id = MongoUtils.objectId(id);
+ }
+
+ @Override
+ public void fromFile(File file) throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String toFileString() {
+ throw new NotImplementedException();
+ }
+
+}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/WordMap.java b/vipra-util/src/main/java/de/vipra/util/model/WordMap.java
new file mode 100644
index 0000000000000000000000000000000000000000..76351d34aa4fa65a6f03142cb87221a86f6f9ba3
--- /dev/null
+++ b/vipra-util/src/main/java/de/vipra/util/model/WordMap.java
@@ -0,0 +1,99 @@
+package de.vipra.util.model;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import de.vipra.util.ex.DatabaseException;
+import de.vipra.util.service.DatabaseService;
+
+public class WordMap extends HashMap<String, Word> {
+
+ private static final long serialVersionUID = 8321873837437524923L;
+ public static final Logger log = LoggerFactory.getLogger(WordMap.class);
+
+ private final DatabaseService<Word> dbWords;
+ private boolean createNow = true;
+ private long newWords = 0;
+
+ public WordMap(DatabaseService<Word> dbWords) {
+ this.dbWords = dbWords;
+ List<Word> words = dbWords.getAll();
+ for (Word word : words)
+ put(word.getWord().toLowerCase(), word);
+ }
+
+ @Override
+ public Word get(Object w) {
+ String strWord = w.toString();
+ Word word = super.get(strWord.toLowerCase());
+ if (word == null) {
+ word = new Word(strWord);
+ createWord(word);
+ }
+ return word;
+ }
+
+ @Override
+ public Word put(String strWord, Word word) {
+ Word currentWord = get(strWord);
+ if (currentWord == null) {
+ if (word == null)
+ word = new Word(strWord);
+ createWord(word);
+ put(strWord, word);
+ currentWord = word;
+ } else {
+ currentWord.setWord(word.getWord());
+ try {
+ dbWords.updateSingle(currentWord);
+ } catch (DatabaseException e) {
+ log.error("could not update word in database", e);
+ throw new RuntimeException(e);
+ }
+ }
+ return currentWord;
+ }
+
+ private Word createWord(Word word) {
+ if (createNow) {
+ try {
+ dbWords.createSingle(word);
+ newWords++;
+ } catch (DatabaseException e) {
+ log.error("could not create word in database", e);
+ throw new RuntimeException(e);
+ }
+ }
+ return word;
+ }
+
+ public Word put(String strWord) {
+ return put(strWord, null);
+ }
+
+ public void create() throws DatabaseException {
+ List<Word> newWords = new ArrayList<>();
+ for (Entry<String, Word> e : this.entrySet())
+ if (e.getValue().getId() == null)
+ newWords.add(e.getValue());
+ dbWords.createMultiple(newWords);
+ this.newWords += newWords.size();
+ }
+
+ public boolean isCreateNow() {
+ return createNow;
+ }
+
+ public void setCreateNow(boolean createNow) {
+ this.createNow = createNow;
+ }
+
+ public long getNewWords() {
+ return newWords;
+ }
+
+}
diff --git a/vipra-util/src/main/java/de/vipra/util/service/DatabaseService.java b/vipra-util/src/main/java/de/vipra/util/service/DatabaseService.java
index 6172a5e2142ceab036ea990dfed87e82e66e207c..ed3f222ac266c81268e5b544225c0bef91b7ac70 100644
--- a/vipra-util/src/main/java/de/vipra/util/service/DatabaseService.java
+++ b/vipra-util/src/main/java/de/vipra/util/service/DatabaseService.java
@@ -1,13 +1,18 @@
package de.vipra.util.service;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
import java.util.List;
+import org.bson.types.ObjectId;
import org.mongodb.morphia.Datastore;
import org.mongodb.morphia.query.Query;
import de.vipra.util.Config;
+import de.vipra.util.ListUtils;
import de.vipra.util.Mongo;
import de.vipra.util.MongoUtils;
+import de.vipra.util.an.QueryIgnore;
import de.vipra.util.ex.ConfigException;
import de.vipra.util.ex.DatabaseException;
import de.vipra.util.model.Model;
@@ -16,31 +21,70 @@ public class DatabaseService<T extends Model> implements Service<T, DatabaseExce
private final Datastore datastore;
private final Class<T> clazz;
+ private final String[] ignoredFieldsSingle;
+ private final String[] ignoredFieldsMulti;
public DatabaseService(Mongo mongo, Class<T> clazz) {
this.datastore = mongo.getDatastore();
this.clazz = clazz;
+
+ List<String> ignoreSingle = new ArrayList<>();
+ List<String> ignoreMulti = new ArrayList<>();
+ Field[] fields = clazz.getDeclaredFields();
+ for (Field field : fields) {
+ QueryIgnore qi = field.getDeclaredAnnotation(QueryIgnore.class);
+ if (qi != null) {
+ if (qi.single())
+ ignoreSingle.add(field.getName());
+ if (qi.multi())
+ ignoreMulti.add(field.getName());
+ }
+ }
+ this.ignoredFieldsSingle = ignoreSingle.toArray(new String[ignoreSingle.size()]);
+ this.ignoredFieldsMulti = ignoreMulti.toArray(new String[ignoreMulti.size()]);
}
@Override
public T getSingle(String id) {
- return datastore.get(clazz, MongoUtils.objectId(id));
+ Query<T> q = datastore.createQuery(clazz).field("_id").equal(new ObjectId(id));
+ if (ignoredFieldsSingle.length > 0)
+ q.retrievedFields(false, ignoredFieldsSingle);
+ return q.get();
}
- public List<T> getMultiple(int skip, int limit, String sortBy) {
- Query<T> q = datastore.createQuery(clazz).offset(skip).limit(limit);
+ @Override
+ public List<T> getMultiple(Integer skip, Integer limit, String sortBy) {
+ Query<T> q = datastore.createQuery(clazz);
+ if (skip != null)
+ q.offset(skip);
+ if (limit != null)
+ q.limit(limit);
if (sortBy != null)
- q = q.order(sortBy);
+ q.order(sortBy);
+ if (ignoredFieldsMulti.length > 0)
+ q.retrievedFields(false, ignoredFieldsMulti);
List<T> list = q.asList();
return list;
}
+ @Override
+ public List<T> getAll() {
+ return getMultiple(null, null, null);
+ }
+
@Override
public T createSingle(T t) throws DatabaseException {
datastore.save(t);
return t;
}
+ @Override
+ public List<T> createMultiple(Iterable<T> t) throws DatabaseException {
+ List<T> list = ListUtils.toList(t);
+ datastore.save(list);
+ return list;
+ }
+
@Override
public long deleteSingle(String id) throws DatabaseException {
return datastore.delete(MongoUtils.objectId(id)).getN();
@@ -51,10 +95,12 @@ public class DatabaseService<T extends Model> implements Service<T, DatabaseExce
datastore.save(t);
}
+ @Override
public void drop() {
datastore.getCollection(clazz).drop();
}
+ @Override
public long count() {
return datastore.getCount(clazz);
}
diff --git a/vipra-util/src/main/java/de/vipra/util/service/Service.java b/vipra-util/src/main/java/de/vipra/util/service/Service.java
index f3a4129c8edbdf46a58e888b79831efdaa2e0da2..659a57956c81c6c03b1f76269cfe35980beb1110 100644
--- a/vipra-util/src/main/java/de/vipra/util/service/Service.java
+++ b/vipra-util/src/main/java/de/vipra/util/service/Service.java
@@ -1,17 +1,27 @@
package de.vipra.util.service;
+import java.util.List;
+
import de.vipra.util.model.Model;
public interface Service<T extends Model, E extends Exception> {
T getSingle(String id) throws E;
+ List<T> getMultiple(Integer skip, Integer limit, String sortBy) throws E;
+
+ List<T> getAll() throws E;
+
T createSingle(T t) throws E;
+ List<T> createMultiple(Iterable<T> t) throws E;
+
long deleteSingle(String id) throws E;
void updateSingle(T t) throws E;
- void drop();
+ void drop() throws E;
+
+ long count() throws E;
}