From 184e26a8a0fb564e35bca68401327f417eb55d1d Mon Sep 17 00:00:00 2001
From: Eike Cochu <eike@cochu.com>
Date: Sun, 10 Apr 2016 18:28:17 +0200
Subject: [PATCH] updated querybuilder, added config print command, updated
network view
---
.../vipra/rest/resource/ArticleResource.java | 8 +-
.../rest/resource/BugReportResource.java | 2 +-
.../vipra/rest/resource/SequenceResource.java | 4 +-
.../rest/resource/TextEntityResource.java | 4 +-
.../rest/resource/TopicModelResource.java | 2 +-
.../de/vipra/rest/resource/TopicResource.java | 10 +-
.../vipra/rest/resource/WindowResource.java | 4 +-
.../de/vipra/rest/resource/WordResource.java | 4 +-
vipra-cmd/runcfg/CMD.launch | 2 +-
.../java/de/vipra/cmd/CommandLineOptions.java | 14 +-
.../src/main/java/de/vipra/cmd/Main.java | 10 +-
.../main/java/de/vipra/cmd/lda/Analyzer.java | 22 +-
.../vipra/cmd/option/DeleteModelCommand.java | 4 +-
.../de/vipra/cmd/option/ImportCommand.java | 49 +--
.../vipra/cmd/option/PrintModelCommand.java | 29 ++
.../de/vipra/cmd/text/SpotlightResponse.java | 2 +
vipra-ui/app/html/articles/show.html | 4 +-
.../app/html/directives/window-dropdown.html | 5 +
vipra-ui/app/html/network.html | 8 +-
vipra-ui/app/js/controllers.js | 97 ++++--
vipra-ui/app/js/directives.js | 27 +-
vipra-ui/app/js/helpers.js | 7 +-
vipra-ui/app/less/app.less | 8 +
.../main/java/de/vipra/util/StringUtils.java | 18 +-
.../src/main/java/de/vipra/util/Tuple.java | 35 --
.../de/vipra/util/model/TopicModelConfig.java | 10 +
.../de/vipra/util/service/MongoService.java | 59 +---
.../de/vipra/util/service/QueryBuilder.java | 217 ++++++++++++
.../java/de/vipra/util/service/Service.java | 319 ------------------
29 files changed, 480 insertions(+), 504 deletions(-)
create mode 100644 vipra-cmd/src/main/java/de/vipra/cmd/option/PrintModelCommand.java
create mode 100644 vipra-ui/app/html/directives/window-dropdown.html
delete mode 100644 vipra-util/src/main/java/de/vipra/util/Tuple.java
create mode 100644 vipra-util/src/main/java/de/vipra/util/service/QueryBuilder.java
delete mode 100644 vipra-util/src/main/java/de/vipra/util/service/Service.java
diff --git a/vipra-backend/src/main/java/de/vipra/rest/resource/ArticleResource.java b/vipra-backend/src/main/java/de/vipra/rest/resource/ArticleResource.java
index 162eb7f9..2fae2a24 100644
--- a/vipra-backend/src/main/java/de/vipra/rest/resource/ArticleResource.java
+++ b/vipra-backend/src/main/java/de/vipra/rest/resource/ArticleResource.java
@@ -29,7 +29,7 @@ import de.vipra.util.ex.ConfigException;
import de.vipra.util.model.ArticleFull;
import de.vipra.util.model.TopicModel;
import de.vipra.util.service.MongoService;
-import de.vipra.util.service.Service.QueryBuilder;
+import de.vipra.util.service.QueryBuilder;
@Path("articles")
public class ArticleResource {
@@ -70,13 +70,13 @@ public class ArticleResource {
query.fields(true, StringUtils.getFields(fields));
if (topicModel != null && !topicModel.isEmpty())
- query.criteria("topicModel", new TopicModel(topicModel));
+ query.eq("topicModel", new TopicModel(topicModel));
if (word != null && !word.isEmpty())
- query.criteria("words.id", word);
+ query.eq("words.id", word);
if (entity != null && !entity.isEmpty())
- query.criteria("entities.entity.id", entity);
+ query.eq("entities.entity.id", entity);
final List<ArticleFull> articles = dbArticles.getMultiple(query);
diff --git a/vipra-backend/src/main/java/de/vipra/rest/resource/BugReportResource.java b/vipra-backend/src/main/java/de/vipra/rest/resource/BugReportResource.java
index bd5e0bd1..3f73822f 100644
--- a/vipra-backend/src/main/java/de/vipra/rest/resource/BugReportResource.java
+++ b/vipra-backend/src/main/java/de/vipra/rest/resource/BugReportResource.java
@@ -31,7 +31,7 @@ import de.vipra.util.ex.ConfigException;
import de.vipra.util.ex.DatabaseException;
import de.vipra.util.model.BugReport;
import de.vipra.util.service.MongoService;
-import de.vipra.util.service.Service.QueryBuilder;
+import de.vipra.util.service.QueryBuilder;
@Path("bugreports")
public class BugReportResource {
diff --git a/vipra-backend/src/main/java/de/vipra/rest/resource/SequenceResource.java b/vipra-backend/src/main/java/de/vipra/rest/resource/SequenceResource.java
index aafcb515..1f491b92 100644
--- a/vipra-backend/src/main/java/de/vipra/rest/resource/SequenceResource.java
+++ b/vipra-backend/src/main/java/de/vipra/rest/resource/SequenceResource.java
@@ -26,7 +26,7 @@ import de.vipra.util.ex.ConfigException;
import de.vipra.util.model.SequenceFull;
import de.vipra.util.model.TopicModel;
import de.vipra.util.service.MongoService;
-import de.vipra.util.service.Service.QueryBuilder;
+import de.vipra.util.service.QueryBuilder;
@Path("sequences")
public class SequenceResource {
@@ -54,7 +54,7 @@ public class SequenceResource {
query.fields(true, StringUtils.getFields(fields));
if (topicModel != null && !topicModel.isEmpty())
- query.criteria("topicModel", new TopicModel(topicModel));
+ query.eq("topicModel", new TopicModel(topicModel));
final List<SequenceFull> sequences = dbSequences.getMultiple(query);
diff --git a/vipra-backend/src/main/java/de/vipra/rest/resource/TextEntityResource.java b/vipra-backend/src/main/java/de/vipra/rest/resource/TextEntityResource.java
index 25c59ed4..cbb74e61 100644
--- a/vipra-backend/src/main/java/de/vipra/rest/resource/TextEntityResource.java
+++ b/vipra-backend/src/main/java/de/vipra/rest/resource/TextEntityResource.java
@@ -25,7 +25,7 @@ import de.vipra.util.ex.ConfigException;
import de.vipra.util.model.TextEntityFull;
import de.vipra.util.model.TopicModel;
import de.vipra.util.service.MongoService;
-import de.vipra.util.service.Service.QueryBuilder;
+import de.vipra.util.service.QueryBuilder;
@Path("entities")
public class TextEntityResource {
@@ -56,7 +56,7 @@ public class TextEntityResource {
query.fields(true, StringUtils.getFields(fields));
if (topicModel != null && !topicModel.isEmpty())
- query.criteria("topicModel", new TopicModel(topicModel));
+ query.eq("topicModel", new TopicModel(topicModel));
final List<TextEntityFull> entities = dbEntities.getMultiple(query);
diff --git a/vipra-backend/src/main/java/de/vipra/rest/resource/TopicModelResource.java b/vipra-backend/src/main/java/de/vipra/rest/resource/TopicModelResource.java
index 50b35a6f..c28eafeb 100644
--- a/vipra-backend/src/main/java/de/vipra/rest/resource/TopicModelResource.java
+++ b/vipra-backend/src/main/java/de/vipra/rest/resource/TopicModelResource.java
@@ -22,7 +22,7 @@ import de.vipra.util.StringUtils;
import de.vipra.util.ex.ConfigException;
import de.vipra.util.model.TopicModelFull;
import de.vipra.util.service.MongoService;
-import de.vipra.util.service.Service.QueryBuilder;
+import de.vipra.util.service.QueryBuilder;
@Path("topicmodels")
public class TopicModelResource {
diff --git a/vipra-backend/src/main/java/de/vipra/rest/resource/TopicResource.java b/vipra-backend/src/main/java/de/vipra/rest/resource/TopicResource.java
index 70849ce7..794a8631 100644
--- a/vipra-backend/src/main/java/de/vipra/rest/resource/TopicResource.java
+++ b/vipra-backend/src/main/java/de/vipra/rest/resource/TopicResource.java
@@ -32,7 +32,7 @@ import de.vipra.util.model.Topic;
import de.vipra.util.model.TopicFull;
import de.vipra.util.model.TopicModel;
import de.vipra.util.service.MongoService;
-import de.vipra.util.service.Service.QueryBuilder;
+import de.vipra.util.service.QueryBuilder;
@Path("topics")
public class TopicResource {
@@ -62,10 +62,10 @@ public class TopicResource {
query.fields(true, StringUtils.getFields(fields));
if (topicModel != null && !topicModel.isEmpty())
- query.criteria("topicModel", new TopicModel(topicModel));
+ query.eq("topicModel", new TopicModel(topicModel));
if (word != null && !word.isEmpty())
- query.criteria("words.id", word);
+ query.eq("words.id", word);
final List<TopicFull> topics = dbTopics.getMultiple(query);
@@ -118,14 +118,14 @@ public class TopicResource {
final ResponseWrapper<List<ArticleFull>> res = new ResponseWrapper<>();
try {
final Topic topic = new Topic(MongoUtils.objectId(id));
- final QueryBuilder query = QueryBuilder.builder().criteria("topics.topic", topic).skip(skip).limit(limit).sortBy(sortBy);
+ final QueryBuilder query = QueryBuilder.builder().eq("topics.topic", topic).skip(skip).limit(limit).sortBy(sortBy);
if (fields != null && !fields.isEmpty())
query.fields(true, StringUtils.getFields(fields));
final List<ArticleFull> articles = dbArticles.getMultiple(query);
if ((skip != null && skip > 0) || (limit != null && limit > 0))
- res.addHeader("total", dbArticles.count(QueryBuilder.builder().criteria("topics.topic", topic)));
+ res.addHeader("total", dbArticles.count(QueryBuilder.builder().eq("topics.topic", topic)));
else
res.addHeader("total", articles.size());
diff --git a/vipra-backend/src/main/java/de/vipra/rest/resource/WindowResource.java b/vipra-backend/src/main/java/de/vipra/rest/resource/WindowResource.java
index 7d1555c7..3577e152 100644
--- a/vipra-backend/src/main/java/de/vipra/rest/resource/WindowResource.java
+++ b/vipra-backend/src/main/java/de/vipra/rest/resource/WindowResource.java
@@ -21,7 +21,7 @@ import de.vipra.util.ex.ConfigException;
import de.vipra.util.model.TopicModel;
import de.vipra.util.model.WindowFull;
import de.vipra.util.service.MongoService;
-import de.vipra.util.service.Service.QueryBuilder;
+import de.vipra.util.service.QueryBuilder;
@Path("windows")
public class WindowResource {
@@ -49,7 +49,7 @@ public class WindowResource {
query.fields(true, StringUtils.getFields(fields));
if (topicModel != null && !topicModel.isEmpty())
- query.criteria("topicModel", new TopicModel(topicModel));
+ query.eq("topicModel", new TopicModel(topicModel));
final List<WindowFull> windows = dbWindows.getMultiple(query);
diff --git a/vipra-backend/src/main/java/de/vipra/rest/resource/WordResource.java b/vipra-backend/src/main/java/de/vipra/rest/resource/WordResource.java
index a4247aac..572e2a8e 100644
--- a/vipra-backend/src/main/java/de/vipra/rest/resource/WordResource.java
+++ b/vipra-backend/src/main/java/de/vipra/rest/resource/WordResource.java
@@ -24,7 +24,7 @@ import de.vipra.util.ex.ConfigException;
import de.vipra.util.model.TopicModel;
import de.vipra.util.model.WordFull;
import de.vipra.util.service.MongoService;
-import de.vipra.util.service.Service.QueryBuilder;
+import de.vipra.util.service.QueryBuilder;
@Path("words")
public class WordResource {
@@ -52,7 +52,7 @@ public class WordResource {
query.fields(true, StringUtils.getFields(fields));
if (topicModel != null && !topicModel.isEmpty())
- query.criteria("topicModel", new TopicModel(topicModel));
+ query.eq("topicModel", new TopicModel(topicModel));
final List<WordFull> words = dbWords.getMultiple(query);
diff --git a/vipra-cmd/runcfg/CMD.launch b/vipra-cmd/runcfg/CMD.launch
index b86e653f..a9b34fe3 100644
--- a/vipra-cmd/runcfg/CMD.launch
+++ b/vipra-cmd/runcfg/CMD.launch
@@ -11,7 +11,7 @@
</listAttribute>
<stringAttribute key="org.eclipse.jdt.launching.CLASSPATH_PROVIDER" value="org.eclipse.m2e.launchconfig.classpathProvider"/>
<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="de.vipra.cmd.Main"/>
-<stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="-C yearly"/>
+<stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="-dD reuters -C reuters -S reuters -I /home/eike/repos/master/ma-impl/vm/data/reuters.json"/>
<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="vipra-cmd"/>
<stringAttribute key="org.eclipse.jdt.launching.SOURCE_PATH_PROVIDER" value="org.eclipse.m2e.launchconfig.sourcepathProvider"/>
<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-ea"/>
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/CommandLineOptions.java b/vipra-cmd/src/main/java/de/vipra/cmd/CommandLineOptions.java
index bdf43b15..f75c615d 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/CommandLineOptions.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/CommandLineOptions.java
@@ -22,8 +22,9 @@ public class CommandLineOptions {
public static final Option DELETE = Option.builder("D").longOpt("delete").desc("delete existing models").hasArgs().argName("models...").build();
public static final Option EDIT = Option.builder("E").longOpt("edit").desc("edit config of selected models").hasArgs().argName("models...")
.build();
+ public static final Option PRINT = Option.builder("p").longOpt("print").desc("print model configuration").hasArgs().argName("models...").build();
public static final Option IMPORT = Option.builder("I").longOpt("import").desc("import data from json into selected models").hasArgs()
- .argName("models...").build();
+ .argName("files...").build();
public static final Option MODEL = Option.builder("M").longOpt("model").desc("generate topics on selected models").build();
public static final Option SELECT = Option.builder("S").longOpt("select").desc("select models").hasArgs().argName("models...").build();
@@ -32,7 +33,8 @@ public class CommandLineOptions {
private final String cmdName = "vipra";
public CommandLineOptions() {
- final Option[] optionsArray = { CLEAR, DEBUG, HELP, INDEX, LIST, REREAD, SILENT, TEST, ALL, CREATE, DELETE, EDIT, IMPORT, MODEL, SELECT };
+ final Option[] optionsArray = { CLEAR, DEBUG, HELP, INDEX, LIST, REREAD, SILENT, TEST, ALL, CREATE, DELETE, EDIT, PRINT, IMPORT, MODEL,
+ SELECT };
options = new Options();
for (final Option option : optionsArray)
options.addOption(option);
@@ -127,6 +129,14 @@ public class CommandLineOptions {
return getOptionValues(EDIT);
}
+ public boolean isPrint() {
+ return hasOption(PRINT);
+ }
+
+ public String[] modelsToPrint() {
+ return getOptionValues(PRINT);
+ }
+
public boolean isImport() {
return hasOption(IMPORT);
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/Main.java b/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
index 145b7965..e6cb162d 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
@@ -17,6 +17,7 @@ import de.vipra.cmd.option.ImportCommand;
import de.vipra.cmd.option.IndexingCommand;
import de.vipra.cmd.option.ListModelsCommand;
import de.vipra.cmd.option.ModelingCommand;
+import de.vipra.cmd.option.PrintModelCommand;
import de.vipra.cmd.option.TestCommand;
import de.vipra.util.ConsoleUtils;
import de.vipra.util.ex.ConfigException;
@@ -59,18 +60,21 @@ public class Main {
if (opts.isClear())
commands.add(new ClearCommand());
- if (opts.isCreate())
- commands.add(new CreateModelCommand(opts.modelsToCreate()));
-
if (opts.isDelete())
commands.add(new DeleteModelCommand(opts.modelsToDelete()));
+ if (opts.isCreate())
+ commands.add(new CreateModelCommand(opts.modelsToCreate()));
+
if (opts.isList())
commands.add(new ListModelsCommand());
if (opts.isEdit())
commands.add(new EditModelCommand(opts.modelsToEdit()));
+ if (opts.isPrint())
+ commands.add(new PrintModelCommand(opts.modelsToPrint()));
+
if (opts.isImport())
commands.add(new ImportCommand(opts.selectedModels(), opts.filesToImport()));
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/lda/Analyzer.java b/vipra-cmd/src/main/java/de/vipra/cmd/lda/Analyzer.java
index e8df881a..7f331e87 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/lda/Analyzer.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/lda/Analyzer.java
@@ -42,7 +42,7 @@ import de.vipra.util.model.TopicWord;
import de.vipra.util.model.Window;
import de.vipra.util.model.WindowFull;
import de.vipra.util.service.MongoService;
-import de.vipra.util.service.Service.QueryBuilder;
+import de.vipra.util.service.QueryBuilder;
public class Analyzer {
@@ -122,9 +122,8 @@ public class Analyzer {
long lastStarted = System.nanoTime();
long lastDuration = 0;
double avgDuration = 0;
- final double smoothingFactor = 0.005;
-
- printProgress(0, 0, iteration, maxIterationsLength, 0, modelConfig);
+ final double smoothingFactor = 0.1;
+ int lastLength = printProgress(0, 0, iteration, maxIterationsLength, 0, modelConfig, 0);
while ((line = in.readLine()) != null) {
if (line.contains("EM iter")) {
@@ -143,7 +142,7 @@ public class Analyzer {
avgDuration = smoothingFactor * lastDuration + (1 - smoothingFactor) * avgDuration;
final long remainingDuration = (long) avgDuration * (modelConfig.getDynamicMaxIterations() - iteration);
- printProgress(tenthPercent, progress, iteration, maxIterationsLength, remainingDuration, modelConfig);
+ lastLength = printProgress(tenthPercent, progress, iteration, maxIterationsLength, remainingDuration, modelConfig, lastLength);
}
}
@@ -287,7 +286,6 @@ public class Analyzer {
// create sequence
final SequenceFull newSequenceFull = new SequenceFull();
newSequenceFull.setWindow(new Window(newWindows.get(idxSeq)));
- Collections.sort(newSequenceWords, Comparator.reverseOrder());
newSequenceFull.setWords(newSequenceWords);
newSequenceFull.setRelevance(relevance);
newSequenceFull.setRelevanceChange(relevance - prevRelevance);
@@ -441,7 +439,7 @@ public class Analyzer {
// recreate entities
- final QueryBuilder builder = QueryBuilder.builder().criteria("topicModel", new TopicModel(modelConfig.getName()));
+ final QueryBuilder builder = QueryBuilder.builder().eq("topicModel", new TopicModel(modelConfig.getName()));
dbWindows.deleteMultiple(builder);
dbSequences.deleteMultiple(builder);
@@ -455,12 +453,14 @@ public class Analyzer {
dbTopicModels.replaceSingle(topicModel);
}
- private void printProgress(final int tenthPercent, final double progress, final int iteration, final int maxIterationsLength,
- final long remainingNanos, final TopicModelConfig modelConfig) {
- ConsoleUtils.infoNOLF(" [" + StringUtils.repeat("#", tenthPercent) + StringUtils.repeat(" ", 10 - tenthPercent) + "] "
+ private int printProgress(final int tenthPercent, final double progress, final int iteration, final int maxIterationsLength,
+ final long remainingNanos, final TopicModelConfig modelConfig, final int lastLength) {
+ String msg = " [" + StringUtils.repeat("#", tenthPercent) + StringUtils.repeat(" ", 10 - tenthPercent) + "] "
+ StringUtils.pad(Integer.toString((int) Math.floor(progress)), 3, true) + "% ("
+ StringUtils.pad(Integer.toString(iteration), maxIterationsLength, true) + "/" + modelConfig.getDynamicMinIterations() + "-"
- + modelConfig.getDynamicMaxIterations() + ")\r");
+ + modelConfig.getDynamicMaxIterations() + ") " + StringUtils.timeString(remainingNanos, false, true, false) + "\r";
+ ConsoleUtils.infoNOLF(msg);
+ return msg.length() - 1;
}
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteModelCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteModelCommand.java
index 00114a42..f54b707c 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteModelCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteModelCommand.java
@@ -15,7 +15,7 @@ import de.vipra.util.model.TopicModel;
import de.vipra.util.model.WindowFull;
import de.vipra.util.model.WordFull;
import de.vipra.util.service.MongoService;
-import de.vipra.util.service.Service.QueryBuilder;
+import de.vipra.util.service.QueryBuilder;
public class DeleteModelCommand implements Command {
@@ -47,7 +47,7 @@ public class DeleteModelCommand implements Command {
dbTopicModels.deleteMultiple(Arrays.asList(names));
for (final String name : names) {
- final QueryBuilder builder = QueryBuilder.builder().criteria("topicModel", new TopicModel(name));
+ final QueryBuilder builder = QueryBuilder.builder().eq("topicModel", new TopicModel(name));
dbArticles.deleteMultiple(builder);
dbTopics.deleteMultiple(builder);
dbWindows.deleteMultiple(builder);
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
index 4391d21c..4d1fa203 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
@@ -155,31 +155,36 @@ public class ImportCommand implements Command {
// preprocess text
final ProcessedText processedText = processor.process(modelConfig, article.getText());
- // spotlight analysis
- if (spotlightAnalyzer != null) {
- final SpotlightResponse spotlightResponse = spotlightAnalyzer.analyze(article.getText());
-
- String articleText = article.getText();
-
- for (final TextEntityCount textEntityCount : spotlightResponse.getEntities()) {
- // get new text entity
- final TextEntityFull newTextEntity = new TextEntityFull(textEntityCount.getEntity());
- newTextEntity.setTopicModel(new TopicModel(topicModel.getId()));
- newTextEntities.add(newTextEntity);
-
- // insert entity into text
- articleText = articleText.replaceAll("(?i)\\b" + Pattern.quote(textEntityCount.getEntity().getId()) + "\\b(?![^<]*>|[^<>]*</)",
- Matcher.quoteReplacement(textEntityCount.getEntity().aTag()));
- }
-
- article.setEntities(spotlightResponse.getEntities());
- article.setText(articleText);
- }
-
if (modelConfig.getProcessorMode() != ProcessorMode.ENTITIES
&& processedText.getReducedWordCount() < modelConfig.getDocumentMinimumLength()) {
- ConsoleUtils.info(ConsoleUtils.positionString(current, max) + " skipped \"" + object.get("title"));
+ ConsoleUtils.info(ConsoleUtils.positionString(current, max) + " skipped \"" + object.get("title") + "\" ("
+ + processedText.getReducedWordCount() + ")");
} else {
+ // spotlight analysis
+ if (spotlightAnalyzer != null) {
+ final SpotlightResponse spotlightResponse = spotlightAnalyzer.analyze(article.getText());
+
+ String articleText = article.getText();
+
+ final List<TextEntityCount> textEntitiesCounts = spotlightResponse.getEntities();
+ if (textEntitiesCounts != null) {
+ for (final TextEntityCount textEntityCount : textEntitiesCounts) {
+ // get new text entity
+ final TextEntityFull newTextEntity = new TextEntityFull(textEntityCount.getEntity());
+ newTextEntity.setTopicModel(new TopicModel(topicModel.getId()));
+ newTextEntities.add(newTextEntity);
+
+ // insert entity into text
+ articleText = articleText.replaceAll(
+ "(?i)\\b" + Pattern.quote(textEntityCount.getEntity().getId()) + "\\b(?![^<]*>|[^<>]*</)",
+ Matcher.quoteReplacement(textEntityCount.getEntity().aTag()));
+ }
+ }
+
+ article.setEntities(textEntitiesCounts);
+ article.setText(articleText);
+ }
+
article.setProcessedText(processedText.getWords());
article.setWords(processedText.getArticleWords());
article.setTopicModel(new TopicModel(topicModel.getId()));
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/PrintModelCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/PrintModelCommand.java
new file mode 100644
index 00000000..e315b3c0
--- /dev/null
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/PrintModelCommand.java
@@ -0,0 +1,29 @@
+package de.vipra.cmd.option;
+
+import de.vipra.util.Config;
+import de.vipra.util.ConsoleUtils;
+import de.vipra.util.model.TopicModelConfig;
+
+public class PrintModelCommand implements Command {
+
+ private final String[] names;
+ private Config config;
+
+ public PrintModelCommand(final String[] names) {
+ this.names = names;
+ }
+
+ private void printModelConfig(final String name, final TopicModelConfig modelConfig) {
+ ConsoleUtils.info("model: " + name + "\n" + modelConfig.toPrettyString());
+ }
+
+ @Override
+ public void run() throws Exception {
+ config = Config.getConfig();
+
+ for (final String name : names) {
+ printModelConfig(name, config.getTopicModelConfig(name));
+ }
+ }
+
+}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/text/SpotlightResponse.java b/vipra-cmd/src/main/java/de/vipra/cmd/text/SpotlightResponse.java
index 19f2cc40..646fddfa 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/text/SpotlightResponse.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/text/SpotlightResponse.java
@@ -29,6 +29,8 @@ public class SpotlightResponse {
}
public List<TextEntityCount> getEntities() {
+ if (resources == null)
+ return new ArrayList<>();
final CountMap<String> textEntitiesCount = new CountMap<>(resources.size());
final Set<TextEntity> textEntities = new HashSet<>(resources.size());
diff --git a/vipra-ui/app/html/articles/show.html b/vipra-ui/app/html/articles/show.html
index efc9fb98..470ba5bb 100644
--- a/vipra-ui/app/html/articles/show.html
+++ b/vipra-ui/app/html/articles/show.html
@@ -34,7 +34,7 @@
<th>Date</th>
<td ng-bind="::articleDate"></td>
</tr>
- <tr>
+ <tr ng-if="article.url">
<th>URL</th>
<td>
<a ng-href="{{::article.url}}" target="_blank">
@@ -74,7 +74,7 @@
</div>
<div class="col-md-4">
<h3>Share</h3>
- <div class="pie-chart" id="topic-share" highcharts="topicShare" style="height: 200px;"></div>
+ <div class="pie-chart" id="topic-share" highcharts="topicShare" ng-class="{'pie-small':!article.topics.length}"></div>
</div>
</div>
<h3>Similar articles</h3>
diff --git a/vipra-ui/app/html/directives/window-dropdown.html b/vipra-ui/app/html/directives/window-dropdown.html
new file mode 100644
index 00000000..69557e06
--- /dev/null
+++ b/vipra-ui/app/html/directives/window-dropdown.html
@@ -0,0 +1,5 @@
+<ol class="nya-bs-select nya-bs-condensed" ng-model="ngModel" ng-class="{dropup:dropup}">
+ <li value="{{window.id}}" class="nya-bs-option" ng-repeat="window in windows">
+ <a ng-bind="window.label"></a>
+ </li>
+</ol>
diff --git a/vipra-ui/app/html/network.html b/vipra-ui/app/html/network.html
index 98282f1d..a5bdda91 100644
--- a/vipra-ui/app/html/network.html
+++ b/vipra-ui/app/html/network.html
@@ -2,13 +2,17 @@
<div class="fullsize navpadding">
<div class="graph-legend overlay">
<div class="checkbox">
- <input type="checkbox" id="showArticles" ng-model="shown.articles" ng-disabled="type == 'articles'">
+ <input type="checkbox" id="showArticles" ng-model="shown.articles">
<label for="showArticles" style="color:{{colors.articles}}">Articles</label>
</div>
<div class="checkbox">
- <input type="checkbox" id="showTopics" ng-model="shown.topics" ng-disabled="type == 'topics'">
+ <input type="checkbox" id="showTopics" ng-model="shown.topics">
<label for="showTopics" style="color:{{colors.topics}}">Topics</label>
</div>
+ <div class="checkbox">
+ <input type="checkbox" id="showWords" ng-model="shown.words">
+ <label for="showWords">Words</label>
+ </div>
</div>
<div class="fullsize" id="visgraph"></div>
</div>
diff --git a/vipra-ui/app/js/controllers.js b/vipra-ui/app/js/controllers.js
index dff06132..33a80659 100644
--- a/vipra-ui/app/js/controllers.js
+++ b/vipra-ui/app/js/controllers.js
@@ -239,8 +239,8 @@
/**
* Network controller
*/
- app.controller('NetworkController', ['$scope', '$state', '$stateParams', '$timeout', 'ArticleFactory', 'TopicFactory',
- function($scope, $state, $stateParams, $timeout, ArticleFactory, TopicFactory) {
+ app.controller('NetworkController', ['$scope', '$state', '$stateParams', '$timeout', 'ArticleFactory', 'TopicFactory', 'WordFactory', 'WindowFactory',
+ function($scope, $state, $stateParams, $timeout, ArticleFactory, TopicFactory, WordFactory, WindowFactory) {
var id = 0,
ids = {},
@@ -248,7 +248,8 @@
$scope.colors = {
articles: '#BBC9D2',
- topics: '#DBB234'
+ topics: '#DBB234',
+ words: '#FFFFFF'
};
$scope.nodes = new vis.DataSet();
$scope.edges = new vis.DataSet();
@@ -263,7 +264,7 @@
size: 14
},
shape: 'dot',
- borderWidth: 0
+ borderWidth: 1
},
edges: {
color: {
@@ -282,7 +283,8 @@
};
$scope.shown = {
articles: true,
- topics: true
+ topics: true,
+ words: true
};
var factory;
@@ -290,6 +292,8 @@
factory = ArticleFactory;
else if ($stateParams.type === 'topics')
factory = TopicFactory;
+ else if ($stateParams.type === 'word')
+ factory = WordFactory;
else {
console.log('unknown network type');
return;
@@ -311,6 +315,8 @@
$scope.nodes.add([articleNode(data)]);
else if ($stateParams.type === 'topics')
$scope.nodes.add([topicNode(data)]);
+ else if ($stateParams.type === 'words')
+ $scope.nodes.add([wordNode(data)]);
ids[data.id] = id;
// create graph
@@ -329,7 +335,7 @@
return {
id: id,
title: title,
- label: title.multiline(5),
+ label: title.multiline(5).ellipsize(50),
type: type,
show: show,
dbid: dbid,
@@ -352,6 +358,10 @@
return newNode(article.title, 'article', 'articles.show', article.id, $scope.colors.articles, 'square');
};
+ var wordNode = function(word) {
+ return newNode(word.id, 'word', 'word.show', word.id, $scope.colors.words, 'box');
+ };
+
var edgeExists = function(idA, idB) {
if (idB < idA) {
var tmp = idA;
@@ -409,30 +419,65 @@
var node = $scope.nodes.get(props.nodes[0]);
if (node) {
if (node.type === 'article' && $scope.shown.topics) {
- // node is article, load article to get topics
- ArticleFactory.get({
- id: node.dbid
- }, function(data) {
- if (data.topics) {
- for (var i = 0; i < data.topics.length; i++)
- data.topics[i] = data.topics[i].topic;
- constructor(data.topics, node, topicNode);
- }
- });
+ $scope.loadArticle(node);
} else if (node.type === 'topic') {
- // node is topic, load topic to get articles
- if ($scope.shown.articles)
- TopicFactory.articles({
- id: node.dbid
- }, function(data) {
- constructor(data, node, articleNode);
- });
+ $scope.loadTopic(node);
+ } else if (node.type === 'word') {
+ $scope.loadWord(node);
}
$scope.nodes.update(node);
}
}, 500);
};
+ $scope.loadArticle = function(node) {
+ ArticleFactory.get({
+ id: node.dbid
+ }, function(data) {
+ if (data.topics) {
+ for (var i = 0; i < data.topics.length; i++)
+ data.topics[i] = data.topics[i].topic;
+ constructor(data.topics, node, topicNode);
+ }
+ });
+ };
+
+ $scope.loadTopic = function(node) {
+ if ($scope.shown.articles) {
+ TopicFactory.articles({
+ id: node.dbid
+ }, function(data) {
+ constructor(data, node, articleNode);
+ });
+ }
+ if ($scope.shown.words) {
+ TopicFactory.get({
+ id: node.dbid
+ }, function(data) {
+ constructor(data.words, node, wordNode);
+ });
+ }
+ };
+
+ $scope.loadWord = function(node) {
+ if($scope.shown.articles) {
+ ArticleFactory.query({
+ word: node.dbid,
+ topicModel: $scope.rootModels.topicModel.id
+ }, function(data) {
+ constructor(data, node, articleNode);
+ });
+ }
+ if($scope.shown.topics) {
+ TopicFactory.query({
+ word: node.dbid,
+ topicModel: $scope.rootModels.topicModel.id
+ }, function(data) {
+ constructor(data, node, topicNode);
+ });
+ }
+ };
+
// on node open
$scope.open = function(props) {
$timeout.cancel(selectTimeout);
@@ -602,6 +647,12 @@
$scope.$watch('explorerModels.sorttopics', function() {
if (!$scope.topics) return;
+ if($scope.explorerModels.sorttopics === 'name') {
+ $scope.explorerModels.sortdir = false;
+ } else {
+ $scope.explorerModels.sortdir = true;
+ }
+
$timeout(function() {
for (var i = 0; i < $scope.topics.length; i++)
$scope.topics[i].topicCurrValue = $scope.topicCurrValue($scope.topics[i]);
diff --git a/vipra-ui/app/js/directives.js b/vipra-ui/app/js/directives.js
index 80b16697..46a5a8f1 100644
--- a/vipra-ui/app/js/directives.js
+++ b/vipra-ui/app/js/directives.js
@@ -258,7 +258,7 @@
if (newValue) {
for (var i = 0, s; i < $scope.sequences.length; i++) {
s = $scope.sequences[i];
- s.label = Vipra.sequenceLabel(s.window.startDate, s.window.windowResolution);
+ s.label = Vipra.windowLabel(s.window.startDate, s.window.windowResolution);
}
}
});
@@ -267,6 +267,31 @@
};
}]);
+ app.directive('windowDropdown', ['WindowFactory', function(WindowFactory) {
+ return {
+ scope: {
+ ngModel: '=',
+ topicModel: '=',
+ dropup: '@'
+ },
+ link: function($scope) {
+ $scope.dropup = $scope.dropup === 'true';
+ $scope.$watch('topicModel', function(value) {
+ if(value) {
+ WindowFactory.query({
+ topicModel: value.id
+ }, function(data) {
+ for(var i = 0; i < data.length; i++)
+ data.label = Vipra.windowLabel(data[i].startDate, data[i].windowResolution);
+ $scope.windows = data;
+ });
+ }
+ });
+ },
+ templateUrl: '/html/directives/window-dropdown.html'
+ };
+ }]);
+
app.directive('sortBy', [function() {
return {
restrict: 'A',
diff --git a/vipra-ui/app/js/helpers.js b/vipra-ui/app/js/helpers.js
index 8d58156a..c3e02bbf 100644
--- a/vipra-ui/app/js/helpers.js
+++ b/vipra-ui/app/js/helpers.js
@@ -38,7 +38,7 @@
return 'id' + Math.random().toString(36).substring(7);
};
- Vipra.sequenceLabel = function(date, res) {
+ Vipra.windowLabel = function(date, res) {
date = moment(date);
var parts = [];
if (res === 'QUARTER') {
@@ -177,6 +177,11 @@
return this.split(new RegExp("((?:\\w+ ){" + max + "})", "g")).filter(Boolean).join("\n");
};
+ if (typeof String.prototype.ellipsize === 'undefined')
+ String.prototype.ellipsize = function(max) {
+ return this.length <= max ? this : this.substring(0, max) + '...';
+ };
+
if (typeof String.prototype.startsWith === 'undefined')
String.prototype.startsWith = function(start) {
return this.lastIndexOf(start, 0) === 0;
diff --git a/vipra-ui/app/less/app.less b/vipra-ui/app/less/app.less
index efe492d5..54b73042 100644
--- a/vipra-ui/app/less/app.less
+++ b/vipra-ui/app/less/app.less
@@ -498,6 +498,14 @@ entity-menu {
}
}
+.pie-chart {
+ height: 220px;
+
+ &.pie-small {
+ height: 180px;
+ }
+}
+
@-moz-keyframes spin {
100% {
-moz-transform: rotateY(360deg);
diff --git a/vipra-util/src/main/java/de/vipra/util/StringUtils.java b/vipra-util/src/main/java/de/vipra/util/StringUtils.java
index d8b6d69b..c903def6 100644
--- a/vipra-util/src/main/java/de/vipra/util/StringUtils.java
+++ b/vipra-util/src/main/java/de/vipra/util/StringUtils.java
@@ -50,13 +50,15 @@ public class StringUtils {
return join(Arrays.asList(arr), separator, reverse);
}
- public static String timeString(long nanos, final boolean showMillis, final boolean compactHMS) {
+ public static String timeString(long nanos, final boolean showMillis, final boolean compactHMS, final boolean onlyHMS) {
final List<String> parts = new ArrayList<String>(6);
- final long days = TimeUnit.NANOSECONDS.toDays(nanos);
- if (days > 0) {
- parts.add(days + "d");
- nanos -= TimeUnit.DAYS.toNanos(days);
+ if (!onlyHMS) {
+ final long days = TimeUnit.NANOSECONDS.toDays(nanos);
+ if (days > 0) {
+ parts.add(days + "d");
+ nanos -= TimeUnit.DAYS.toNanos(days);
+ }
}
final long hours = TimeUnit.NANOSECONDS.toHours(nanos);
@@ -89,7 +91,7 @@ public class StringUtils {
nanos -= TimeUnit.SECONDS.toNanos(seconds);
}
- if (showMillis) {
+ if (showMillis && !onlyHMS) {
final long millis = TimeUnit.NANOSECONDS.toMillis(nanos);
if (millis > 0 || compactHMS) {
if (compactHMS) {
@@ -108,11 +110,11 @@ public class StringUtils {
}
public static String timeString(final long nanos, final boolean showMillis) {
- return timeString(nanos, showMillis, false);
+ return timeString(nanos, showMillis, false, false);
}
public static String timeString(final long nanos) {
- return timeString(nanos, true, false);
+ return timeString(nanos, true, false, false);
}
public static String padNumber(final long number, final int length) {
diff --git a/vipra-util/src/main/java/de/vipra/util/Tuple.java b/vipra-util/src/main/java/de/vipra/util/Tuple.java
deleted file mode 100644
index 47613e2f..00000000
--- a/vipra-util/src/main/java/de/vipra/util/Tuple.java
+++ /dev/null
@@ -1,35 +0,0 @@
-package de.vipra.util;
-
-public class Tuple<X, Y> {
-
- private X first;
- private Y second;
-
- public Tuple() {}
-
- public Tuple(final X first, final Y second) {
- this.first = first;
- this.second = second;
- }
-
- public X first() {
- return first;
- }
-
- public void setFirst(final X first) {
- this.first = first;
- }
-
- public Y second() {
- return second;
- }
-
- public void setSecond(final Y second) {
- this.second = second;
- }
-
- public static <X, Y> Tuple<X, Y> pair(final X first, final Y second) {
- return new Tuple<>(first, second);
- }
-
-}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TopicModelConfig.java b/vipra-util/src/main/java/de/vipra/util/model/TopicModelConfig.java
index 7e842401..aa6fe9f1 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/TopicModelConfig.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/TopicModelConfig.java
@@ -241,4 +241,14 @@ public class TopicModelConfig implements Serializable {
+ dynamicMinIterations + "-" + dynamicMaxIterations + "]";
}
+ public String toPrettyString() {
+ return " kTopics: " + kTopics + "\n kTopWords: " + kTopWords + "\n dynamicMinIterations: " + dynamicMinIterations
+ + "\n dynamicMaxIterations: " + dynamicMaxIterations + "\n staticIterations: " + staticIterations + "\n topicAutoNamingWords: "
+ + topicAutoNamingWords + "\n maxSimilarDocuments: " + maxSimilarDocuments + "\n documentMinimumLength: " + documentMinimumLength
+ + "\n documentMinimumWordFrequency: " + documentMinimumWordFrequency + "\n spotlightSupport: " + spotlightSupport
+ + "\n spotlightConfidence: " + spotlightConfidence + "\n minTopicShare: " + minTopicShare + "\n minRelativeProbability: "
+ + minRelativeProbability + "\n risingDecayLambda: " + risingDecayLambda + "\n maxSimilarDocumentsDivergence: "
+ + maxSimilarDocumentsDivergence + "\n windowResolution: " + windowResolution + "\n processorMode: " + processorMode;
+ }
+
}
diff --git a/vipra-util/src/main/java/de/vipra/util/service/MongoService.java b/vipra-util/src/main/java/de/vipra/util/service/MongoService.java
index 0462fafe..b0c10474 100644
--- a/vipra-util/src/main/java/de/vipra/util/service/MongoService.java
+++ b/vipra-util/src/main/java/de/vipra/util/service/MongoService.java
@@ -13,14 +13,13 @@ import org.mongodb.morphia.query.UpdateOperations;
import de.vipra.util.Config;
import de.vipra.util.ListUtils;
import de.vipra.util.Mongo;
-import de.vipra.util.Tuple;
import de.vipra.util.an.QueryIgnore;
import de.vipra.util.an.UpdateIgnore;
import de.vipra.util.ex.ConfigException;
import de.vipra.util.ex.DatabaseException;
import de.vipra.util.model.Model;
-public class MongoService<Type extends Model<IdType>, IdType> implements Service<Type, IdType, DatabaseException> {
+public class MongoService<Type extends Model<IdType>, IdType> {
private final Datastore datastore;
private final Class<Type> clazz;
@@ -56,12 +55,10 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
this.ignoredFieldsMultiQuery = ignoreMulti.toArray(new String[ignoreMulti.size()]);
}
- @Override
public Type getSingle(final IdType id, final String... fields) throws DatabaseException {
return getSingle(id, QueryBuilder.builder().fields(true, fields));
}
- @Override
public Type getSingle(final IdType id, final QueryBuilder builder) throws DatabaseException {
final Query<Type> query = datastore.createQuery(clazz).field("_id").equal(id);
@@ -83,39 +80,16 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
return t;
}
- @Override
public List<Type> getMultiple(final Integer skip, final Integer limit, final String sortBy, final String... fields) {
return getMultiple(QueryBuilder.builder().skip(skip).limit(limit).sortBy(sortBy).fields(true, fields));
}
- @Override
- public List<Type> getMultiple(final Integer skip, final Integer limit, final String sortBy, final Tuple<String, Object> criteria,
- final String... fields) {
- return getMultiple(QueryBuilder.builder().skip(skip).limit(limit).sortBy(sortBy).fields(true, fields).criteria(criteria));
- }
-
- @Override
public List<Type> getMultiple(final QueryBuilder builder) {
final Query<Type> query = datastore.createQuery(clazz);
if (builder != null) {
- if (builder.getSkip() != null && builder.getSkip() > 0)
- query.offset(builder.getSkip());
- if (builder.getLimit() != null && builder.getLimit() > 0)
- query.limit(builder.getLimit());
- if (builder.getSortBy() != null)
- query.order(builder.getSortBy());
- if (builder.getCriteria() != null)
- for (final Tuple<String, Object> criteria : builder.getCriteria())
- query.field(criteria.first()).equal(criteria.second());
- if (builder.getFields() != null) {
- final String[] fields = builder.getFields();
- if (builder.isInclude()) {
- query.retrievedFields(true, fields);
- } else {
- query.retrievedFields(false, fields);
- }
- } else if (!builder.isAllFields() && ignoredFieldsMultiQuery.length > 0) {
+ builder.build(query);
+ if (!builder.isAllFields() && ignoredFieldsMultiQuery.length > 0) {
query.retrievedFields(false, ignoredFieldsMultiQuery);
}
} else if (ignoredFieldsMultiQuery.length > 0) {
@@ -125,12 +99,10 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
return list;
}
- @Override
public List<Type> getAll(final String... fields) {
return getMultiple(QueryBuilder.builder().fields(true, fields));
}
- @Override
public Type createSingle(final Type t) throws DatabaseException {
if (t == null)
throw new DatabaseException(new NullPointerException("entity is null"));
@@ -139,7 +111,6 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
return t;
}
- @Override
public List<Type> createMultiple(final Iterable<Type> t) throws DatabaseException {
if (t == null)
throw new DatabaseException(new NullPointerException("entities are null"));
@@ -149,7 +120,6 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
return list;
}
- @Override
public long deleteSingle(final IdType id) throws DatabaseException {
if (id == null)
throw new DatabaseException(new NullPointerException("id is null"));
@@ -158,7 +128,6 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
return deleted;
}
- @Override
public long deleteMultiple(final Iterable<IdType> ids) throws DatabaseException {
if (ids == null)
throw new DatabaseException(new NullPointerException("ids are null"));
@@ -167,20 +136,15 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
return deleted;
}
- @Override
public long deleteMultiple(final QueryBuilder builder) throws DatabaseException {
final Query<Type> query = datastore.createQuery(clazz);
- if (builder != null) {
- if (builder.getCriteria() != null)
- for (final Tuple<String, Object> criteria : builder.getCriteria())
- query.field(criteria.first()).equal(criteria.second());
- }
+ if (builder != null)
+ builder.build(query);
final int deleted = datastore.delete(query).getN();
return deleted;
}
- @Override
public void replaceSingle(final Type t) throws DatabaseException {
if (t == null)
throw new DatabaseException(new NullPointerException("entity is null"));
@@ -190,7 +154,6 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
datastore.save(t);
}
- @Override
public void replaceMultiple(final Iterable<Type> ts) throws DatabaseException {
if (ts == null)
throw new DatabaseException(new NullPointerException("entities are null"));
@@ -198,7 +161,6 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
datastore.save(ts);
}
- @Override
public void updateSingle(final Type t, final boolean upsert, final String... fields) throws DatabaseException {
if (t == null)
throw new DatabaseException(new NullPointerException("entity is null"));
@@ -233,29 +195,20 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
}
}
- @Override
public void updateSingle(final Type t, final String... fields) throws DatabaseException {
updateSingle(t, false, fields);
}
- @Override
public void drop() {
datastore.getCollection(clazz).drop();
}
- @Override
public long count(final QueryBuilder builder) {
if (builder == null)
return datastore.getCount(clazz);
final Query<Type> query = datastore.createQuery(clazz);
- if (builder.getSkip() != null && builder.getSkip() > 0)
- query.offset(builder.getSkip());
- if (builder.getLimit() != null && builder.getLimit() > 0)
- query.limit(builder.getLimit());
- if (builder.getCriteria() != null)
- for (final Tuple<String, Object> criteria : builder.getCriteria())
- query.field(criteria.first()).equal(criteria.second());
+ builder.build(query);
return datastore.getCount(query);
}
diff --git a/vipra-util/src/main/java/de/vipra/util/service/QueryBuilder.java b/vipra-util/src/main/java/de/vipra/util/service/QueryBuilder.java
new file mode 100644
index 00000000..3fe6fd54
--- /dev/null
+++ b/vipra-util/src/main/java/de/vipra/util/service/QueryBuilder.java
@@ -0,0 +1,217 @@
+package de.vipra.util.service;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.mongodb.morphia.query.Query;
+
+/**
+ * QueryBuilder instances are used to create complex queries for use with the
+ * getMultiple method
+ *
+ * @see {@link Service#getMultiple(QueryBuilder)}
+ */
+public class QueryBuilder {
+
+ public static enum CriterionType {
+ EQ,
+ LT,
+ LTE,
+ GT,
+ GTE
+ };
+
+ public static class Criterion {
+ public final String field;
+ public final CriterionType type;
+ public final Object value;
+
+ public Criterion(final String field, final CriterionType type, final Object value) {
+ this.field = field;
+ this.type = type;
+ this.value = value;
+ }
+
+ public void apply(final Query<?> query) {
+ switch (type) {
+ case EQ:
+ query.field(field).equal(value);
+ break;
+ case LT:
+ query.field(field).lessThan(value);
+ break;
+ case LTE:
+ query.field(field).lessThanOrEq(value);
+ break;
+ case GT:
+ query.field(field).greaterThan(value);
+ break;
+ case GTE:
+ query.field(field).greaterThanOrEq(value);
+ break;
+ }
+ }
+ }
+
+ private Integer skip;
+ private Integer limit;
+ private String sortBy;
+ private Set<Criterion> criteria;
+ private String[] fields;
+ private boolean include;
+ private boolean allFields;
+
+ private QueryBuilder() {
+ this.criteria = new HashSet<>();
+ }
+
+ public static QueryBuilder builder() {
+ return new QueryBuilder();
+ }
+
+ /**
+ * Skip n entries
+ *
+ * @param skip
+ * entries to skip
+ * @return QueryBuilder instance
+ */
+ public QueryBuilder skip(final Integer skip) {
+ if (skip == null || skip >= 0)
+ this.skip = skip;
+ return this;
+ }
+
+ /**
+ * Limit return size.
+ *
+ * @param limit
+ * maximum return size
+ * @return QueryBuilder instance
+ */
+ public QueryBuilder limit(final Integer limit) {
+ if (limit == null || limit >= 0)
+ this.limit = limit;
+ return this;
+ }
+
+ /**
+ * Sort result by field
+ *
+ * @param sortBy
+ * field to sort by.
+ * @return QueryBuilder instance
+ */
+ public QueryBuilder sortBy(final String sortBy) {
+ if (sortBy == null || !sortBy.isEmpty())
+ this.sortBy = sortBy;
+ return this;
+ }
+
+ public QueryBuilder eq(final String field, final Object value) {
+ if (field != null && !field.isEmpty() && value != null)
+ criteria.add(new Criterion(field, CriterionType.EQ, value));
+ return this;
+ }
+
+ public QueryBuilder lt(final String field, final Object value) {
+ if (field != null && !field.isEmpty() && value != null)
+ criteria.add(new Criterion(field, CriterionType.LT, value));
+ return this;
+ }
+
+ public QueryBuilder lte(final String field, final Object value) {
+ if (field != null && !field.isEmpty() && value != null)
+ criteria.add(new Criterion(field, CriterionType.LTE, value));
+ return this;
+ }
+
+ public QueryBuilder gt(final String field, final Object value) {
+ if (field != null && !field.isEmpty() && value != null)
+ criteria.add(new Criterion(field, CriterionType.GT, value));
+ return this;
+ }
+
+ public QueryBuilder gte(final String field, final Object value) {
+ if (field != null && !field.isEmpty() && value != null)
+ criteria.add(new Criterion(field, CriterionType.GTE, value));
+ return this;
+ }
+
+ /**
+ * Fields to return. Set include to false to exclude. Cannot be applied
+ * multiple times, previous calls will be overwritten by later calls.
+ *
+ * @param include
+ * true to include, false to exclude
+ * @param fields
+ * fields to in/exclude
+ * @return QueryBuilder instance
+ */
+ public QueryBuilder fields(final boolean include, final String... fields) {
+ if (fields != null) {
+ this.include = include;
+ for (final String field : fields) {
+ if (field.equalsIgnoreCase("_all")) {
+ this.allFields = true;
+ this.fields = null;
+ return this;
+ }
+ }
+ this.fields = fields;
+ }
+ return this;
+ }
+
+ public Integer getSkip() {
+ return skip;
+ }
+
+ public Integer getLimit() {
+ return limit;
+ }
+
+ public String getSortBy() {
+ if (sortBy == null)
+ return null;
+ return sortBy.startsWith("+") ? sortBy.substring(1) : sortBy;
+ }
+
+ public Set<Criterion> getCriteria() {
+ return criteria;
+ }
+
+ public boolean isInclude() {
+ return include;
+ }
+
+ public String[] getFields() {
+ return fields;
+ }
+
+ public boolean isAllFields() {
+ return allFields;
+ }
+
+ public Query<?> build(final Query<?> query) {
+ if (getSkip() != null && getSkip() > 0)
+ query.offset(getSkip());
+ if (getLimit() != null && getLimit() > 0)
+ query.limit(getLimit());
+ if (getSortBy() != null)
+ query.order(getSortBy());
+ if (getCriteria() != null)
+ for (final Criterion criterion : getCriteria())
+ criterion.apply(query);
+ if (getFields() != null) {
+ final String[] fields = getFields();
+ if (isInclude()) {
+ query.retrievedFields(true, fields);
+ } else {
+ query.retrievedFields(false, fields);
+ }
+ }
+ return query;
+ }
+
+}
\ No newline at end of file
diff --git a/vipra-util/src/main/java/de/vipra/util/service/Service.java b/vipra-util/src/main/java/de/vipra/util/service/Service.java
deleted file mode 100644
index d198189b..00000000
--- a/vipra-util/src/main/java/de/vipra/util/service/Service.java
+++ /dev/null
@@ -1,319 +0,0 @@
-package de.vipra.util.service;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import de.vipra.util.Tuple;
-import de.vipra.util.model.Model;
-
-/**
- * Generic service interface, implemented by various database services to
- * support multiple database engines.
- *
- * @param <Type>
- * Model type that is returned by service functions
- * @param <IdType>
- * Id type that is used to identify models in the database
- * @param <E>
- * Type of exception
- */
-public interface Service<Type extends Model<IdType>, IdType, E extends Exception> {
-
- /**
- * @see {@link Service#getSingle(Object, QueryBuilder)}
- */
- Type getSingle(IdType id, String... fields) throws E;
-
- /**
- * Returns a single entity from the database or null
- *
- * @param id
- * id of the entity
- * @param builder
- * query builder
- * @return retrieved entity or null
- * @throws E
- */
- Type getSingle(IdType id, QueryBuilder builder) throws E;
-
- /**
- * @see {@link Service#getMultiple(QueryBuilder)}
- */
- List<Type> getMultiple(Integer skip, Integer limit, String sortBy, String... fields) throws E;
-
- /**
- * @see {@link Service#getMultiple(QueryBuilder)}
- */
- List<Type> getMultiple(Integer skip, Integer limit, String sortBy, Tuple<String, Object> criteria, String... fields) throws E;
-
- /**
- * Returns multiple entities from the database.
- *
- * @param builder
- * query builder
- * @return found entities
- * @throws E
- * @see {@link QueryBuilder}
- */
- List<Type> getMultiple(QueryBuilder builder) throws E;
-
- /**
- * Return all entities.
- *
- * @param fields
- * fields to be returned
- * @return all entities
- * @throws E
- */
- List<Type> getAll(String... fields) throws E;
-
- /**
- * Create a single entity in the database
- *
- * @param t
- * Entity to be created
- * @return Created entity, with inserted id if supported
- * @throws E
- */
- Type createSingle(Type t) throws E;
-
- /**
- * Create multiple entities in the database
- *
- * @param t
- * Entities to be created
- * @return Created entities, with inserted ids if supported
- * @throws E
- */
- List<Type> createMultiple(Iterable<Type> t) throws E;
-
- /**
- * Deletes a single entity from the database
- *
- * @param id
- * id of entity to be deleted
- * @return number of deleted entities
- * @throws E
- */
- long deleteSingle(IdType id) throws E;
-
- /**
- * Deletes multiple entries from the database
- *
- * @param ids
- * Entities to be deleted
- * @return number of deleted entries
- * @throws E
- */
- long deleteMultiple(Iterable<IdType> ids) throws E;
-
- long deleteMultiple(QueryBuilder builder) throws E;
-
- /**
- * Replaces a single entity in the database
- *
- * @param t
- * Entity to be updated
- * @throws E
- */
- void replaceSingle(Type t) throws E;
-
- /**
- * Replaces multiple entities in the database
- *
- * @param ts
- * Entities to be updated
- * @throws E
- */
- void replaceMultiple(Iterable<Type> ts) throws E;
-
- /**
- * Updates a single entity in the database
- *
- * @param t
- * Entity to be updated
- * @param upsert
- * true to insert if not exists
- * @param fields
- * Fields to be updated
- * @throws E
- */
- void updateSingle(Type t, boolean upsert, String... fields) throws E;
-
- /**
- * Updates a single entity in the database
- *
- * @param t
- * Entity to be updated
- * @param fields
- * Fields to be updated
- * @throws E
- */
- void updateSingle(Type t, String... fields) throws E;
-
- /**
- * Drop all entities from the database
- *
- * @throws E
- */
- void drop() throws E;
-
- /**
- * Count entities in the database
- *
- * @return number of entities in the database
- * @throws E
- */
- long count(QueryBuilder builder) throws E;
-
- /**
- * QueryBuilder instances are used to create complex queries for use with
- * the getMultiple method
- *
- * @see {@link Service#getMultiple(QueryBuilder)}
- */
- public static class QueryBuilder {
-
- private Integer skip;
- private Integer limit;
- private String sortBy;
- private List<Tuple<String, Object>> criteria;
- private String[] fields;
- private boolean include;
- private boolean allFields;
-
- private QueryBuilder() {}
-
- public static QueryBuilder builder() {
- return new QueryBuilder();
- }
-
- /**
- * Skip n entries
- *
- * @param skip
- * entries to skip
- * @return QueryBuilder instance
- */
- public QueryBuilder skip(final Integer skip) {
- if (skip == null || skip >= 0)
- this.skip = skip;
- return this;
- }
-
- /**
- * Limit return size.
- *
- * @param limit
- * maximum return size
- * @return QueryBuilder instance
- */
- public QueryBuilder limit(final Integer limit) {
- if (limit == null || limit >= 0)
- this.limit = limit;
- return this;
- }
-
- /**
- * Sort result by field
- *
- * @param sortBy
- * field to sort by.
- * @return QueryBuilder instance
- */
- public QueryBuilder sortBy(final String sortBy) {
- if (sortBy == null || !sortBy.isEmpty())
- this.sortBy = sortBy;
- return this;
- }
-
- /**
- * Criteria used for filtering by field
- *
- * @param field
- * field to compare
- * @param value
- * value to compare to
- * @return QueryBuilder instance
- */
- public QueryBuilder criteria(final String field, final Object value) {
- if (field != null && value != null && !field.isEmpty())
- criteria(Tuple.pair(field, value));
- return this;
- }
-
- /**
- * Criteria used for filtering by field
- *
- * @param pair
- * field value pair to compare
- * @return QueryBuilder instance
- */
- public QueryBuilder criteria(final Tuple<String, Object> pair) {
- if (pair != null) {
- if (criteria == null) {
- criteria = new ArrayList<>();
- }
- criteria.add(pair);
- }
- return this;
- }
-
- /**
- * Fields to return. Set include to false to exclude. Cannot be applied
- * multiple times, previous calls will be overwritten by later calls.
- *
- * @param include
- * true to include, false to exclude
- * @param fields
- * fields to in/exclude
- * @return QueryBuilder instance
- */
- public QueryBuilder fields(final boolean include, final String... fields) {
- if (fields != null) {
- this.include = include;
- for (final String field : fields) {
- if (field.equalsIgnoreCase("_all")) {
- this.allFields = true;
- this.fields = null;
- return this;
- }
- }
- this.fields = fields;
- }
- return this;
- }
-
- public Integer getSkip() {
- return skip;
- }
-
- public Integer getLimit() {
- return limit;
- }
-
- public String getSortBy() {
- if (sortBy == null)
- return null;
- return sortBy.startsWith("+") ? sortBy.substring(1) : sortBy;
- }
-
- public List<Tuple<String, Object>> getCriteria() {
- return criteria;
- }
-
- public boolean isInclude() {
- return include;
- }
-
- public String[] getFields() {
- return fields;
- }
-
- public boolean isAllFields() {
- return allFields;
- }
-
- }
-
-}
--
GitLab