diff --git a/vipra-cmd/runcfg/CMD.launch b/vipra-cmd/runcfg/CMD.launch
index 345beb2a746a7c1d1948eb7730ca56f20c95d4ca..7f3fd45a4483d2684146995337cec4f7df7d6ee9 100644
--- a/vipra-cmd/runcfg/CMD.launch
+++ b/vipra-cmd/runcfg/CMD.launch
@@ -11,7 +11,7 @@
</listAttribute>
<stringAttribute key="org.eclipse.jdt.launching.CLASSPATH_PROVIDER" value="org.eclipse.m2e.launchconfig.classpathProvider"/>
<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="de.vipra.cmd.Main"/>
-<stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="-eC bbc:test"/>
+<stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="-tdp"/>
<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="vipra-cmd"/>
<stringAttribute key="org.eclipse.jdt.launching.SOURCE_PATH_PROVIDER" value="org.eclipse.m2e.launchconfig.sourcepathProvider"/>
<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-ea"/>
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/CommandLineOptions.java b/vipra-cmd/src/main/java/de/vipra/cmd/CommandLineOptions.java
index aa975d692b3f2dbd441b6825ad07d84b4ebb9c33..23a021709650a87db646b18c757e01f2079aeceb 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/CommandLineOptions.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/CommandLineOptions.java
@@ -20,6 +20,8 @@ public class CommandLineOptions {
public static final Option TEST = Option.builder("t").longOpt("test").desc("test database connections").build();
public static final Option ALL = Option.builder("A").longOpt("all").desc("select all models (-S all)").build();
public static final Option VERSION = Option.builder("v").longOpt("version").desc("print the version number").build();
+ public static final Option PERF = Option.builder("p").longOpt("perf").desc("generate performance statistics").hasArg().optionalArg(true)
+ .argName("file").build();
public static final Option INDEX = Option.builder("i").longOpt("index").desc("create index for models").hasArgs().argName("[models...]")
.optionalArg(true).build();
@@ -31,7 +33,7 @@ public class CommandLineOptions {
.optionalArg(true).build();
public static final Option EDIT = Option.builder("E").longOpt("edit").desc("edit config of selected models").hasArgs().argName("[models...]")
.optionalArg(true).build();
- public static final Option PRINT = Option.builder("p").longOpt("print").desc("print model configuration").hasArgs().argName("[models...]")
+ public static final Option PRINT = Option.builder("m").longOpt("print").desc("print model configuration").hasArgs().argName("[models...]")
.optionalArg(true).build();
public static final Option IMPORT = Option.builder("I").longOpt("import").desc("import data for models").hasArgs().argName("[models...]")
.optionalArg(true).build();
@@ -43,8 +45,9 @@ public class CommandLineOptions {
.optionalArg(true).build();
public static final Option SELECT = Option.builder("S").longOpt("select").desc("select models").hasArgs().argName("models...").build();
public static final Option BACKUP = Option.builder("B").longOpt("backup").desc("backup data/filebase").hasArg().argName("path").build();
- public static final Option RESTORE = Option.builder("R").longOpt("restore").desc("restore data/filebase").hasArg().argName("path").build();
- public static final Option LOG = Option.builder("o").longOpt("out").desc("print output to file").hasArg().optionalArg(true).build();
+ public static final Option RESTORE = Option.builder("R").longOpt("restore").desc("restore data/filebase").hasArg().argName("file").build();
+ public static final Option LOG = Option.builder("o").longOpt("out").desc("print output to file").hasArg().optionalArg(true).argName("file")
+ .build();
private final Options options;
private CommandLine cmd;
@@ -52,7 +55,7 @@ public class CommandLineOptions {
public CommandLineOptions() {
final Option[] optionsArray = { ERASE, DEBUG, HELP, INDEX, LIST, REREAD, SILENT, TEST, ALL, CREATE, DELETE, EDIT, PRINT, IMPORT, MODEL,
- RENAME, CLEAR, SELECT, BACKUP, RESTORE, VERSION, LOG };
+ RENAME, CLEAR, SELECT, BACKUP, RESTORE, VERSION, PERF, LOG };
options = new Options();
for (final Option option : optionsArray)
options.addOption(option);
@@ -112,7 +115,7 @@ public class CommandLineOptions {
final String[] models = getOptionValues(INDEX);
if (models != null && models.length > 0)
return stripGroups(models);
- return selectedModels();
+ return stripGroups(selectedModels());
}
public boolean isList() {
@@ -127,7 +130,7 @@ public class CommandLineOptions {
final String[] models = getOptionValues(REREAD);
if (models != null && models.length > 0)
return stripGroups(models);
- return selectedModels();
+ return stripGroups(selectedModels());
}
public boolean isSilent() {
@@ -149,7 +152,7 @@ public class CommandLineOptions {
public String[] modelsToCreate() {
final String[] models = getOptionValues(CREATE);
if (models != null && models.length > 0)
- return stripGroups(models);
+ return models;
return selectedModels();
}
@@ -161,7 +164,7 @@ public class CommandLineOptions {
final String[] models = getOptionValues(DELETE);
if (models != null && models.length > 0)
return stripGroups(models);
- return selectedModels();
+ return stripGroups(selectedModels());
}
public boolean isEdit() {
@@ -172,7 +175,7 @@ public class CommandLineOptions {
final String[] models = getOptionValues(EDIT);
if (models != null && models.length > 0)
return stripGroups(models);
- return selectedModels();
+ return stripGroups(selectedModels());
}
public boolean isPrint() {
@@ -183,18 +186,19 @@ public class CommandLineOptions {
final String[] models = getOptionValues(PRINT);
if (models != null && models.length > 0)
return stripGroups(models);
- return selectedModels();
+ return stripGroups(selectedModels());
}
public boolean isImport() {
return hasOption(IMPORT);
}
+ public String[] modelsForImport() {
+ return stripGroups(selectedModels());
+ }
+
public String[] filesToImport() {
- final String[] models = getOptionValues(IMPORT);
- if (models != null && models.length > 0)
- return stripGroups(models);
- return selectedModels();
+ return getOptionValues(IMPORT);
}
public boolean isModel() {
@@ -205,7 +209,7 @@ public class CommandLineOptions {
final String[] models = getOptionValues(MODEL);
if (models != null && models.length > 0)
return stripGroups(models);
- return selectedModels();
+ return stripGroups(selectedModels());
}
public boolean isSelect() {
@@ -220,7 +224,7 @@ public class CommandLineOptions {
models = getOptionValues(SELECT);
if (models != null && models.length > 0)
- return stripGroups(models);
+ return models;
throw new RuntimeException("select at least one model");
}
@@ -241,6 +245,14 @@ public class CommandLineOptions {
return getOptionValue(RESTORE);
}
+ public boolean isPerf() {
+ return hasOption(PERF);
+ }
+
+ public String perfPath() {
+ return getOptionValue(PERF);
+ }
+
public boolean isVersion() {
return hasOption(VERSION);
}
@@ -253,7 +265,7 @@ public class CommandLineOptions {
final String[] models = getOptionValues(RENAME);
if (models != null && models.length > 0)
return stripGroups(models);
- return selectedModels();
+ return stripGroups(selectedModels());
}
public boolean isClear() {
@@ -264,7 +276,7 @@ public class CommandLineOptions {
final String[] models = getOptionValues(CLEAR);
if (models != null && models.length > 0)
return stripGroups(models);
- return selectedModels();
+ return stripGroups(selectedModels());
}
public boolean isLog() {
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/Main.java b/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
index 34a78f76d74b32a79285f01b17deb4937a1da64d..85315441c3f7237c709361a8f18ea07daa9b0dcf 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
@@ -28,19 +28,19 @@ import de.vipra.cmd.option.VersionCommand;
import de.vipra.util.Config;
import de.vipra.util.ConsoleUtils;
import de.vipra.util.LockFile;
+import de.vipra.util.Statistics;
import de.vipra.util.ex.LockFileException;
public class Main {
private static final LockFile lockFile = new LockFile();
+ public static Statistics stats = new Statistics();
static {
// set morphia log level
MorphiaLoggerFactory.registerLogger(SLF4JLoggerImplFactory.class);
-
// set corenlp log level, close stderr to mute corenlp messages
System.err.close();
-
}
private static void lock() {
@@ -62,6 +62,7 @@ public class Main {
}
public static void main(final String[] args) {
+
final CommandLineOptions opts = new CommandLineOptions();
try {
opts.parse(args);
@@ -76,6 +77,9 @@ public class Main {
return;
}
+ if (opts.isPerf())
+ stats.begin();
+
// logger configuration
ConsoleUtils.setSilent(opts.isSilent());
@@ -97,50 +101,54 @@ public class Main {
final List<Command> commands = new ArrayList<>();
- if (opts.isVersion())
- commands.add(new VersionCommand());
+ try {
+ if (opts.isVersion())
+ commands.add(new VersionCommand());
- if (opts.isTest())
- commands.add(new TestCommand());
+ if (opts.isTest())
+ commands.add(new TestCommand());
- if (opts.isBackup())
- commands.add(new BackupCommand(opts.backupPath()));
+ if (opts.isBackup())
+ commands.add(new BackupCommand(opts.backupPath()));
- if (opts.isRestore())
- commands.add(new RestoreCommand(opts.restorePath()));
+ if (opts.isRestore())
+ commands.add(new RestoreCommand(opts.restorePath()));
- if (opts.isErase())
- commands.add(new EraseCommand());
+ if (opts.isErase())
+ commands.add(new EraseCommand());
- if (opts.isClear())
- commands.add(new ClearModelCommand(opts.modelsToClear()));
+ if (opts.isClear())
+ commands.add(new ClearModelCommand(opts.modelsToClear()));
- if (opts.isDelete())
- commands.add(new DeleteModelCommand(opts.modelsToDelete()));
+ if (opts.isDelete())
+ commands.add(new DeleteModelCommand(opts.modelsToDelete()));
- if (opts.isCreate())
- commands.add(new CreateModelCommand(opts.modelsToCreate()));
+ if (opts.isCreate())
+ commands.add(new CreateModelCommand(opts.modelsToCreate()));
- if (opts.isRename())
- commands.add(new RenameModelCommand(opts.modelsToRename()));
+ if (opts.isRename())
+ commands.add(new RenameModelCommand(opts.modelsToRename()));
- if (opts.isList())
- commands.add(new ListModelsCommand());
+ if (opts.isList())
+ commands.add(new ListModelsCommand());
- if (opts.isEdit())
- commands.add(new EditModelCommand(opts.modelsToEdit()));
+ if (opts.isEdit())
+ commands.add(new EditModelCommand(opts.modelsToEdit()));
- if (opts.isPrint())
- commands.add(new PrintModelCommand(opts.modelsToPrint()));
+ if (opts.isPrint())
+ commands.add(new PrintModelCommand(opts.modelsToPrint()));
- if (opts.isImport())
- commands.add(new ImportCommand(opts.selectedModels(), opts.filesToImport()));
+ if (opts.isImport())
+ commands.add(new ImportCommand(opts.modelsForImport(), opts.filesToImport()));
- if (opts.isModel() || opts.isReread())
- commands.add(new ModelingCommand(opts.selectedModels(), opts.isReread()));
+ if (opts.isModel() || opts.isReread())
+ commands.add(new ModelingCommand(opts.modelsToModel(), opts.isReread()));
- if (opts.isIndex())
- commands.add(new IndexingCommand(opts.selectedModels()));
+ if (opts.isIndex())
+ commands.add(new IndexingCommand(opts.modelsToIndex()));
+ } catch (final Exception e) {
+ ConsoleUtils.error(e.getMessage());
+ }
// run
@@ -153,6 +161,7 @@ public class Main {
locked = true;
}
try {
+ Main.stats.start(c.getName());
c.run();
} catch (final Exception e) {
final Throwable cause = e.getCause();
@@ -162,6 +171,8 @@ public class Main {
ConsoleUtils.error(e.getMessage());
if (opts.isDebug() && !opts.isSilent())
e.printStackTrace(System.out);
+ } finally {
+ Main.stats.stop(c.getName());
}
}
} else {
@@ -169,6 +180,14 @@ public class Main {
}
unlock();
+
+ if (opts.isPerf()) {
+ try {
+ stats.finish(opts.perfPath());
+ } catch (final IOException e) {
+ ConsoleUtils.error("could not write statistics: " + e.getMessage());
+ }
+ }
}
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/BackupCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/BackupCommand.java
index d7981dd179a8e2206a3674952cc95ad8b1ad54a5..316fe489c492c2b702c31d95cd4a6a70cc63dcdd 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/BackupCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/BackupCommand.java
@@ -7,6 +7,7 @@ import org.fusesource.jansi.Ansi;
import org.fusesource.jansi.Ansi.Color;
import org.zeroturnaround.zip.ZipUtil;
+import de.vipra.cmd.Main;
import de.vipra.util.Config;
import de.vipra.util.ConsoleUtils;
import de.vipra.util.FileUtils;
@@ -24,27 +25,31 @@ public class BackupCommand implements Command {
@Override
public void run() throws Exception {
try {
+ Main.stats.start("backup");
final Timer timer = new Timer();
- ConsoleUtils.info("creating backup");
final Config config = Config.getConfig();
final File tmpTarget = FileUtils.getTempFile("vipra-dump");
org.apache.commons.io.FileUtils.deleteDirectory(tmpTarget);
+ Main.stats.start("backup.database");
ConsoleUtils.infoNOLF(" " + ConsoleUtils.PATH_T + " backup database...");
final Process p = Runtime.getRuntime().exec("mongodump -d " + config.getDatabaseName() + " -h " + config.getDatabaseHost() + " --port "
+ config.getDatabasePort() + " -o " + new File(tmpTarget, "db"));
p.waitFor();
ConsoleUtils.print(Ansi.ansi().fg(Color.GREEN).a("OK").reset().toString());
+ Main.stats.next("backup.database", "backup.filebase");
ConsoleUtils.infoNOLF(" " + ConsoleUtils.PATH_T + " backup filebase...");
org.apache.commons.io.FileUtils.copyDirectory(config.getDataDirectory(), new File(tmpTarget, "fb"));
ConsoleUtils.print(Ansi.ansi().fg(Color.GREEN).a("OK").reset().toString());
+ Main.stats.next("backup.filebase", "backup.configuration");
ConsoleUtils.infoNOLF(" " + ConsoleUtils.PATH_T + " backup configuration...");
org.apache.commons.io.FileUtils.copyDirectory(Config.getGenericConfigDir(), new File(tmpTarget, "config"));
ConsoleUtils.print(Ansi.ansi().fg(Color.GREEN).a("OK").reset().toString());
+ Main.stats.next("backup.configuration", "backup.compress");
ConsoleUtils.infoNOLF(" " + ConsoleUtils.PATH_L + " compressing...");
File target = new File(path);
if (target.exists() && target.isDirectory())
@@ -55,6 +60,7 @@ public class BackupCommand implements Command {
ConsoleUtils.info("saved to file: " + target.getAbsolutePath());
ConsoleUtils.info("done in " + StringUtils.timeString(timer.total()));
+ Main.stats.stop("backup.compress");
} catch (final Exception e) {
ConsoleUtils.print(Ansi.ansi().fg(Color.RED).a("FAILED").reset().toString());
if (e.getMessage().contains("mongodump")) {
@@ -71,4 +77,9 @@ public class BackupCommand implements Command {
return true;
}
+ @Override
+ public String getName() {
+ return "backup";
+ }
+
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearModelCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearModelCommand.java
index d9b8ad9122fea1d727a0f331376523b6c6e95152..a41e96975009c41fd132116e436e2d03be47ef78 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearModelCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearModelCommand.java
@@ -7,6 +7,7 @@ import java.util.List;
import org.apache.commons.io.FileUtils;
import org.bson.types.ObjectId;
+import de.vipra.cmd.Main;
import de.vipra.util.Config;
import de.vipra.util.ConsoleUtils;
import de.vipra.util.model.ArticleFull;
@@ -42,6 +43,7 @@ public class ClearModelCommand implements Command {
final List<TopicModelFull> topicModels = dbTopicModels.getMultiple(QueryBuilder.builder().in("name", Arrays.asList(names)).allFields());
for (final TopicModelFull topicModel : topicModels) {
+ Main.stats.start("clear." + topicModel.getName());
final File modelDir = new File(config.getDataDirectory(), topicModel.getName());
final QueryBuilder builder = QueryBuilder.builder().eq("topicModel", new TopicModel(topicModel.getId()));
@@ -64,6 +66,7 @@ public class ClearModelCommand implements Command {
topicModel.setWordCount(0L);
ConsoleUtils.info("model cleared: " + topicModel.getName());
+ Main.stats.stop("clear." + topicModel.getName());
}
dbTopicModels.replaceMultiple(topicModels);
@@ -74,4 +77,9 @@ public class ClearModelCommand implements Command {
return true;
}
+ @Override
+ public String getName() {
+ return "clear";
+ }
+
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/Command.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/Command.java
index 48af4869f603f45cb26506fae08562c2582ea839..49f51b7390beb12a6e346d6e51ed96e99351443c 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/Command.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/Command.java
@@ -6,4 +6,6 @@ public interface Command {
public boolean requiresLock();
+ public String getName();
+
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/CreateModelCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/CreateModelCommand.java
index ba1bc1f0d1b8ba871cb8ea5c1867b6ead423a78d..14df7a19542cedd76f590774ee1757e7efd361db 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/CreateModelCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/CreateModelCommand.java
@@ -9,6 +9,7 @@ import org.bson.types.ObjectId;
import com.fasterxml.jackson.core.JsonGenerationException;
import com.fasterxml.jackson.databind.JsonMappingException;
+import de.vipra.cmd.Main;
import de.vipra.util.Config;
import de.vipra.util.ConsoleUtils;
import de.vipra.util.ex.DatabaseException;
@@ -57,6 +58,7 @@ public class CreateModelCommand implements Command {
}
for (final String name : names) {
+ Main.stats.start("create." + name);
final String msg = config.validModelName(name);
if (msg != null)
throw new Exception(msg);
@@ -69,6 +71,7 @@ public class CreateModelCommand implements Command {
final TopicModelFull topicModel = createModel(name, modelConfig, modelDir);
ConsoleUtils
.info("model created: " + topicModel.getName() + (topicModel.getGroup() == null ? "" : " in group: " + topicModel.getGroup()));
+ Main.stats.stop("create." + name);
}
}
@@ -77,4 +80,9 @@ public class CreateModelCommand implements Command {
return false;
}
+ @Override
+ public String getName() {
+ return "create";
+ }
+
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteModelCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteModelCommand.java
index 0d33b269c5c126e69c14915d7633789194bd9bd2..c958ce9d9085ba3c1781c7b5b5c3d24844ca39fe 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteModelCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteModelCommand.java
@@ -7,6 +7,7 @@ import java.util.stream.Collectors;
import org.bson.types.ObjectId;
+import de.vipra.cmd.Main;
import de.vipra.util.Config;
import de.vipra.util.ConsoleUtils;
import de.vipra.util.model.ArticleFull;
@@ -42,6 +43,7 @@ public class DeleteModelCommand implements Command {
final List<TopicModelFull> topicModels = dbTopicModels.getMultiple(QueryBuilder.builder().in("name", Arrays.asList(names)));
for (final TopicModelFull topicModel : topicModels) {
+ Main.stats.start("delete." + topicModel.getName());
final File modelDir = new File(config.getDataDirectory(), topicModel.getName());
final QueryBuilder builder = QueryBuilder.builder().eq("topicModel", new TopicModel(topicModel.getId()));
@@ -66,6 +68,7 @@ public class DeleteModelCommand implements Command {
if (deleted > 0)
ConsoleUtils.info("model deleted: " + topicModel.getName());
+ Main.stats.stop("delete." + topicModel.getName());
}
dbTopicModels.deleteMultiple(topicModels.stream().map((t) -> t.getId()).collect(Collectors.toList()));
@@ -76,4 +79,9 @@ public class DeleteModelCommand implements Command {
return true;
}
+ @Override
+ public String getName() {
+ return "delete";
+ }
+
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/EditModelCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/EditModelCommand.java
index 600899af46f9373c7d12aaf617b3976394eb47c3..2cb8e1bc21ddca4df208409ef41cf2df571b83d7 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/EditModelCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/EditModelCommand.java
@@ -9,6 +9,7 @@ import org.bson.types.ObjectId;
import com.fasterxml.jackson.core.JsonGenerationException;
import com.fasterxml.jackson.databind.JsonMappingException;
+import de.vipra.cmd.Main;
import de.vipra.util.Config;
import de.vipra.util.ConsoleUtils;
import de.vipra.util.Constants.WindowResolution;
@@ -31,6 +32,7 @@ public class EditModelCommand implements Command {
private void editModel(final TopicModelFull topicModel)
throws DatabaseException, JsonGenerationException, JsonMappingException, IOException, ConfigException {
+ Main.stats.start("edit." + topicModel.getName());
ConsoleUtils.info("editing model: " + topicModel.getName());
final TopicModelConfig topicModelConfig = topicModel.getModelConfig();
@@ -79,6 +81,7 @@ public class EditModelCommand implements Command {
dbTopicModels.updateSingle(topicModel, "modelConfig");
topicModelConfig.saveToFile(topicModelConfig.getModelDir(config.getDataDirectory()));
config.setTopicModelConfig(topicModelConfig);
+ Main.stats.stop("edit." + topicModel.getName());
}
@Override
@@ -98,4 +101,9 @@ public class EditModelCommand implements Command {
return true;
}
+ @Override
+ public String getName() {
+ return "edit";
+ }
+
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/EraseCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/EraseCommand.java
index edde5a1492331d838f408c9e5a2cec98f934d1a9..8327a409a3ddb890aee3828e0c3259b37d4489a7 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/EraseCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/EraseCommand.java
@@ -48,4 +48,9 @@ public class EraseCommand implements Command {
return true;
}
+ @Override
+ public String getName() {
+ return "erase";
+ }
+
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
index 825364be0e87816a8c48fcba57e5a77fa113e79f..427a830610a7b097ffb4a9a539a5a2627426f3d3 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
@@ -8,6 +8,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
@@ -17,6 +18,7 @@ import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
+import de.vipra.cmd.Main;
import de.vipra.cmd.file.Filebase;
import de.vipra.cmd.file.FilebaseException;
import de.vipra.cmd.file.FilebaseWindowIndex;
@@ -228,12 +230,21 @@ public class ImportCommand implements Command {
}
}
+ // insert word links
for (final ArticleWord word : processedText.getArticleWords()) {
if (blockedWords.contains(word.getWord()))
continue;
articleText = articleText.replaceAll("(?i)\\b(" + word.getWord() + ")\\b(?![^<]*>|[^<>]*</)", word.aTag("$1"));
}
+ // insert more word links
+ for (final Map.Entry<String, String> entry : processedText.getLemmas().entrySet()) {
+ if (blockedWords.contains(entry.getKey().toLowerCase()))
+ continue;
+ articleText = articleText.replaceAll("(?i)\\b(" + entry.getKey() + ")\\b(?![^<]*>|[^<>]*</)",
+ ArticleWord.aTag(entry.getValue(), "$1"));
+ }
+
article.setProcessedText(processedText.getWords());
article.setWords(articleWords);
article.setTopicModel(new TopicModel(topicModelFull.getId()));
@@ -306,6 +317,7 @@ public class ImportCommand implements Command {
}
private void importForModel(final TopicModelConfig modelConfig) throws Exception {
+ Main.stats.start("import." + modelConfig.getName());
ConsoleUtils.info("importing for model: " + modelConfig.getName());
this.modelConfig = modelConfig;
@@ -391,6 +403,7 @@ public class ImportCommand implements Command {
* run information
*/
ConsoleUtils.info("done in " + StringUtils.timeString(timer.total()));
+ Main.stats.stop("import." + modelConfig.getName());
}
@Override
@@ -411,4 +424,9 @@ public class ImportCommand implements Command {
return true;
}
+ @Override
+ public String getName() {
+ return "import";
+ }
+
}
\ No newline at end of file
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/IndexingCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/IndexingCommand.java
index f929eb0c538b117b104e4562cf1ba2f5c2354402..92710d5a4015ee732500c5344a8ba1854f718e41 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/IndexingCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/IndexingCommand.java
@@ -9,6 +9,7 @@ import org.bson.types.ObjectId;
import org.elasticsearch.client.Client;
import org.elasticsearch.index.IndexNotFoundException;
+import de.vipra.cmd.Main;
import de.vipra.cmd.file.FilebaseIDDateIndex;
import de.vipra.cmd.file.FilebaseIDDateIndexEntry;
import de.vipra.util.Config;
@@ -40,6 +41,7 @@ public class IndexingCommand implements Command {
}
private void indexForModel(final TopicModelConfig modelConfig) throws ParseException, IOException, ConfigException, DatabaseException {
+ Main.stats.start("indexing." + modelConfig.getName());
ConsoleUtils.info("indexing for model: " + modelConfig.getName());
final Timer timer = new Timer();
@@ -79,6 +81,7 @@ public class IndexingCommand implements Command {
// run information
ConsoleUtils.info("done in " + StringUtils.timeString(timer.total()));
+ Main.stats.stop("indexing." + modelConfig.getName());
}
@Override
@@ -98,4 +101,9 @@ public class IndexingCommand implements Command {
return true;
}
+ @Override
+ public String getName() {
+ return "indexing";
+ }
+
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ListModelsCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ListModelsCommand.java
index 4852d0e6f5a0c22333def610f9cf3ff1217b2373..c899a979326d816daff6b3eba6a76486cdabecdc 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ListModelsCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ListModelsCommand.java
@@ -28,4 +28,9 @@ public class ListModelsCommand implements Command {
return false;
}
+ @Override
+ public String getName() {
+ return "list";
+ }
+
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ModelingCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ModelingCommand.java
index 8340d85fcddd1fa30f53e97bce23ffa953851bb0..f2c95e6397c907be577f27ad35d6737d2342ceb2 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ModelingCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ModelingCommand.java
@@ -5,6 +5,7 @@ import java.io.IOException;
import java.text.ParseException;
import java.util.function.Consumer;
+import de.vipra.cmd.Main;
import de.vipra.cmd.lda.Analyzer;
import de.vipra.cmd.lda.AnalyzerException;
import de.vipra.util.Config;
@@ -27,6 +28,7 @@ public class ModelingCommand implements Command {
private void modelForModel(final TopicModelConfig modelConfig, final Consumer<Double> progressCallback)
throws AnalyzerException, ConfigException, DatabaseException, ParseException, IOException, InterruptedException {
+ Main.stats.start("modeling." + modelConfig.getName());
if (reread)
ConsoleUtils.info("rereading model: " + modelConfig.getName());
else
@@ -48,6 +50,7 @@ public class ModelingCommand implements Command {
* run information
*/
ConsoleUtils.info("done in " + StringUtils.timeString(timer.total()));
+ Main.stats.stop("modeling." + modelConfig.getName());
}
@Override
@@ -67,4 +70,9 @@ public class ModelingCommand implements Command {
return true;
}
+ @Override
+ public String getName() {
+ return "modeling";
+ }
+
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/PrintModelCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/PrintModelCommand.java
index b335c621ad4b032dac2558134fe47bf5d190ea90..1c255c8ad0d16152f707ab77846f35e17d45a6bb 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/PrintModelCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/PrintModelCommand.java
@@ -31,4 +31,9 @@ public class PrintModelCommand implements Command {
return false;
}
+ @Override
+ public String getName() {
+ return "print";
+ }
+
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/RenameModelCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/RenameModelCommand.java
index dac38bc986bbf12ab5271c0c540e5288d4dc3b6d..231c584f96c3219845ff1fd47b713285f0f73a7c 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/RenameModelCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/RenameModelCommand.java
@@ -66,4 +66,9 @@ public class RenameModelCommand implements Command {
return false;
}
+ @Override
+ public String getName() {
+ return "rename";
+ }
+
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/RestoreCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/RestoreCommand.java
index 20371031510ae095e9609ea27f5785a6d02ab5eb..d21276237095992b095226d563cccf962f2b42bc 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/RestoreCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/RestoreCommand.java
@@ -7,6 +7,7 @@ import org.fusesource.jansi.Ansi;
import org.fusesource.jansi.Ansi.Color;
import org.zeroturnaround.zip.ZipUtil;
+import de.vipra.cmd.Main;
import de.vipra.util.Config;
import de.vipra.util.ConsoleUtils;
import de.vipra.util.FileUtils;
@@ -27,6 +28,7 @@ public class RestoreCommand implements Command {
final Timer timer = new Timer();
ConsoleUtils.info("restoring from file");
+ Main.stats.start("restore.unzip");
final File zip = new File(path);
if (!zip.isFile())
throw new FileNotFoundException(path);
@@ -34,20 +36,24 @@ public class RestoreCommand implements Command {
ZipUtil.unpack(zip, tmpTarget);
final Config config = Config.getConfig();
+ Main.stats.next("restore.unzip", "restore.database");
ConsoleUtils.infoNOLF(" " + ConsoleUtils.PATH_T + " restore database...");
final Process p = Runtime.getRuntime().exec(
"mongorestore --drop -h " + config.getDatabaseHost() + " --port " + config.getDatabasePort() + " " + new File(tmpTarget, "db"));
p.waitFor();
ConsoleUtils.print(Ansi.ansi().fg(Color.GREEN).a("OK").reset().toString());
+ Main.stats.next("restore.database", "restore.filebase");
ConsoleUtils.infoNOLF(" " + ConsoleUtils.PATH_T + " restore filebase...");
org.apache.commons.io.FileUtils.copyDirectory(new File(tmpTarget, "fb"), config.getDataDirectory());
ConsoleUtils.print(Ansi.ansi().fg(Color.GREEN).a("OK").reset().toString());
+ Main.stats.next("restore.filebase", "restore.configuration");
ConsoleUtils.infoNOLF(" " + ConsoleUtils.PATH_L + " restore configuration...");
org.apache.commons.io.FileUtils.copyDirectory(new File(tmpTarget, "config"), Config.getGenericConfigDir());
ConsoleUtils.print(Ansi.ansi().fg(Color.GREEN).a("OK").reset().toString());
+ Main.stats.stop("restore.configuration");
org.apache.commons.io.FileUtils.deleteDirectory(tmpTarget);
ConsoleUtils.info("restored");
@@ -68,4 +74,9 @@ public class RestoreCommand implements Command {
return true;
}
+ @Override
+ public String getName() {
+ return "restore";
+ }
+
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/TestCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/TestCommand.java
index 04fef5075bc43608f5a4a93ed94c0ef69d22394b..14455611bf62a0d4dbd036f4a77137a5eaef144e 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/TestCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/TestCommand.java
@@ -11,6 +11,7 @@ import org.elasticsearch.client.transport.TransportClient;
import org.fusesource.jansi.Ansi;
import org.fusesource.jansi.Ansi.Color;
+import de.vipra.cmd.Main;
import de.vipra.util.Config;
import de.vipra.util.ConsoleUtils;
import de.vipra.util.ESClient;
@@ -22,6 +23,7 @@ public class TestCommand implements Command {
@Override
public void run() throws Exception {
try {
+ Main.stats.start("test command");
ConsoleUtils.info("testing system");
// test if configuration readable
@@ -67,6 +69,8 @@ public class TestCommand implements Command {
} catch (final Exception e) {
ConsoleUtils.print(Ansi.ansi().fg(Color.RED).a("FAILED").reset().toString());
throw e;
+ } finally {
+ Main.stats.stop("test command");
}
}
@@ -75,4 +79,9 @@ public class TestCommand implements Command {
return false;
}
+ @Override
+ public String getName() {
+ return "test";
+ }
+
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/VersionCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/VersionCommand.java
index 44bddcac6928d0a977f358910987ff8f6f50d638..7286d5c93b2561f694e4792a62801f2468341057 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/VersionCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/VersionCommand.java
@@ -26,4 +26,9 @@ public class VersionCommand implements Command {
return false;
}
+ @Override
+ public String getName() {
+ return "version";
+ }
+
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/text/ProcessedText.java b/vipra-cmd/src/main/java/de/vipra/cmd/text/ProcessedText.java
index 8646b9c97428855bbc1a7ee23e35f20bca619154..d18bc7b4405a721d85f2fa5a731347235b07b8b8 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/text/ProcessedText.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/text/ProcessedText.java
@@ -4,6 +4,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
+import java.util.Map;
import java.util.Map.Entry;
import de.vipra.util.CountMap;
@@ -17,8 +18,9 @@ public class ProcessedText {
private final double reductionRatio;
private final CountMap<String> wordCounts;
private final List<ArticleWord> articleWords;
+ private final Map<String, String> lemmas;
- public ProcessedText(final String text, final long wordCount) {
+ public ProcessedText(final String text, final long wordCount, final Map<String, String> lemmas) {
final String[] allWords = text.toLowerCase().trim().split("\\s+");
final List<String> wordList = new ArrayList<>(allWords.length);
for (final String word : allWords)
@@ -28,6 +30,7 @@ public class ProcessedText {
originalWordCount = wordCount;
reducedWordCount = words.length;
reductionRatio = 1 - ((double) reducedWordCount / wordCount);
+ this.lemmas = lemmas;
wordCounts = new CountMap<>();
for (final String word : words)
@@ -63,4 +66,8 @@ public class ProcessedText {
return wordCounts;
}
+ public Map<String, String> getLemmas() {
+ return lemmas;
+ }
+
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/text/Processor.java b/vipra-cmd/src/main/java/de/vipra/cmd/text/Processor.java
index 41723aa8e8ea1bd3b9731435a4460ee54cad0e1f..115502257cfa7234b601f4bac38845d33ea36dc1 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/text/Processor.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/text/Processor.java
@@ -1,6 +1,8 @@
package de.vipra.cmd.text;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import de.vipra.util.Constants;
@@ -35,9 +37,10 @@ public class Processor {
}
public ProcessedText process(final TopicModelConfig modelConfig, final String input) throws ProcessorException {
- final Annotation doc = new Annotation(input.toLowerCase());
+ final Annotation doc = new Annotation(input);
nlp.annotate(doc);
final StringBuilder sb = new StringBuilder();
+ final Map<String, String> lemmas = new HashMap<>();
long wordCount = 0;
// loop sentences
for (final CoreMap sentence : doc.get(SentencesAnnotation.class)) {
@@ -53,15 +56,20 @@ public class Processor {
final Long count = word.get(FrequencyAnnotator.class);
if (count != null && count >= modelConfig.getDocumentMinimumWordFrequency()) {
final String lemma = word.get(LemmaAnnotation.class);
- // collect unique words
- sb.append(lemma).append(" ");
+ if (lemma != null) {
+ // collect unique words
+ sb.append(lemma.toLowerCase()).append(" ");
+
+ if (!lemma.equals(word.word()))
+ lemmas.put(word.word(), lemma);
+ }
}
}
}
}
final String text = clean(sb.toString());
- return new ProcessedText(text, wordCount);
+ return new ProcessedText(text, wordCount, lemmas);
}
public static String clean(final String in) {
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/text/StopwordsAnnotator.java b/vipra-cmd/src/main/java/de/vipra/cmd/text/StopwordsAnnotator.java
index b977de033e3a2a91e5b24d0993f1e359158c07f9..2a4c19856e5bc291688548246e6bdd659753aa25 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/text/StopwordsAnnotator.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/text/StopwordsAnnotator.java
@@ -20,7 +20,7 @@ public class StopwordsAnnotator implements Annotator, CoreAnnotation<Boolean> {
private final Set<String> stopWords;
public StopwordsAnnotator(final String input, final Properties props) {
- stopWords = new HashSet<String>(Arrays.asList(props.getProperty(NAME).split(" ")));
+ stopWords = new HashSet<>(Arrays.asList(props.getProperty(NAME).split(" ")));
stopWords.addAll(Arrays.asList("-LRB-", "-RRB-", "-LSB-", "-RSB-", "-LCB-", "-RCB-"));
}
diff --git a/vipra-ui/app/html/articles/show.html b/vipra-ui/app/html/articles/show.html
index 1bce4b665b89a6a7389d11c54401bb5f49abf632..26513b5e6b84e84b9009ba867cb8b2e1d7e4b26d 100644
--- a/vipra-ui/app/html/articles/show.html
+++ b/vipra-ui/app/html/articles/show.html
@@ -99,7 +99,13 @@
</table>
<p class="text-muted" ng-hide="article.similarArticles.length">No similar articles</p>
<hr>
- <div class="text-justify" ng-bind-html="::article.text"></div>
+ <div class="pull-right hide-links-button" ng-click="articlesShowModels.hideLinks=!articlesShowModels.hideLinks">
+ <span class="fa-stack">
+ <i class="fa fa-link fa-stack-1x"></i>
+ <i class="fa fa-ban fa-stack-2x text-danger" ng-show="!articlesShowModels.hideLinks"></i>
+ </span>
+ </div>
+ <div class="text-justify" ng-bind-html="::article.text" ng-class="{'hide-links':articlesShowModels.hideLinks}"></div>
</div>
</div>
<div class="loading" ng-hide="article">Loading...</div>
diff --git a/vipra-ui/app/html/directives/topicmodel-button.html b/vipra-ui/app/html/directives/topicmodel-button.html
new file mode 100644
index 0000000000000000000000000000000000000000..9e45cceb1cd49b22b12ebd0df7525dbfbda246f5
--- /dev/null
+++ b/vipra-ui/app/html/directives/topicmodel-button.html
@@ -0,0 +1,10 @@
+<button type="button" class="list-group-item topic-model" ng-click="changeTopicModel(topicModel)" ng-class="{'active selected-model':rootModels.topicModel.id===topicModel.id}" analytics-on analytics-event="Topic model" analytics-category="Topic model actions">
+ <span class="badge badge-group">
+ <span class="badge-part" ng-if="!topicModel.lastGenerated" title="Model was never generated">Non-generated</span>
+ <span class="badge-part" ng-bind="::topicModel.articleCount" ng-show="topicModel.articleCount" ng-attr-title="{{::topicModel.articleCount + ' article(s)'}}" ng-cloak></span>
+ <span class="badge-part" ng-bind="::topicModel.topicCount" ng-show="topicModel.topicCount" ng-attr-title="{{::topicModel.topicCount + ' topic(s)'}}" ng-cloak></span>
+ </span>
+ <span ng-bind="::topicModel.name"></span>
+ <br ng-show="topicModel.modelConfig.description" ng-cloak>
+ <small ng-bind="::topicModel.modelConfig.description"></small>
+</button>
\ No newline at end of file
diff --git a/vipra-ui/app/index.html b/vipra-ui/app/index.html
index 5286a53a7f863c41e0e197d48072377d8607a77f..91122d3fce881df0f9a27164fe9a719fc379df51 100644
--- a/vipra-ui/app/index.html
+++ b/vipra-ui/app/index.html
@@ -93,22 +93,26 @@
<input tabindex="0" type="text" class="form-control" placeholder="Filter..." ng-model="rootModels.topicModelFilter">
<i class="form-control-feedback glyphicon glyphicon-search text-muted"></i>
</div>
- <div class="list-group nomargin" ng-show="topicModels.length" ng-cloak>
- <button type="button" class="list-group-item topic-model" ng-repeat="topicModel in topicModels | filter:{name:rootModels.topicModelFilter}" ng-click="changeTopicModel(topicModel)" ng-class="{'active selected-model':rootModels.topicModel.id===topicModel.id}" analytics-on analytics-event="Topic model" analytics-category="Topic model actions">
- <span class="badge badge-group">
- <span class="badge-part" ng-if="!topicModel.lastGenerated" title="Model was never generated">Non-generated</span>
- <span class="badge-part" ng-bind="::topicModel.articleCount" ng-show="topicModel.articleCount" ng-attr-title="{{::topicModel.articleCount + ' article(s)'}}" ng-cloak></span>
- <span class="badge-part" ng-bind="::topicModel.topicCount" ng-show="topicModel.topicCount" ng-attr-title="{{::topicModel.topicCount + ' topic(s)'}}" ng-cloak></span>
- </span>
- <span ng-bind="::topicModel.name"></span>
- <br ng-show="topicModel.modelConfig.description" ng-cloak>
- <small ng-bind="::topicModel.modelConfig.description"></small>
- </button>
+ <div class="panel panel-default" ng-repeat="(groupName, group) in groups" ng-show="filtered.length">
+ <div class="panel-heading pointer" ng-click="group.collapsed=!group.collapsed">
+ <i class="fa folder-icon" ng-class="{'fa-folder-open-o':!group.collapsed,'fa-folder-o':group.collapsed}"></i>
+ <span ng-bind="::groupName"></span>
+ <div class="pull-right">
+ <span class="badge" ng-bind="::group.length" ng-attr-title="{{::group.length}} topic model(s)"></span>
+ <i class="fa fa-chevron-down pointer"></i>
+ </div>
+ </div>
+ <div ng-attr-id="{{::groupName}}" class="list-group nomargin collapse in" ng-show="topicModelsCount && !group.collapsed" ng-cloak>
+ <topicmodel-button ng-repeat="topicModel in filtered = (group | filter:{name:rootModels.topicModelFilter})"></topicmodel-button>
+ </div>
+ </div>
+ <div class="list-group nomargin" ng-show="topicModelsCount" ng-cloak>
+ <topicmodel-button ng-repeat="topicModel in topicModels | filter:{name:rootModels.topicModelFilter}"></topicmodel-button>
</div>
<p class="text-center" ng-show="loading.any" ng-cloak>
Loading...
</p>
- <p ng-hide="topicModels.length || loading.any">
+ <p ng-hide="topicModelsCount || loading.any">
No topic models in the database. Create a topic model and import data into it to begin.
</p>
</div>
diff --git a/vipra-ui/app/js/controllers.js b/vipra-ui/app/js/controllers.js
index b39b4bed23639b979c5ab5be1bb08e88ec07976e..372fae3248284d971ea90f42fc991e9f9bc098c6 100644
--- a/vipra-ui/app/js/controllers.js
+++ b/vipra-ui/app/js/controllers.js
@@ -50,11 +50,23 @@
TopicModelFactory.query({
fields: '_all'
}, function(data) {
+ var groups = {};
+ var models = [];
for(var i = 0; i < data.length; i++) {
data[i].lastGeneratedString = data[i].lastGenerated ? Vipra.formatDateTime(data[i].lastGenerated) : 'never';
data[i].lastIndexedString = data[i].lastIndexed ? Vipra.formatDateTime(data[i].lastIndexed) : 'never';
+ if(data[i].group) {
+ if(!groups.hasOwnProperty(data[i].group))
+ groups[data[i].group] = [data[i]];
+ else
+ groups[data[i].group].push(data[i]);
+ } else {
+ models.push(data[i]);
+ }
}
- $scope.topicModels = data;
+ $scope.groups = groups;
+ $scope.topicModels = models;
+ $scope.topicModelsCount = data.length;
});
};
@@ -1256,7 +1268,7 @@
$scope.prepareText = function(text) {
var entityBase = $state.href('entities', {}, {absolute: true}),
wordBase = $state.href('words', {}, {absolute: true});
- return text.replace(/data-entity="([^"]*)"/g, 'href="' + entityBase + '/$1"').replace(/data-word="([^"]*)"/g, 'href="' + wordBase + '/$1"');
+ return text.replace(/<e=([^>]*)/g, '<a href="' + entityBase + '/$1"').replace(/<w=([^>]*)/g, '<a href="' + wordBase + '/$1"');
};
var topicShareChartElement = $('#topic-share');
diff --git a/vipra-ui/app/js/directives.js b/vipra-ui/app/js/directives.js
index 2c59a6159418c53bf263934cd98bf88ae50dd850..9370e889bfc8a14f891c9991d7759a46a2f14ba4 100644
--- a/vipra-ui/app/js/directives.js
+++ b/vipra-ui/app/js/directives.js
@@ -571,16 +571,10 @@
};
}]);
- app.directive('compileHtml', ['$compile', function($compile) {
+ app.directive('topicmodelButton', [function() {
return {
- link: function($scope, $elem, $attrs) {
- $scope.$watch(function () {
- return $scope.$eval($attrs.compileHtml);
- }, function (value) {
- $elem.html(value);
- $compile($elem.contents())($scope);
- });
- }
+ replace: true,
+ templateUrl: 'html/directives/topicmodel-button.html'
};
}]);
diff --git a/vipra-ui/app/less/app.less b/vipra-ui/app/less/app.less
index f767b795d3a85673a13b194f99c255043470f783..f61828b4c53c6646bfa92dd7daf398dba28d0e47 100644
--- a/vipra-ui/app/less/app.less
+++ b/vipra-ui/app/less/app.less
@@ -951,6 +951,7 @@ entity-menu {
.article-dropdown {
position: relative;
+ padding-left: 10px;
}
.topic-badge {
@@ -1065,6 +1066,25 @@ entity-menu {
display: inline-block;
}
+.folder-icon {
+ width: 15px;
+}
+
+.hide-links {
+ a {
+ color: #333;
+ }
+}
+
+.hide-links-button {
+ cursor: pointer;
+ margin-top: -34px;
+ border: 1px solid #eee;
+ border-radius: 5px;
+ background: #fff;
+ padding: 0 5px;
+}
+
[ng\:cloak], [ng-cloak], .ng-cloak {
display: none !important;
}
diff --git a/vipra-util/src/main/java/de/vipra/util/Config.java b/vipra-util/src/main/java/de/vipra/util/Config.java
index a5aec3b052e7b4dafb5c89d53ad5288927c2f3d6..7c7fe7e08bce278382310456ef247c9a27ab07c7 100644
--- a/vipra-util/src/main/java/de/vipra/util/Config.java
+++ b/vipra-util/src/main/java/de/vipra/util/Config.java
@@ -203,13 +203,24 @@ public class Config {
return new File(base, Constants.FILEBASE_DIR);
}
+ public static File getGenericStatsDir() {
+ final File dir = new File(getGenericLogDir(), "vipra-stats");
+ if (!dir.exists())
+ dir.mkdirs();
+ return dir;
+ }
+
+ public static File getGenericLogDir() {
+ return PathUtils.tempDir();
+ }
+
/**
- * Returns a generic logging directory
+ * Returns a generic logging file
*
- * @return generic logging directory
+ * @return generic logging file
*/
public static File getGenericLogFile() {
- final File base = PathUtils.tempDir();
+ final File base = getGenericLogDir();
return new File(base, Constants.LOG_FILE);
}
diff --git a/vipra-util/src/main/java/de/vipra/util/MathUtils.java b/vipra-util/src/main/java/de/vipra/util/MathUtils.java
new file mode 100644
index 0000000000000000000000000000000000000000..5c2eecdff8dda9de81f3401f5ac6c59ac462dd3b
--- /dev/null
+++ b/vipra-util/src/main/java/de/vipra/util/MathUtils.java
@@ -0,0 +1,44 @@
+package de.vipra.util;
+
+import java.util.Collection;
+import java.util.List;
+
+public class MathUtils {
+
+ public static Double mean(final Collection<Long> numbers) {
+ double sum = 0;
+ for (final Long l : numbers)
+ sum += l;
+ return sum / numbers.size();
+ }
+
+ public static Double median(final List<Long> numbers) {
+ final int size = numbers.size();
+ final int middle = size / 2;
+ if (size % 2 == 1) {
+ return (double) numbers.get(middle);
+ } else {
+ return (numbers.get(middle - 1) + numbers.get(middle)) / 2.0;
+ }
+ }
+
+ public static long mode(final List<Long> numbers) {
+ long maxValue = 0;
+ long maxCount = 0;
+
+ for (int i = 0; i < numbers.size(); ++i) {
+ int count = 0;
+ for (int j = 0; j < numbers.size(); ++j) {
+ if (numbers.get(j) == numbers.get(i))
+ ++count;
+ }
+ if (count > maxCount) {
+ maxCount = count;
+ maxValue = numbers.get(i);
+ }
+ }
+
+ return maxValue;
+ }
+
+}
diff --git a/vipra-util/src/main/java/de/vipra/util/MongoUtils.java b/vipra-util/src/main/java/de/vipra/util/MongoUtils.java
index ad076611f5de8385c88541cba323f103d8a72e19..6aa2984a8e45db5c861560fddc7ec73ca31253c3 100644
--- a/vipra-util/src/main/java/de/vipra/util/MongoUtils.java
+++ b/vipra-util/src/main/java/de/vipra/util/MongoUtils.java
@@ -15,7 +15,7 @@ public class MongoUtils {
if (sortBy == null)
return null;
final String[] sortKeys = sortBy.split(",");
- final ArrayList<Bson> sorts = new ArrayList<Bson>(sortKeys.length);
+ final ArrayList<Bson> sorts = new ArrayList<>(sortKeys.length);
for (String sort : sortKeys) {
if (sort.startsWith("-")) {
sorts.add(descending(sort.substring(1)));
diff --git a/vipra-util/src/main/java/de/vipra/util/MultiMap.java b/vipra-util/src/main/java/de/vipra/util/MultiMap.java
index 8d677f4471a587df88e4b1b45640aa3c5341f62a..4a4e11ccc8972241c790ef6ae781da1563d7371d 100644
--- a/vipra-util/src/main/java/de/vipra/util/MultiMap.java
+++ b/vipra-util/src/main/java/de/vipra/util/MultiMap.java
@@ -1,41 +1,57 @@
package de.vipra.util;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.function.Supplier;
public class MultiMap<K, V> {
- private final Map<K, Set<V>> map;
+ private final Map<K, Collection<V>> map;
+ private final Supplier<Collection<V>> supplier;
public MultiMap() {
- this.map = new HashMap<K, Set<V>>();
+ this(false);
+ }
+
+ public MultiMap(final boolean duplicates) {
+ if (duplicates)
+ supplier = () -> new ArrayList<>();
+ else
+ supplier = () -> new HashSet<>();
+ this.map = new HashMap<>();
+ }
+
+ public MultiMap(final Supplier<Collection<V>> supplier) {
+ this.supplier = supplier;
+ this.map = new HashMap<>();
}
public void put(final K key, final V value) {
- Set<V> set = map.get(key);
- if (set == null)
- set = new HashSet<>();
- set.add(value);
- map.put(key, set);
+ Collection<V> c = map.get(key);
+ if (c == null)
+ c = supplier.get();
+ c.add(value);
+ map.put(key, c);
}
public void put(final K key, final Collection<V> values) {
- Set<V> set = map.get(key);
- if (set == null)
- set = new HashSet<>();
- set.addAll(values);
- map.put(key, set);
+ Collection<V> c = map.get(key);
+ if (c == null)
+ c = supplier.get();
+ c.addAll(values);
+ map.put(key, c);
}
- public Set<V> get(final K key) {
+ public Collection<V> get(final K key) {
return map.get(key);
}
- public Set<Entry<K, Set<V>>> entrySet() {
+ public Set<Entry<K, Collection<V>>> entrySet() {
return map.entrySet();
}
diff --git a/vipra-util/src/main/java/de/vipra/util/Statistics.java b/vipra-util/src/main/java/de/vipra/util/Statistics.java
new file mode 100644
index 0000000000000000000000000000000000000000..064420cdbfada6a1215c645dd60d29cfd11c2548
--- /dev/null
+++ b/vipra-util/src/main/java/de/vipra/util/Statistics.java
@@ -0,0 +1,174 @@
+package de.vipra.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.databind.JsonMappingException;
+
+public class Statistics {
+
+ public static class RunInfo {
+ private final String name;
+ private final List<Long> times;
+ private final Double mean;
+ private final Double median;
+ private final Long mode;
+
+ public RunInfo(final String name, final Collection<Long> times) {
+ this.name = name;
+ this.times = new ArrayList<>(times.size());
+ try {
+ for (final Long time : times)
+ this.times.add(time);
+ } catch (final Exception e) {
+ e.printStackTrace();
+ }
+
+ mean = MathUtils.mean(times);
+ median = MathUtils.median(this.times);
+ mode = MathUtils.mode(this.times);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public List<Long> getTimes() {
+ return times;
+ }
+
+ public Double getMean() {
+ return mean;
+ }
+
+ public Double getMedian() {
+ return median;
+ }
+
+ public Long getMode() {
+ return mode;
+ }
+ }
+
+ private long startTime;
+ private long stopTime;
+ private long runTime;
+ private List<RunInfo> runInfo;
+
+ @JsonIgnore
+ private final MultiMap<String, Long> times = new MultiMap<>(true);
+
+ @JsonIgnore
+ private final Map<String, Long> running = new HashMap<>();
+
+ public Statistics() {
+ begin();
+ }
+
+ public String name() {
+ return "run-" + startTime;
+ }
+
+ public void begin() {
+ startTime = System.nanoTime();
+ }
+
+ public void finish(final String path) throws JsonGenerationException, JsonMappingException, IOException {
+ stopAll();
+ stopTime = System.nanoTime();
+ runTime = stopTime - startTime;
+ generateStats();
+ save(path);
+ }
+
+ public void generateStats() {
+ runInfo = new ArrayList<>();
+ for (final Map.Entry<String, Collection<Long>> entry : times.entrySet()) {
+ runInfo.add(new RunInfo(entry.getKey(), entry.getValue()));
+ }
+ }
+
+ public void save(final String path) throws JsonGenerationException, JsonMappingException, IOException {
+ File target;
+ if (path != null && !path.isEmpty()) {
+ final File file = new File(path);
+ if (file.exists()) {
+ if (file.isDirectory())
+ target = new File(file, name() + ".json");
+ else
+ target = file;
+ } else {
+ target = file;
+ }
+ } else {
+ target = new File(Config.getGenericStatsDir(), name() + ".json");
+ }
+ Config.mapper.writeValue(target, this);
+ }
+
+ public void start(final String name) {
+ running.put(name, System.nanoTime());
+ }
+
+ public void stop(final String name) {
+ final long curr = System.nanoTime();
+ final Long start = running.remove(name);
+ if (start != null) {
+ times.put(name, curr - start);
+ }
+ }
+
+ public void stopAll() {
+ final long curr = System.nanoTime();
+ for (Map.Entry<String, Long> run : running.entrySet())
+ times.put(run.getKey(), curr - run.getValue());
+ running.clear();
+ }
+
+ public void stop(final String... names) {
+ final long curr = System.nanoTime();
+ for (final String name : names) {
+ final Long v = running.remove(name);
+ if (v != null)
+ times.put(name, curr - v);
+ }
+ }
+
+ public void next(final String stopName, final String startName) {
+ final long curr = System.nanoTime();
+ final Long start = running.remove(stopName);
+ if (start != null) {
+ times.put(stopName, curr - start);
+ }
+
+ running.put(startName, curr);
+ }
+
+ /*
+ * getters
+ */
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public long getStopTime() {
+ return stopTime;
+ }
+
+ public long getRunTime() {
+ return runTime;
+ }
+
+ public List<RunInfo> getRunInfo() {
+ return runInfo;
+ }
+
+}
diff --git a/vipra-util/src/main/java/de/vipra/util/StringUtils.java b/vipra-util/src/main/java/de/vipra/util/StringUtils.java
index f215a12a290c73d7ad80ce7bfd9cf46c5fed281c..ba5db5acdc453b832751cdf447d40acf8c2dedfb 100644
--- a/vipra-util/src/main/java/de/vipra/util/StringUtils.java
+++ b/vipra-util/src/main/java/de/vipra/util/StringUtils.java
@@ -51,7 +51,7 @@ public class StringUtils {
}
public static String timeString(long nanos, final boolean showMillis, final boolean compactHMS, final boolean onlyHMS) {
- final List<String> parts = new ArrayList<String>(6);
+ final List<String> parts = new ArrayList<>(6);
if (!onlyHMS) {
final long days = TimeUnit.NANOSECONDS.toDays(nanos);
diff --git a/vipra-util/src/main/java/de/vipra/util/model/ArticleWord.java b/vipra-util/src/main/java/de/vipra/util/model/ArticleWord.java
index e3ee8969a5cf990f3fd2c9a7642c0c0b6e87321d..4d2f669a71830ba1d7e29f2248683d9ae2b5717c 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/ArticleWord.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/ArticleWord.java
@@ -40,7 +40,11 @@ public class ArticleWord implements Comparable<ArticleWord>, Serializable {
}
public String aTag(final String word) {
- return "<a data-word=\"" + this.word + "\">" + word + "</a>";
+ return aTag(this.word, word);
+ }
+
+ public static String aTag(final String word, final String label) {
+ return "<w=" + word + ">" + label + "</a>";
}
@Override
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TextEntity.java b/vipra-util/src/main/java/de/vipra/util/model/TextEntity.java
index 80f1cdb2ba54ace0a044d3d0cfa4bcb4821d98a7..7bc4c2c53fabba686ff17b28124e373705f80879 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/TextEntity.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/TextEntity.java
@@ -74,10 +74,6 @@ public class TextEntity implements Serializable {
this.types = types;
}
- public String aTag() {
- return "<a href=\"" + url + "\">" + entity + "</a>";
- }
-
public String realEntity() {
return TextEntityFull.realEntity(url);
}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TextEntityFull.java b/vipra-util/src/main/java/de/vipra/util/model/TextEntityFull.java
index 776829e3e012ac6d5df477c6613953bfbc9a92f7..5942cb6e6a583eb31485f524f35f142e2e786779 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/TextEntityFull.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/TextEntityFull.java
@@ -156,7 +156,7 @@ public class TextEntityFull implements Model<String>, Serializable, Comparable<T
}
public String aTag(final String entity) {
- return "<a data-entity=\"" + this.entity + "\">" + entity + "</a>";
+ return "<e=" + this.entity + ">" + entity + "</a>";
}
public String realEntity() {
diff --git a/vipra-util/src/main/java/de/vipra/util/service/MongoService.java b/vipra-util/src/main/java/de/vipra/util/service/MongoService.java
index 90ea54693c35ca8f07122f6ffae926946ab12b45..783ce4dd0556169a66e966864b952589871ff5df 100644
--- a/vipra-util/src/main/java/de/vipra/util/service/MongoService.java
+++ b/vipra-util/src/main/java/de/vipra/util/service/MongoService.java
@@ -238,7 +238,7 @@ public class MongoService<Type extends Model<IdType>, IdType> {
public static <Type extends Model<IdType>, IdType> MongoService<Type, IdType> getDatabaseService(final Config config, final Class<Type> clazz)
throws ConfigException {
final Mongo mongo = config.getMongo();
- return new MongoService<Type, IdType>(mongo, clazz);
+ return new MongoService<>(mongo, clazz);
}
public static void dropDatabase(final Config config) throws ConfigException {