From 8cee759c458a326538b2dfa5261ad7a331b8994c Mon Sep 17 00:00:00 2001
From: Eike Cochu <eike@cochu.com>
Date: Wed, 6 Apr 2016 22:12:07 +0200
Subject: [PATCH] updated console output, added progress bar for modeling
---
.../main/java/de/vipra/cmd/lda/Analyzer.java | 15 ++--
.../de/vipra/cmd/option/ImportCommand.java | 2 +
.../de/vipra/cmd/option/IndexingCommand.java | 3 +-
.../de/vipra/cmd/option/ModelingCommand.java | 3 +-
vipra-ui/app/html/directives/topic-link.html | 1 +
vipra-ui/app/html/topics/index.html | 1 -
.../main/java/de/vipra/util/ConsoleUtils.java | 69 ++++++++++++++++---
.../java/de/vipra/util/model/ArticleFull.java | 2 +-
8 files changed, 77 insertions(+), 19 deletions(-)
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/lda/Analyzer.java b/vipra-cmd/src/main/java/de/vipra/cmd/lda/Analyzer.java
index 1882c49e..d6e93023 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/lda/Analyzer.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/lda/Analyzer.java
@@ -117,12 +117,21 @@ public class Analyzer {
String line;
int iteration = 0;
+ int maxIterationsLength = Integer.toString(modelConfig.getDynamicMaxIterations()).length();
while ((line = in.readLine()) != null) {
if (line.contains("EM iter")) {
- ConsoleUtils.info("iteration " + iteration++);
+ iteration++;
+ double progress = (double) iteration / modelConfig.getDynamicMaxIterations() * 100.0;
+ int tenthPercent = (int) (progress - progress % 10) / 10;
+ ConsoleUtils.infoNOLF("[" + StringUtils.repeat("#", tenthPercent) + StringUtils.repeat(" ", 10 - tenthPercent) + "] "
+ + StringUtils.pad(Integer.toString((int) Math.floor(progress)), 3, true) + "% ("
+ + StringUtils.pad(Integer.toString(iteration), maxIterationsLength, true) + "/" + modelConfig.getDynamicMinIterations()
+ + "-" + modelConfig.getDynamicMaxIterations() + ")\r");
}
}
+ ConsoleUtils.clearLine();
+
in.close();
p.waitFor();
}
@@ -176,10 +185,6 @@ public class Analyzer {
topicModel.setArticleCount(articleCount);
topicModel.setTopicCount(topicCount);
- ConsoleUtils.info("vocabulary size: " + wordCount);
- ConsoleUtils.info("sequences: " + windowCount);
- ConsoleUtils.info("topics: " + topicCount);
-
final boolean seqRelativeCutoff = modelConfig.getMinRelativeProbability() > 0;
// create sequence windows
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
index 9eefdddd..485b3d41 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
@@ -248,6 +248,8 @@ public class ImportCommand implements Command {
private void importForModel(final TopicModelConfig modelConfig)
throws java.text.ParseException, IOException, ConfigException, ParseException, InterruptedException, DatabaseException {
+ ConsoleUtils.info("importing for model: " + modelConfig.getName());
+
this.modelConfig = modelConfig;
if (config.getSpotlightUrl() != null)
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/IndexingCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/IndexingCommand.java
index 515efd85..197a4a30 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/IndexingCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/IndexingCommand.java
@@ -34,8 +34,9 @@ public class IndexingCommand implements Command {
}
private void indexForModel(final TopicModelConfig modelConfig) throws ParseException, IOException, ConfigException, DatabaseException {
- final FilebaseIDDateIndex index = new FilebaseIDDateIndex(modelConfig.getModelDir(config.getDataDirectory()));
+ ConsoleUtils.info("indexing for model: " + modelConfig.getName());
+ final FilebaseIDDateIndex index = new FilebaseIDDateIndex(modelConfig.getModelDir(config.getDataDirectory()));
final String indexName = modelConfig.getName() + "-articles";
try {
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ModelingCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ModelingCommand.java
index 3a61ed45..f7fef6eb 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ModelingCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ModelingCommand.java
@@ -25,6 +25,8 @@ public class ModelingCommand implements Command {
private void modelForModel(final TopicModelConfig modelConfig)
throws AnalyzerException, ConfigException, DatabaseException, ParseException, IOException, InterruptedException {
+ ConsoleUtils.info("generating model: " + modelConfig.getName());
+
final Analyzer analyzer = new Analyzer();
final Timer timer = new Timer();
@@ -33,7 +35,6 @@ public class ModelingCommand implements Command {
/*
* do topic modeling
*/
- ConsoleUtils.info("topic modeling");
analyzer.analyze(modelConfig, reread);
timer.lap("topic modeling");
diff --git a/vipra-ui/app/html/directives/topic-link.html b/vipra-ui/app/html/directives/topic-link.html
index bcdd3845..4cd594c7 100644
--- a/vipra-ui/app/html/directives/topic-link.html
+++ b/vipra-ui/app/html/directives/topic-link.html
@@ -4,4 +4,5 @@
<span ng-bind="topic.name"></span>
<ng-transclude/>
</a>
+ <span class="badge pull-right" ng-bind="::topic.articlesCount" ng-attr-title="{{::topic.articlesCount}} article(s)"></span>
</span>
diff --git a/vipra-ui/app/html/topics/index.html b/vipra-ui/app/html/topics/index.html
index 956f11c4..a39d5eb0 100644
--- a/vipra-ui/app/html/topics/index.html
+++ b/vipra-ui/app/html/topics/index.html
@@ -25,7 +25,6 @@
<tr ng-repeat="topic in topics">
<td>
<topic-link topic="topic" />
- <span class="badge pull-right" ng-bind="::topic.articlesCount" ng-attr-title="{{::topic.articlesCount}} article(s)"></span>
</td>
</tr>
</tbody>
diff --git a/vipra-util/src/main/java/de/vipra/util/ConsoleUtils.java b/vipra-util/src/main/java/de/vipra/util/ConsoleUtils.java
index 29cadf29..0e2ef6fa 100644
--- a/vipra-util/src/main/java/de/vipra/util/ConsoleUtils.java
+++ b/vipra-util/src/main/java/de/vipra/util/ConsoleUtils.java
@@ -3,40 +3,89 @@ package de.vipra.util;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.io.PrintStream;
import org.fusesource.jansi.Ansi;
import org.fusesource.jansi.Ansi.Color;
public class ConsoleUtils {
+ public static enum Level {
+ INFO,
+ WARN,
+ ERROR
+ };
+
private static boolean silent = false;
private static int pad = 5;
+ private static int lastLineLength = 0;
public static void setSilent(final boolean s) {
silent = s;
}
+ private static void print(final PrintStream ps, final boolean newLine, final String msg) {
+ if (!silent) {
+ if(msg != null)
+ lastLineLength = msg.length();
+ if (newLine)
+ ps.println(msg);
+ else
+ ps.print(msg);
+ }
+ }
+
+ private static String label(final String label) {
+ return StringUtils.pad(label, pad);
+ }
+
+ public static void print(final Level level, final String msg) {
+ switch (level) {
+ case INFO:
+ info(msg);
+ break;
+ case WARN:
+ warn(msg);
+ break;
+ case ERROR:
+ error(msg);
+ break;
+ }
+ }
+
+ public static void clearLine() {
+ if(lastLineLength > 0) {
+ System.out.print(StringUtils.repeat(" ", lastLineLength) + "\r");
+ lastLineLength = 0;
+ }
+ }
+
public static void info(final String msg) {
- if (!silent)
- System.out.println(label("INFO") + " - " + msg);
+ print(System.out, true, label("INFO") + " - " + msg);
+ }
+
+ public static void infoNOLF(final String msg) {
+ print(System.out, false, label("INFO") + " - " + msg);
}
public static void warn(final String msg) {
- if (!silent)
- System.out.println(label("WARN") + " - " + msg);
+ print(System.out, true, label("WARN") + " - " + msg);
+ }
+
+ public static void warnNOLF(final String msg) {
+ print(System.out, false, label("WARN") + " - " + msg);
}
public static void error(final String msg) {
- if (!silent)
- System.out.println(label("ERROR") + " - " + Ansi.ansi().fg(Color.RED).a(msg).reset());
+ print(System.out, true, label("ERROR") + " - " + Ansi.ansi().fg(Color.RED).a(msg).reset());
}
- public static void error(final Throwable t) {
- error(t.getMessage());
+ public static void errorNOLF(final String msg) {
+ print(System.out, false, label("ERROR") + " - " + Ansi.ansi().fg(Color.RED).a(msg).reset());
}
- private static String label(final String label) {
- return StringUtils.pad(label, pad);
+ public static void error(final Throwable t) {
+ error(t.getMessage());
}
public static double readDouble(String message, final Double def, final Double min, final Double max, final boolean showBounds) {
diff --git a/vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java b/vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java
index be3c96a6..20b09a25 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java
@@ -119,7 +119,7 @@ public class ArticleFull implements Model<ObjectId>, Serializable {
@ElasticIndex("excerpt")
public String serializeExcerpt() {
- return StringUtils.ellipsize(text, Constants.EXCERPT_LENGTH);
+ return StringUtils.ellipsize(text.replaceAll("<[^>]*>", ""), Constants.EXCERPT_LENGTH);
}
@ElasticIndex("text")
--
GitLab