From 2dafc985413f1c4a10b82de993c27515465bd93f Mon Sep 17 00:00:00 2001
From: Eike Cochu <eike@cochu.com>
Date: Tue, 23 Feb 2016 20:29:47 +0100
Subject: [PATCH] updated build script, modeling step
moved indexing to own command, IndexingCommand
moved modeling step mostly to specific analyzer
removed topic index, obsolete
fixed ui router nested views
added topic similar topics, articles pages
renamed pair to tuple
---
build.sh | 13 ++
.../de/vipra/rest/resource/TopicResource.java | 22 +-
vipra-backend/src/main/webapp/WEB-INF/web.xml | 32 +--
.../main/java/de/vipra/cmd/CmdOptions.java | 4 +
.../src/main/java/de/vipra/cmd/Main.java | 5 +
.../main/java/de/vipra/cmd/lda/Analyzer.java | 26 ---
.../java/de/vipra/cmd/lda/DTMAnalyzer.java | 39 +---
.../java/de/vipra/cmd/lda/JGibbAnalyzer.java | 193 +++++++++---------
.../de/vipra/cmd/option/IndexingCommand.java | 54 +++++
.../de/vipra/cmd/option/ModelingCommand.java | 117 -----------
.../src/main/resources/config.properties | 3 +-
vipra-ui/app/html/about.html | 6 +-
vipra-ui/app/html/articles/index.html | 8 +-
vipra-ui/app/html/articles/show.html | 6 +-
vipra-ui/app/html/index.html | 6 +-
vipra-ui/app/html/modal.html | 11 -
vipra-ui/app/html/network.html | 6 +-
vipra-ui/app/html/topics/articles.html | 44 ++++
vipra-ui/app/html/topics/index.html | 8 +-
vipra-ui/app/html/topics/show.html | 15 +-
vipra-ui/app/html/topics/similar.html | 5 +
vipra-ui/app/html/words/index.html | 8 +-
vipra-ui/app/html/words/show.html | 6 +-
vipra-ui/app/index.html | 4 +-
vipra-ui/app/js/app.js | 88 +++++---
vipra-ui/app/js/config.js | 5 +-
vipra-ui/app/js/constants.js | 12 --
vipra-ui/app/js/controllers.js | 43 ++--
vipra-ui/app/js/directives.js | 9 +-
vipra-ui/app/less/app.less | 8 +-
vipra-ui/bower.json | 3 +-
vipra-ui/build.sh | 9 +
vipra-ui/gulpfile.js | 1 +
vipra-ui/package.json | 3 +-
.../java/de/vipra/util/ConvertStream.java | 118 -----------
.../src/main/java/de/vipra/util/CountMap.java | 32 +++
.../java/de/vipra/util/FrequencyList.java | 27 ---
.../src/main/java/de/vipra/util/Pair.java | 35 ----
.../src/main/java/de/vipra/util/Tuple.java | 35 ++++
.../java/de/vipra/util/model/ArticleFull.java | 2 +-
.../java/de/vipra/util/model/TopicFull.java | 5 +-
.../java/de/vipra/util/model/TopicRef.java | 14 +-
.../de/vipra/util/service/MongoService.java | 12 +-
.../java/de/vipra/util/service/Service.java | 12 +-
44 files changed, 515 insertions(+), 599 deletions(-)
create mode 100644 vipra-cmd/src/main/java/de/vipra/cmd/option/IndexingCommand.java
delete mode 100644 vipra-ui/app/html/modal.html
create mode 100644 vipra-ui/app/html/topics/articles.html
create mode 100644 vipra-ui/app/html/topics/similar.html
delete mode 100644 vipra-ui/app/js/constants.js
create mode 100755 vipra-ui/build.sh
delete mode 100644 vipra-util/src/main/java/de/vipra/util/ConvertStream.java
create mode 100644 vipra-util/src/main/java/de/vipra/util/CountMap.java
delete mode 100644 vipra-util/src/main/java/de/vipra/util/FrequencyList.java
delete mode 100644 vipra-util/src/main/java/de/vipra/util/Pair.java
create mode 100644 vipra-util/src/main/java/de/vipra/util/Tuple.java
diff --git a/build.sh b/build.sh
index 8de1b577..63f31a0d 100755
--- a/build.sh
+++ b/build.sh
@@ -50,6 +50,19 @@ if [ $? -ne 0 ]; then
exit 1
fi
+echo "" >> $LOG
+echo "-------------------------------" >> $LOG
+echo "compiling vipra-ui" | tee -a $LOG
+echo "-------------------------------" >> $LOG
+cd ./vipra-ui
+./build.sh >> $LOG 2>&1
+cd ..
+cp -r ./vipra-ui/public/* ./vipra-backend/src/main/webapp
+if [ $? -ne 0 ]; then
+ echo "error"
+ exit 1
+fi
+
echo "" >> $LOG
echo "-------------------------------" >> $LOG
echo "compiling vipra-backend" | tee -a $LOG
diff --git a/vipra-backend/src/main/java/de/vipra/rest/resource/TopicResource.java b/vipra-backend/src/main/java/de/vipra/rest/resource/TopicResource.java
index acbbf0a9..de832f21 100644
--- a/vipra-backend/src/main/java/de/vipra/rest/resource/TopicResource.java
+++ b/vipra-backend/src/main/java/de/vipra/rest/resource/TopicResource.java
@@ -79,7 +79,6 @@ public class TopicResource {
@GET
@Produces(MediaType.APPLICATION_JSON)
- @Consumes(MediaType.APPLICATION_JSON)
@Path("{id}")
public Response getTopic(@PathParam("id") String id, @QueryParam("fields") String fields)
throws ConfigException, IOException {
@@ -110,7 +109,6 @@ public class TopicResource {
@GET
@Produces(MediaType.APPLICATION_JSON)
- @Consumes(MediaType.APPLICATION_JSON)
@Path("{id}/articles")
public Response getArticles(@PathParam("id") String id, @QueryParam("skip") Integer skip,
@QueryParam("limit") Integer limit, @QueryParam("sort") @DefaultValue("title") String sortBy,
@@ -138,6 +136,26 @@ public class TopicResource {
}
}
+ @GET
+ @Produces(MediaType.APPLICATION_JSON)
+ @Path("{id}/similar/by-words")
+ public Response similarTopicsByWords(@PathParam("id") String id, @QueryParam("skip") Integer skip,
+ @QueryParam("limit") Integer limit, @QueryParam("sort") @DefaultValue("title") String sortBy,
+ @QueryParam("fields") String fields) {
+ // TODO implement
+ return null;
+ }
+
+ @GET
+ @Produces(MediaType.APPLICATION_JSON)
+ @Path("{id}/similar/by-articles")
+ public Response similarTopicsByArticles(@PathParam("id") String id, @QueryParam("skip") Integer skip,
+ @QueryParam("limit") Integer limit, @QueryParam("sort") @DefaultValue("title") String sortBy,
+ @QueryParam("fields") String fields) {
+ // TODO implement
+ return null;
+ }
+
@PUT
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
diff --git a/vipra-backend/src/main/webapp/WEB-INF/web.xml b/vipra-backend/src/main/webapp/WEB-INF/web.xml
index 90458a45..d28582b4 100644
--- a/vipra-backend/src/main/webapp/WEB-INF/web.xml
+++ b/vipra-backend/src/main/webapp/WEB-INF/web.xml
@@ -1,15 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
-<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://xmlns.jcp.org/xml/ns/javaee" xsi:schemaLocation="http://xmlns.jcp.org/xml/ns/javaee http://xmlns.jcp.org/xml/ns/javaee/web-app_3_1.xsd" version="3.1">
- <servlet>
- <servlet-name>jersey</servlet-name>
- <servlet-class>org.glassfish.jersey.servlet.ServletContainer</servlet-class>
- <init-param>
- <param-name>javax.ws.rs.Application</param-name>
- <param-value>de.vipra.rest.Application</param-value>
- </init-param>
- </servlet>
- <servlet-mapping>
- <servlet-name>jersey</servlet-name>
- <url-pattern>/rest/*</url-pattern>
- </servlet-mapping>
+<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://xmlns.jcp.org/xml/ns/javaee"
+ xsi:schemaLocation="http://xmlns.jcp.org/xml/ns/javaee http://xmlns.jcp.org/xml/ns/javaee/web-app_3_1.xsd"
+ version="3.1">
+ <welcome-file-list>
+ <welcome-file>index.html</welcome-file>
+ </welcome-file-list>
+ <servlet>
+ <servlet-name>jersey</servlet-name>
+ <servlet-class>org.glassfish.jersey.servlet.ServletContainer</servlet-class>
+ <init-param>
+ <param-name>javax.ws.rs.Application</param-name>
+ <param-value>de.vipra.rest.Application</param-value>
+ </init-param>
+ </servlet>
+ <servlet-mapping>
+ <servlet-name>jersey</servlet-name>
+ <url-pattern>/rest/*</url-pattern>
+ </servlet-mapping>
</web-app>
\ No newline at end of file
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/CmdOptions.java b/vipra-cmd/src/main/java/de/vipra/cmd/CmdOptions.java
index 0610a0a4..8158313b 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/CmdOptions.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/CmdOptions.java
@@ -41,6 +41,9 @@ public class CmdOptions extends Options {
public static final String OPT_MODELING = "m";
public static final String OPT_MODELING_LONG = "modeling";
+ public static final String OPT_INDEXING = "e";
+ public static final String OPT_INDEXING_LONG = "indexing";
+
public CmdOptions() {
addOption(Option.builder(OPT_HELP).longOpt(OPT_HELP_LONG).desc("print this message").build());
addOption(Option.builder(OPT_SHELL).longOpt(OPT_SHELL_LONG).hasArg(true).argName("name")
@@ -56,6 +59,7 @@ public class CmdOptions extends Options {
addOption(Option.builder(OPT_SILENT).longOpt(OPT_SILENT_LONG).desc("mute all output").build());
addOption(Option.builder(OPT_CONFIG).longOpt(OPT_CONFIG_LONG).desc("show configuration").build());
addOption(Option.builder(OPT_MODELING).longOpt(OPT_MODELING_LONG).desc("regenerate topic model").build());
+ addOption(Option.builder(OPT_INDEXING).longOpt(OPT_INDEXING_LONG).desc("regenerate search index").build());
}
public void printHelp(String cmd) {
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/Main.java b/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
index a8839486..70842fdf 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
@@ -11,6 +11,7 @@ import static de.vipra.cmd.CmdOptions.OPT_SHELL;
import static de.vipra.cmd.CmdOptions.OPT_SILENT;
import static de.vipra.cmd.CmdOptions.OPT_STATS;
import static de.vipra.cmd.CmdOptions.OPT_TEST;
+import static de.vipra.cmd.CmdOptions.OPT_INDEXING;
import java.util.ArrayList;
import java.util.List;
@@ -37,6 +38,7 @@ import de.vipra.cmd.option.ClearCommand;
import de.vipra.cmd.option.Command;
import de.vipra.cmd.option.ConfigCommand;
import de.vipra.cmd.option.ImportCommand;
+import de.vipra.cmd.option.IndexingCommand;
import de.vipra.cmd.option.ModelingCommand;
import de.vipra.cmd.option.StatsCommand;
import de.vipra.cmd.option.TestCommand;
@@ -112,6 +114,9 @@ public class Main {
if (cline.hasOption(OPT_MODELING))
commands.add(new ModelingCommand());
+ if (cline.hasOption(OPT_INDEXING))
+ commands.add(new IndexingCommand());
+
if (cline.hasOption(OPT_STATS))
commands.add(new StatsCommand());
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/lda/Analyzer.java b/vipra-cmd/src/main/java/de/vipra/cmd/lda/Analyzer.java
index 01cc8d15..1d8e0f20 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/lda/Analyzer.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/lda/Analyzer.java
@@ -1,13 +1,8 @@
package de.vipra.cmd.lda;
-import java.util.List;
-
import de.vipra.cmd.ex.AnalyzerException;
import de.vipra.util.Config;
-import de.vipra.util.ConvertStream;
import de.vipra.util.WordMap;
-import de.vipra.util.model.TopicFull;
-import de.vipra.util.model.TopicRef;
public abstract class Analyzer {
@@ -25,27 +20,6 @@ public abstract class Analyzer {
public abstract void analyze() throws AnalyzerException;
- /**
- * Returns a converting stream of topics, read from the topic definition
- * file. Usually, a topic definition consists of a list of words, that are
- * assigned to that topic with a certain likeliness.
- *
- * @return topic definition stream
- * @throws AnalyzerException
- */
- public abstract ConvertStream<TopicFull> getTopicDefinitions() throws AnalyzerException;
-
- /**
- * Returns a converting stream of lists of topic references. Normally, topic
- * modeling outputs topics for each word of each document. These references
- * are returned by this function.
- *
- * @return stream of lists of topic references per document (ordered by
- * index)
- * @throws AnalyzerException
- */
- public abstract ConvertStream<List<TopicRef>> getTopics() throws AnalyzerException;
-
public static Analyzer getAnalyzer(Config config, WordMap wordMap) throws AnalyzerException {
Analyzer analyzer = null;
switch (config.analyzer) {
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/lda/DTMAnalyzer.java b/vipra-cmd/src/main/java/de/vipra/cmd/lda/DTMAnalyzer.java
index 54d2a8b8..306ed272 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/lda/DTMAnalyzer.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/lda/DTMAnalyzer.java
@@ -4,27 +4,26 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
-import java.util.List;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import de.vipra.cmd.ex.AnalyzerException;
-import de.vipra.cmd.file.DTMVocabulary;
import de.vipra.util.Config;
import de.vipra.util.Constants;
-import de.vipra.util.ConvertStream;
import de.vipra.util.StringUtils;
import de.vipra.util.WordMap;
import de.vipra.util.ex.ConfigException;
-import de.vipra.util.model.TopicFull;
-import de.vipra.util.model.TopicRef;
public class DTMAnalyzer extends Analyzer {
public static final Logger log = LogManager.getLogger(DTMAnalyzer.class);
public static final String NAME = "dtm";
+ public static final int dynamicMinIter = 100;
+ public static final int dynamicMaxIter = 1000;
+ public static final int staticIter = 100;
+
private String command;
private File modelDir;
private File outDir;
@@ -68,11 +67,11 @@ public class DTMAnalyzer extends Analyzer {
// alpha (default -10)
"--alpha=0.01",
// minimum number if iterations
- "--lda_sequence_min_iter=5",
+ "--lda_sequence_min_iter=" + dynamicMinIter,
// maximum number of iterations
- "--lda_sequence_max_iter=10",
+ "--lda_sequence_max_iter=" + dynamicMaxIter,
// em iter (default 20)
- "--lda_max_em_iter=20",
+ "--lda_max_em_iter=" + staticIter,
// input file prefix
"--corpus_prefix=" + corpusPrefix,
// output directory
@@ -101,32 +100,10 @@ public class DTMAnalyzer extends Analyzer {
in.close();
p.waitFor();
- log.info("done");
+ // TODO save model
} catch (IOException | InterruptedException e) {
throw new AnalyzerException(e);
}
}
- @Override
- public ConvertStream<TopicFull> getTopicDefinitions() throws AnalyzerException {
- DTMVocabulary vocab;
- try {
- vocab = new DTMVocabulary(modelDir, false);
- } catch (IOException e) {
- throw new AnalyzerException(e);
- }
- return null;
- }
-
- @Override
- public ConvertStream<List<TopicRef>> getTopics() throws AnalyzerException {
- DTMVocabulary vocab;
- try {
- vocab = new DTMVocabulary(modelDir, false);
- } catch (IOException e) {
- throw new AnalyzerException(e);
- }
- return null;
- }
-
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbAnalyzer.java b/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbAnalyzer.java
index 8fa94370..3af056a2 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbAnalyzer.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbAnalyzer.java
@@ -1,29 +1,38 @@
package de.vipra.cmd.lda;
+import java.io.BufferedReader;
import java.io.File;
-import java.io.FileNotFoundException;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import org.bson.types.ObjectId;
import de.vipra.cmd.ex.AnalyzerException;
+import de.vipra.cmd.file.FilebaseIndex;
import de.vipra.util.Config;
import de.vipra.util.Constants;
-import de.vipra.util.ConvertStream;
-import de.vipra.util.NumberUtils;
-import de.vipra.util.StringUtils;
+import de.vipra.util.CountMap;
+import de.vipra.util.FileUtils;
import de.vipra.util.WordMap;
import de.vipra.util.ex.ConfigException;
+import de.vipra.util.ex.DatabaseException;
+import de.vipra.util.model.ArticleFull;
+import de.vipra.util.model.Topic;
import de.vipra.util.model.TopicFull;
import de.vipra.util.model.TopicRef;
import de.vipra.util.model.TopicWord;
import de.vipra.util.model.Word;
+import de.vipra.util.service.MongoService;
import jgibblda.Estimator;
import jgibblda.LDACmdOption;
@@ -36,7 +45,9 @@ public class JGibbAnalyzer extends Analyzer {
private File modelDir;
private File modelFile;
private LDACmdOption options;
- private WordMap wordMap;
+ private MongoService<ArticleFull, ObjectId> dbArticles;
+ private MongoService<TopicFull, ObjectId> dbTopics;
+ private FilebaseIndex index;
protected JGibbAnalyzer() {
super("JGibb Analyzer");
@@ -64,7 +75,14 @@ public class JGibbAnalyzer extends Analyzer {
options.modelName = NAME;
- this.wordMap = wordMap;
+ try {
+ config = Config.getConfig();
+ dbArticles = MongoService.getDatabaseService(config, ArticleFull.class);
+ dbTopics = MongoService.getDatabaseService(config, TopicFull.class);
+ index = new FilebaseIndex(new File(modelDir, "index"));
+ } catch (Exception e) {
+ throw new AnalyzerException(e);
+ }
}
@Override
@@ -75,103 +93,88 @@ public class JGibbAnalyzer extends Analyzer {
Estimator estimator = new Estimator();
estimator.init(options);
estimator.estimate();
- }
- @Override
- public ConvertStream<TopicFull> getTopicDefinitions() throws AnalyzerException {
+ // read topic definitions and save
+
File twords = new File(modelDir, NAME + ".twords");
+ List<String> lines;
try {
- return new ConvertStream<TopicFull>(twords) {
- @Override
- public TopicFull convert(String line) {
- TopicFull topicDef = new TopicFull();
- List<TopicWord> topicWords = new ArrayList<>();
-
- // get index of topic
- // the index will be used as a temporary id until the topic
- // is created in the database
- Integer index = StringUtils.getFirstNumber(line);
- if (index == null) {
- log.error("could not extract topic index from line: " + line);
- } else {
- topicDef.setIndex(index);
- }
-
- // get all lines that follow until the next topic is
- // discovered (line does not start with a tab). Cut off
- // after k words are gathered.
- String nextLine;
- while ((nextLine = nextLine()) != null) {
- if (nextLine.startsWith("\t")) {
- String[] parts = nextLine.trim().split("\\s+");
- try {
- Word word = wordMap.get(parts[0]);
- // round likeliness precision
- double likeliness = NumberUtils.roundToPrecision(Double.parseDouble(parts[1]),
- Constants.LIKELINESS_PRECISION);
-
- // check if word likely enough to relate to
- // topic
- if (likeliness < Constants.MINIMUM_LIKELINESS)
- continue;
-
- TopicWord topicWord = new TopicWord(word, likeliness);
- topicWords.add(topicWord);
- } catch (NumberFormatException e) {
- log.error("could not parse number in line: " + nextLine);
- } catch (ArrayIndexOutOfBoundsException e) {
- log.error("ill formatted topic definition in line: " + nextLine);
- }
- } else {
- buffer(nextLine);
- break;
- }
- }
- Collections.sort(topicWords, Collections.reverseOrder());
- topicDef.setWords(topicWords);
- topicDef.setName(TopicFull.getNameFromWords(topicWords));
- return topicDef;
- }
- };
- } catch (FileNotFoundException e) {
+ lines = FileUtils.readFile(twords);
+ } catch (IOException e) {
throw new AnalyzerException(e);
}
- }
- @Override
- public ConvertStream<List<TopicRef>> getTopics() throws AnalyzerException {
+ List<TopicFull> newTopics = new ArrayList<>(options.K);
+ Map<Integer, Topic> newTopicsMap = new HashMap<>(options.K);
+
+ // for each topic
+ for (int topicNum = 0; topicNum < options.K; topicNum++) {
+ TopicFull newTopic = new TopicFull();
+ List<TopicWord> topicWords = new ArrayList<>(options.twords);
+
+ // for each word
+ for (int wordNum = 0, lineNum = topicNum * options.K + 1; wordNum < options.twords; wordNum++, lineNum++) {
+ String[] parts = lines.get(lineNum).trim().split("\\s+");
+ TopicWord topicWord = new TopicWord(new Word(parts[0]), Double.parseDouble(parts[1]));
+ topicWords.add(topicWord);
+ }
+
+ newTopic.setWords(topicWords);
+ newTopics.add(newTopic);
+ newTopicsMap.put(topicNum, new Topic(newTopic.getId()));
+ }
+
+ dbTopics.drop();
+ try {
+ dbTopics.createMultiple(newTopics);
+ } catch (DatabaseException e) {
+ throw new AnalyzerException(e);
+ }
+
+ // read documents and reference topics
+
File tassign = new File(modelDir, NAME + ".tassign");
+ Pattern topicIndexPattern = Pattern.compile(":(\\d+)");
+ BufferedReader in = null;
+
try {
- return new ConvertStream<List<TopicRef>>(tassign) {
- @Override
- public List<TopicRef> convert(String line) {
- // count topics
- Map<String, Integer> countMap = new HashMap<>();
- String[] wordList = line.split("\\s+");
- for (String word : wordList) {
- // the file uses a simple integer as topic ids. Use this
- // id as a temporary id until the topic is created in
- // the database
- String topic = word.split(":")[1];
- Integer count = countMap.get(topic);
- countMap.put(topic, count == null ? 1 : count + 1);
- }
-
- // turn into list
- List<TopicRef> topicCount = new ArrayList<>(countMap.size());
- for (Entry<String, Integer> e : countMap.entrySet()) {
- TopicRef tc = new TopicRef();
- tc.setTopicIndex(e.getKey());
- tc.setCount(e.getValue());
- topicCount.add(tc);
- }
-
- Collections.sort(topicCount, Collections.reverseOrder());
-
- return topicCount;
+ in = new BufferedReader(new InputStreamReader(new FileInputStream(tassign)));
+ String line;
+ int articleIndex = 0;
+
+ // each line in the tassign file is a document, formatted with
+ // <word-id>:<topic-id>
+ while ((line = in.readLine()) != null) {
+ // extract topic ids and count them
+ CountMap<String> countMap = new CountMap<>();
+ Matcher matcher = topicIndexPattern.matcher(line);
+ while (matcher.find()) {
+ countMap.count(matcher.group(1));
+ }
+
+ // create list of topics refs referencing topics with counted
+ // occurrences
+ List<TopicRef> newTopicRefs = new ArrayList<>();
+ for (Entry<String, Integer> entry : countMap.entrySet()) {
+ TopicRef ref = new TopicRef();
+ ref.setCount(entry.getValue());
+ ref.setTopic(newTopicsMap.get(Integer.parseInt(entry.getKey())));
+ newTopicRefs.add(ref);
+ }
+
+ // update article with topic references (partial update)
+ ArticleFull article = new ArticleFull();
+ article.setId(index.get(articleIndex++));
+ article.setTopics(newTopicRefs);
+ try {
+ // TODO: using field name here. Hard to refactor
+ dbArticles.updateSingle(article, "topics");
+ } catch (DatabaseException e) {
+ log.error(e);
}
- };
- } catch (FileNotFoundException e) {
+ }
+ in.close();
+ } catch (IOException e) {
throw new AnalyzerException(e);
}
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/IndexingCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/IndexingCommand.java
new file mode 100644
index 00000000..29977e57
--- /dev/null
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/IndexingCommand.java
@@ -0,0 +1,54 @@
+package de.vipra.cmd.option;
+
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.bson.types.ObjectId;
+import org.elasticsearch.client.Client;
+
+import de.vipra.cmd.file.Filebase;
+import de.vipra.cmd.file.FilebaseIndex;
+import de.vipra.util.Config;
+import de.vipra.util.ESClient;
+import de.vipra.util.ESSerializer;
+import de.vipra.util.MongoUtils;
+import de.vipra.util.model.ArticleFull;
+import de.vipra.util.service.MongoService;
+
+public class IndexingCommand implements Command {
+
+ public static final Logger log = LogManager.getLogger(IndexingCommand.class);
+
+ @Override
+ public void run() throws Exception {
+ Config config = Config.getConfig();
+ MongoService<ArticleFull, ObjectId> dbArticles = MongoService.getDatabaseService(config, ArticleFull.class);
+ Filebase filebase = Filebase.getFilebase(config);
+ FilebaseIndex index = filebase.getIndex();
+ Client elasticClient = ESClient.getClient(config);
+ ESSerializer<ArticleFull> elasticSerializer = new ESSerializer<>(ArticleFull.class);
+
+ // clear index
+ elasticClient.admin().indices().prepareDelete("_all").get();
+
+ Iterator<String> indexIter = index.iterator();
+ while (indexIter.hasNext()) {
+ // get article from database
+ String id = indexIter.next();
+ ArticleFull article = dbArticles.getSingle(MongoUtils.objectId(id), true);
+ if (article == null) {
+ log.error("no article found in db for id " + id);
+ continue;
+ }
+
+ // index article
+ Map<String, Object> source = elasticSerializer.serialize(article);
+ elasticClient.prepareIndex("articles", "article", article.getId().toString()).setSource(source).get();
+ }
+
+ elasticClient.close();
+ }
+
+}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ModelingCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ModelingCommand.java
index 0bb226b5..fd5a5a6e 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ModelingCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ModelingCommand.java
@@ -1,35 +1,13 @@
package de.vipra.cmd.option;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import org.bson.types.ObjectId;
-import org.elasticsearch.client.Client;
-import de.vipra.cmd.file.Filebase;
-import de.vipra.cmd.file.FilebaseIndex;
import de.vipra.cmd.lda.Analyzer;
import de.vipra.util.Config;
-import de.vipra.util.Constants;
-import de.vipra.util.ConvertStream;
-import de.vipra.util.ESClient;
-import de.vipra.util.ESSerializer;
-import de.vipra.util.MongoUtils;
import de.vipra.util.StringUtils;
import de.vipra.util.Timer;
import de.vipra.util.WordMap;
-import de.vipra.util.ex.DatabaseException;
-import de.vipra.util.model.ArticleFull;
-import de.vipra.util.model.Topic;
-import de.vipra.util.model.TopicFull;
-import de.vipra.util.model.TopicRef;
import de.vipra.util.model.Word;
import de.vipra.util.service.MongoService;
@@ -38,26 +16,16 @@ public class ModelingCommand implements Command {
public static final Logger log = LogManager.getLogger(ModelingCommand.class);
private Config config;
- private MongoService<ArticleFull, ObjectId> dbArticles;
- private MongoService<TopicFull, ObjectId> dbTopics;
private MongoService<Word, String> dbWords;
- private Filebase filebase;
private WordMap wordMap;
private Analyzer analyzer;
- private Client elasticClient;
- private ESSerializer<ArticleFull> elasticSerializer;
@Override
public void run() throws Exception {
config = Config.getConfig();
- dbArticles = MongoService.getDatabaseService(config, ArticleFull.class);
- dbTopics = MongoService.getDatabaseService(config, TopicFull.class);
dbWords = MongoService.getDatabaseService(config, Word.class);
- filebase = Filebase.getFilebase(config);
wordMap = new WordMap(dbWords);
analyzer = Analyzer.getAnalyzer(config, wordMap);
- elasticClient = ESClient.getClient(config);
- elasticSerializer = new ESSerializer<>(ArticleFull.class);
log.info("using analyzer: " + analyzer.getName());
@@ -71,94 +39,9 @@ public class ModelingCommand implements Command {
analyzer.analyze();
timer.lap("topic modeling");
- /*
- * save topic model
- */
- log.info("saving topic definitions");
- int batchSize = 100;
- ConvertStream<TopicFull> topicDefs = analyzer.getTopicDefinitions();
- Map<String, TopicFull> topicIndexMap = new HashMap<>();
- dbTopics.drop();
- List<TopicFull> newTopicDefs = new ArrayList<>(batchSize);
- List<Topic> newTopicRefs = new ArrayList<>();
- Iterator<TopicFull> it = topicDefs.iterator();
- while (it.hasNext()) {
- newTopicDefs.add(it.next());
- if (newTopicDefs.size() == batchSize || !it.hasNext()) {
- dbTopics.createMultiple(newTopicDefs);
- for (TopicFull newTopicDef : newTopicDefs) {
- topicIndexMap.put(Integer.toString(newTopicDef.getIndex()), newTopicDef);
- newTopicRefs.add(new Topic(newTopicDef.getId()));
- }
- newTopicDefs.clear();
- }
- }
- timer.lap("saving topics");
-
- /*
- * save topic refs and index article
- */
- log.info("saving document topics");
- ConvertStream<List<TopicRef>> topicStream = analyzer.getTopics();
- FilebaseIndex index = filebase.getIndex();
- Iterator<String> indexIter = index.iterator();
- Iterator<List<TopicRef>> topicRefsListIter = topicStream.iterator();
- elasticClient.admin().indices().prepareDelete("_all").get();
- while (indexIter.hasNext() && topicRefsListIter.hasNext()) {
- // get article from database
- String id = indexIter.next();
- ArticleFull article = dbArticles.getSingle(MongoUtils.objectId(id), true);
- if (article == null) {
- log.error("no article found in db for id " + id);
- continue;
- }
-
- double wordCount = article.getStats().getWordCount();
-
- // insert topic references into article, ignoring low refs
- List<TopicRef> topicRefs = topicRefsListIter.next();
- for (ListIterator<TopicRef> topicRefsIter = topicRefs.listIterator(); topicRefsIter.hasNext();) {
- TopicRef topicRef = topicRefsIter.next();
- if ((topicRef.getCount() / wordCount) < Constants.TOPIC_THRESHOLD) {
- topicRefsIter.remove();
- continue;
- }
- TopicFull topicFull = topicIndexMap.get(topicRef.getTopicIndex());
- if (topicFull != null) {
- topicRef.setTopic(new Topic(topicFull.getId(), topicFull.getName()));
- } else
- log.error("no object id for topic index " + topicRef.getTopicIndex());
- }
-
- article.setTopics(topicRefs);
-
- try {
- dbArticles.replaceSingle(article);
- } catch (DatabaseException e) {
- log.error("could not update article: " + article.getTitle() + " (" + article.getId() + ")");
- }
-
- // index article
- Map<String, Object> source = elasticSerializer.serialize(article);
- elasticClient.prepareIndex("articles", "article", article.getId().toString()).setSource(source).get();
- }
-
- /*
- * save words
- */
- log.info("saving words");
- Set<Word> importedWords = wordMap.getNewWords();
- timer.lap("saving topic refs and indexing");
- wordMap.create();
- timer.lap("saving words");
-
- elasticClient.close();
-
/*
* run information
*/
- int newWordsCount = importedWords.size();
- log.info("imported " + newWordsCount + " new " + StringUtils.quantity(newWordsCount, "word"));
log.info(timer.toString());
log.info("done in " + StringUtils.timeString(timer.total()));
}
diff --git a/vipra-cmd/src/main/resources/config.properties b/vipra-cmd/src/main/resources/config.properties
index ca312cfd..01b9444d 100644
--- a/vipra-cmd/src/main/resources/config.properties
+++ b/vipra-cmd/src/main/resources/config.properties
@@ -3,5 +3,4 @@ db.port=27017
db.name=test
tm.processor=corenlp
tm.analyzer=jgibb
-tm.saveallwords=false
-tm.dtmpath=/home/eike/Downloads/dtm_release/dtm/main
\ No newline at end of file
+tm.dtmpath=/home/eike/repos/master/dtm_release/dtm/main
\ No newline at end of file
diff --git a/vipra-ui/app/html/about.html b/vipra-ui/app/html/about.html
index edd025ff..d84cf035 100644
--- a/vipra-ui/app/html/about.html
+++ b/vipra-ui/app/html/about.html
@@ -1,4 +1,4 @@
-<div ng-cloak>
+<div ng-cloak ng-hide="$state.current.name !== 'about'">
<div class="page-header">
<h1>About</h1>
</div>
@@ -261,4 +261,6 @@
</tr>
</tbody>
</table>
-</div>
\ No newline at end of file
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/articles/index.html b/vipra-ui/app/html/articles/index.html
index 103bfb96..f30bfbbc 100644
--- a/vipra-ui/app/html/articles/index.html
+++ b/vipra-ui/app/html/articles/index.html
@@ -1,4 +1,4 @@
-<div ng-cloak>
+<div ng-cloak ng-hide="$state.current.name !== 'articles'">
<div class="text-muted">
Found <ng-pluralize count="articlesTotal||0" when="{0:'no articles',1:'1 article',other:'{} articles'}"></ng-pluralize> in the database.
<span ng-show="articlesTotal">
@@ -26,5 +26,7 @@
</li>
</ol>
- <pagination total="articlesTotal" page="page" limit="limit" change="changePage"/>
-</div>
\ No newline at end of file
+ <pagination total="articlesTotal" page="page" limit="limit"/>
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/articles/show.html b/vipra-ui/app/html/articles/show.html
index 27afa347..fe66fe1e 100644
--- a/vipra-ui/app/html/articles/show.html
+++ b/vipra-ui/app/html/articles/show.html
@@ -1,4 +1,4 @@
-<div ng-cloak>
+<div ng-cloak ng-hide="$state.current.name !== 'articles.show'">
<div class="page-header">
<h1 ng-bind="::article.title"></h1>
@@ -82,4 +82,6 @@
<hr>
<p ng-bind-html="::article.text" class="text-justify"></p>
-</div>
\ No newline at end of file
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/index.html b/vipra-ui/app/html/index.html
index cbbe697b..833c9fcc 100644
--- a/vipra-ui/app/html/index.html
+++ b/vipra-ui/app/html/index.html
@@ -1,4 +1,4 @@
-<div ng-cloak>
+<div ng-cloak ng-hide="$state.current.name !== 'index'">
<div class="row" ng-hide="search">
<div class="col-md-12">
<div class="heading"></div>
@@ -59,4 +59,6 @@
</ul>
</div>
</div>
-</div>
\ No newline at end of file
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/modal.html b/vipra-ui/app/html/modal.html
deleted file mode 100644
index e441d0bf..00000000
--- a/vipra-ui/app/html/modal.html
+++ /dev/null
@@ -1,11 +0,0 @@
-<div class="modal-header">
- <h3 class="modal-title">I'm a modal!</h3>
-</div>
-
-<div class="modal-body">
-</div>
-
-<div class="modal-footer">
- <button class="btn btn-primary" type="button" ng-click="ok()">OK</button>
- <button class="btn btn-warning" type="button" ng-click="cancel()">Cancel</button>
-</div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/network.html b/vipra-ui/app/html/network.html
index b05ce398..431188c6 100644
--- a/vipra-ui/app/html/network.html
+++ b/vipra-ui/app/html/network.html
@@ -1,4 +1,4 @@
-<div ng-cloak>
+<div ng-cloak ng-hide="$state.current.name !== 'network'">
<div class="fullsize navpadding">
<div class="graph-legend overlay">
<label style="color:{{colors.articles}}">
@@ -13,4 +13,6 @@
</div>
<div class="fullsize navpadding" id="visgraph"></div>
</div>
-</div>
\ No newline at end of file
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/topics/articles.html b/vipra-ui/app/html/topics/articles.html
new file mode 100644
index 00000000..2818921c
--- /dev/null
+++ b/vipra-ui/app/html/topics/articles.html
@@ -0,0 +1,44 @@
+<div ng-cloak ng-hide="$state.current.name !== 'topics.show.articles'">
+ <div class="page-header">
+ <h1 ng-bind-template="Articles for topic '{{::topic.name}}'"></h1>
+
+ <table class="item-actions">
+ <tr>
+ <td>
+ <a class="btn btn-default" ui-sref="^">Back</a>
+ </td>
+ </tr>
+ </table>
+ </div>
+
+ <div class="text-muted">
+ Found <ng-pluralize count="articlesTotal||0" when="{0:'no articles',1:'1 article',other:'{} articles'}"></ng-pluralize> in the database.
+ <span ng-show="articlesTotal">
+ Sort by
+ <ol class="nya-bs-select nya-bs-condensed" ng-model="sort" ng-model-store="sort" ng-model-default="'date'">
+ <li value="title" class="nya-bs-option"><a>Title</a></li>
+ <li value="date" class="nya-bs-option"><a>Date</a></li>
+ <li value="created" class="nya-bs-option"><a>Added</a></li>
+ </ol>
+ Direction
+ <ol class="nya-bs-select nya-bs-condensed" ng-model="order" ng-model-store="order" ng-model-default="'+'">
+ <li value="+" class="nya-bs-option"><a>Ascending</a></li>
+ <li value="-" class="nya-bs-option"><a>Descending</a></li>
+ </ol>
+ </span>
+ <br>
+ Page <span ng-bind="page||1"></span> of <span ng-bind="maxPage||1"></span>.
+ </div>
+
+ <pagination total="articlesTotal" page="page" limit="limit" change="changePage"/>
+
+ <ol ng-attr-start="{{(page-1)*limit+1}}">
+ <li ng-repeat="article in articles">
+ <a ui-sref="articles.show({id: article.id})" ng-bind="::article.title"></a>
+ </li>
+ </ol>
+
+ <pagination total="articlesTotal" page="page" limit="limit"/>
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/topics/index.html b/vipra-ui/app/html/topics/index.html
index 8d5a2496..b4273145 100644
--- a/vipra-ui/app/html/topics/index.html
+++ b/vipra-ui/app/html/topics/index.html
@@ -1,4 +1,4 @@
-<div ng-cloak>
+<div ng-cloak ng-hide="$state.current.name !== 'topics'">
<div class="text-muted">
Found <ng-pluralize count="topicsTotal||0" when="{0:'no topics',1:'1 topic',other:'{} topics'}"></ng-pluralize> in the database.
<span ng-show="topicsTotal">
@@ -25,5 +25,7 @@
</li>
</ol>
- <pagination total="topicsTotal" page="page" limit="limit" change="changePage"/>
-</div>
\ No newline at end of file
+ <pagination total="topicsTotal" page="page" limit="limit"/>
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/topics/show.html b/vipra-ui/app/html/topics/show.html
index 964d9743..2875d8c7 100644
--- a/vipra-ui/app/html/topics/show.html
+++ b/vipra-ui/app/html/topics/show.html
@@ -1,4 +1,4 @@
-<div ng-cloak>
+<div ng-cloak ng-hide="$state.current.name !== 'topics.show'">
<div class="page-header">
<h1>
<div ng-bind="topic.name" ng-hide="isRename"></div>
@@ -29,6 +29,15 @@
<td>
<a class="btn btn-default" ui-sref="network({type:'topics', id:topic.id})">Network graph</a>
</td>
+ <td>
+ <a class="btn btn-default" ui-sref="topics.show.articles({id:topic.id})">Articles</a>
+ </td>
+ <td>
+ <bs-dropdown label="Similar Topics">
+ <li><a ui-sref="topics.show.similar({id:topic.id, type:'by-words'})">By word share</a></li>
+ <li><a ui-sref="topics.show.similar({id:topic.id, type:'by-articles'})">By article share</a></li>
+ </bs-dropdown>
+ </td>
</tr>
</table>
</div>
@@ -84,4 +93,6 @@
</table>
</div>
</div>
-</div>
\ No newline at end of file
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/topics/similar.html b/vipra-ui/app/html/topics/similar.html
new file mode 100644
index 00000000..e868dc6e
--- /dev/null
+++ b/vipra-ui/app/html/topics/similar.html
@@ -0,0 +1,5 @@
+<div ng-cloak ng-hide="$state.current.name !== 'topics.show.similar'">
+
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/words/index.html b/vipra-ui/app/html/words/index.html
index 5da54c2d..d89dba08 100644
--- a/vipra-ui/app/html/words/index.html
+++ b/vipra-ui/app/html/words/index.html
@@ -1,4 +1,4 @@
-<div ng-cloak>
+<div ng-cloak ng-hide="$state.current.name !== 'words'">
<div class="text-muted">
Found <ng-pluralize count="wordsTotal||0" when="{0:'no words',1:'1 word',other:'{} words'}"></ng-pluralize> in the database.
<span ng-show="wordsTotal">
@@ -43,5 +43,7 @@
</div>
</div>
- <pagination total="wordsTotal" page="page" limit="limit" change="changePage"/>
-</div>
\ No newline at end of file
+ <pagination total="wordsTotal" page="page" limit="limit"/>
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/words/show.html b/vipra-ui/app/html/words/show.html
index 891b8f3d..d1ca723b 100644
--- a/vipra-ui/app/html/words/show.html
+++ b/vipra-ui/app/html/words/show.html
@@ -1,4 +1,4 @@
-<div ng-cloak>
+<div ng-cloak ng-hide="$state.current.name !== 'words.show'">
<div class="page-header">
<h1 ng-bind="::word.id"></h1>
</div>
@@ -29,4 +29,6 @@
</ol>
</div>
</div>
-</div>
\ No newline at end of file
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/index.html b/vipra-ui/app/index.html
index b4995c13..dbb7057b 100644
--- a/vipra-ui/app/index.html
+++ b/vipra-ui/app/index.html
@@ -35,7 +35,7 @@
<script src="js/templates.js"></script>
</head>
<body>
- <nav class="navbar navbar-default navbar-static-top">
+ <nav class="navbar navbar-default navbar-static-top navbar-breadcrumbs">
<div class="container-fluid">
<!-- Brand and toggle get grouped for better mobile display -->
<div class="navbar-header">
@@ -73,6 +73,8 @@
</div><!-- /.container-fluid -->
</nav>
+ <div ncy-breadcrumb ng-hide="$state.current.name === 'index'"></div>
+
<div class="container" ui-view ng-cloak></div>
</body>
</html>
\ No newline at end of file
diff --git a/vipra-ui/app/js/app.js b/vipra-ui/app/js/app.js
index bfb10682..f348a0cb 100644
--- a/vipra-ui/app/js/app.js
+++ b/vipra-ui/app/js/app.js
@@ -10,6 +10,7 @@
'ngWebSocket',
'ui.router',
'nya.bootstrap.select',
+ 'ncy-angular-breadcrumb',
'vipra.controllers',
'vipra.directives',
'vipra.factories',
@@ -25,37 +26,48 @@
$stateProvider.state('index', {
url: '/',
- templateUrl: Vipra.const.tplBase + '/index.html',
- controller: 'IndexController'
+ templateUrl: 'html/index.html',
+ controller: 'IndexController',
+ ncyBreadcrumb: {
+ label: 'Start'
+ }
});
$stateProvider.state('about', {
url: '/about',
- templateUrl: Vipra.const.tplBase + '/about.html',
- controller: 'AboutController'
+ templateUrl: 'html/about.html',
+ controller: 'AboutController',
+ ncyBreadcrumb: {
+ label: 'About'
+ }
});
$stateProvider.state('network', {
url: '/network/:type/:id',
- templateUrl: Vipra.const.tplBase + '/network.html',
- controller: 'NetworkController'
+ templateUrl: 'html/network.html',
+ controller: 'NetworkController',
+ ncyBreadcrumb: {
+ label: 'Network'
+ }
});
// states: articles
$stateProvider.state('articles', {
url: '/articles',
- templateUrl: Vipra.const.tplBase + '/articles/index.html',
- controller: 'ArticlesIndexController'
+ templateUrl: 'html/articles/index.html',
+ controller: 'ArticlesIndexController',
+ ncyBreadcrumb: {
+ label: 'Articles'
+ }
});
$stateProvider.state('articles.show', {
url: '/:id',
- views: {
- "@": {
- templateUrl: Vipra.const.tplBase + '/articles/show.html',
- controller: 'ArticlesShowController'
- }
+ templateUrl: 'html/articles/show.html',
+ controller: 'ArticlesShowController',
+ ncyBreadcrumb: {
+ label: '{{article.title.ellipsize(30)}}'
}
});
@@ -63,17 +75,37 @@
$stateProvider.state('topics', {
url: '/topics',
- templateUrl: Vipra.const.tplBase + '/topics/index.html',
- controller: 'TopicsIndexController'
+ templateUrl: 'html/topics/index.html',
+ controller: 'TopicsIndexController',
+ ncyBreadcrumb: {
+ label: 'Topics'
+ }
});
$stateProvider.state('topics.show', {
url: '/:id',
- "views": {
- "@": {
- templateUrl: Vipra.const.tplBase + '/topics/show.html',
- controller: 'TopicsShowController'
- }
+ templateUrl: 'html/topics/show.html',
+ controller: 'TopicsShowController',
+ ncyBreadcrumb: {
+ label: '{{topic.name}}'
+ }
+ });
+
+ $stateProvider.state('topics.show.articles', {
+ url: '/articles',
+ templateUrl: 'html/topics/articles.html',
+ controller: 'TopicsArticlesController',
+ ncyBreadcrumb: {
+ label: 'Topic Articles'
+ }
+ });
+
+ $stateProvider.state('topics.show.similar', {
+ url: '/similar/:type',
+ templateUrl: 'html/topics/similar.html',
+ controller: 'TopicsSimilarController',
+ ncyBreadcrumb: {
+ label: 'Similar Topics (by {{typeLabel}})'
}
});
@@ -81,17 +113,19 @@
$stateProvider.state('words', {
url: '/words',
- templateUrl: Vipra.const.tplBase + '/words/index.html',
- controller: 'WordsIndexController'
+ templateUrl: 'html/words/index.html',
+ controller: 'WordsIndexController',
+ ncyBreadcrumb: {
+ label: 'Words'
+ }
});
$stateProvider.state('words.show', {
url: '/:id',
- "views": {
- "@": {
- templateUrl: Vipra.const.tplBase + '/words/show.html',
- controller: 'WordsShowController'
- }
+ templateUrl: 'html/words/show.html',
+ controller: 'WordsShowController',
+ ncyBreadcrumb: {
+ label: '{{word.id}}'
}
});
diff --git a/vipra-ui/app/js/config.js b/vipra-ui/app/js/config.js
index 643b5e33..e6f400bd 100644
--- a/vipra-ui/app/js/config.js
+++ b/vipra-ui/app/js/config.js
@@ -4,7 +4,10 @@
Vipra.config = {
restUrl: '//' + location.hostname + ':8000/vipra/rest',
- websocketUrl: 'ws://' + location.hostname + ':8000/vipra/ws'
+ websocketUrl: 'ws://' + location.hostname + ':8000/vipra/ws',
+ latestItems: 3,
+ searchResults: 10,
+ pageSize: 100
};
})();
\ No newline at end of file
diff --git a/vipra-ui/app/js/constants.js b/vipra-ui/app/js/constants.js
deleted file mode 100644
index b60fe8af..00000000
--- a/vipra-ui/app/js/constants.js
+++ /dev/null
@@ -1,12 +0,0 @@
-(function() {
-
- window.Vipra = window.Vipra || {};
-
- Vipra.const = {
- tplBase: 'html',
- latestItems: 3,
- searchResults: 10,
- pageSize: 100
- };
-
-})();
\ No newline at end of file
diff --git a/vipra-ui/app/js/controllers.js b/vipra-ui/app/js/controllers.js
index 5fa9f2e5..daa5d471 100644
--- a/vipra-ui/app/js/controllers.js
+++ b/vipra-ui/app/js/controllers.js
@@ -21,15 +21,15 @@
$scope.search = $location.search().query;
- ArticleFactory.query({limit:Vipra.const.latestItems, sort:'-created'}, function(data) {
+ ArticleFactory.query({limit:Vipra.config.latestItems, sort:'-created'}, function(data) {
$scope.latestArticles = data;
});
- TopicFactory.query({limit:Vipra.const.latestItems, sort:'-created'}, function(data) {
+ TopicFactory.query({limit:Vipra.config.latestItems, sort:'-created'}, function(data) {
$scope.latestTopics = data;
});
- WordFactory.query({limit:Vipra.const.latestItems, sort:'-created'}, function(data) {
+ WordFactory.query({limit:Vipra.config.latestItems, sort:'-created'}, function(data) {
$scope.latestWords = data;
});
@@ -37,7 +37,7 @@
if($scope.search) {
$location.search('query', $scope.search);
$scope.searching = true;
- SearchFactory.query({limit:Vipra.const.searchResults, query:$scope.search}, function(data) {
+ SearchFactory.query({limit:Vipra.config.searchResults, query:$scope.search}, function(data) {
$scope.searching = false;
$scope.searchResults = data;
});
@@ -266,11 +266,7 @@
function($scope, $state, $location, ArticleFactory, Store) {
$scope.page = Math.max($location.search().page || 1, 1);
- $scope.limit = Vipra.const.pageSize;
-
- $scope.changePage = function(page) {
- $scope.page = page;
- };
+ $scope.limit = Vipra.config.pageSize;
$scope.$watchGroup(['page','sort','order'], function() {
ArticleFactory.query({
@@ -348,14 +344,10 @@
function($scope, $location, Store, TopicFactory) {
$scope.page = Math.max($location.search().page || 1, 1);
- $scope.limit = Vipra.const.pageSize;
+ $scope.limit = Vipra.config.pageSize;
$scope.sort = Store('sorttopics') || 'name';
$scope.order = Store('ordertopics') || '+';
- $scope.changePage = function(page) {
- $scope.page = page;
- };
-
$scope.$watchGroup(['page','sort','order'], function() {
TopicFactory.query({
skip: ($scope.page-1)*$scope.limit,
@@ -423,14 +415,10 @@
function($scope, $stateParams, $location, Store, TopicFactory) {
$scope.page = Math.max($location.search().page || 1, 1);
- $scope.limit = Vipra.const.pageSize;
+ $scope.limit = Vipra.config.pageSize;
$scope.sort = Store('sortarticles') || 'title';
$scope.order = Store('orderarticles') || '+';
- $scope.changePage = function(page) {
- $scope.page = page;
- };
-
$scope.$watchGroup(['page','sort','order'], function() {
TopicFactory.articles({
id: $stateParams.id,
@@ -446,6 +434,16 @@
}]);
+ /**
+ * Topic Show Similar route
+ */
+ app.controller('TopicsSimilarController', ['$scope', '$stateParams', 'TopicFactory',
+ function($scope, $stateParams, TopicFactory) {
+
+ $scope.typeLabel = $stateParams.type.substring(3);
+
+ }]);
+
/****************************************************************************
* Word Controllers
****************************************************************************/
@@ -461,10 +459,6 @@
$scope.sort = Store('sortwords') || 'id';
$scope.order = Store('orderwords') || '+';
- $scope.changePage = function(page) {
- $scope.page = page;
- };
-
$scope.$watchGroup(['page','sort','order'], function() {
WordFactory.query({
skip: ($scope.page-1)*$scope.limit,
@@ -531,11 +525,8 @@
$scope.calculatePages();
- var change = $scope.change() || function() {};
-
$scope.changePage = function(page) {
$scope.page = page;
- change(page);
window.scrollTo(0,0);
};
diff --git a/vipra-ui/app/js/directives.js b/vipra-ui/app/js/directives.js
index 469d9c48..6493dbce 100644
--- a/vipra-ui/app/js/directives.js
+++ b/vipra-ui/app/js/directives.js
@@ -135,14 +135,17 @@
app.directive('bsDropdown', function() {
return {
+ scope: {
+ label: '@',
+ align: '@'
+ },
templateUrl: 'html/directives/dropdown.html',
transclude: true,
replace: true,
- link: function($scope, $elem, $attrs) {
+ link: function($scope) {
$scope.dropdownId = Vipra.randomId();
- $scope.label = $attrs.label;
$scope.align = 'dropdown-menu-left';
- if($attrs.align === 'right')
+ if($scope.align === 'right')
$scope.align = 'dropdown-menu-right';
}
};
diff --git a/vipra-ui/app/less/app.less b/vipra-ui/app/less/app.less
index da98522a..e48007ed 100644
--- a/vipra-ui/app/less/app.less
+++ b/vipra-ui/app/less/app.less
@@ -126,7 +126,7 @@ ul.dashed {
.graph-legend {
position: absolute;
- top: 60px;
+ top: 97px;
left: 10px;
font-weight: bold;
padding: 10px;
@@ -174,7 +174,7 @@ ul.dashed {
}
.navpadding {
- padding-top: 50px;
+ padding-top: 87px;
}
.form-control-inline {
@@ -245,6 +245,10 @@ ul.dashed {
margin-top: -2px;
}
+.navbar-breadcrumbs {
+ margin-bottom: 0;
+}
+
revolve-select, [revolve-select] {
.noselect;
cursor: pointer;
diff --git a/vipra-ui/bower.json b/vipra-ui/bower.json
index 02a607df..d1495f65 100644
--- a/vipra-ui/bower.json
+++ b/vipra-ui/bower.json
@@ -28,6 +28,7 @@
"angular-websocket": "^1.0.14",
"moment": "^2.11.2",
"nya-bootstrap-select": "^2.1.3",
- "nprogress": "^0.2.0"
+ "nprogress": "^0.2.0",
+ "angular-breadcrumb": "^0.4.1"
}
}
diff --git a/vipra-ui/build.sh b/vipra-ui/build.sh
new file mode 100755
index 00000000..ee2f1c88
--- /dev/null
+++ b/vipra-ui/build.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+rm -rf public
+
+npm install
+bower install
+gulp build
+
+exit 0
diff --git a/vipra-ui/gulpfile.js b/vipra-ui/gulpfile.js
index 4492076a..3a7fd600 100644
--- a/vipra-ui/gulpfile.js
+++ b/vipra-ui/gulpfile.js
@@ -15,6 +15,7 @@ var assets = {
'bower_components/angular-sanitize/angular-sanitize.min.js',
'bower_components/angular-ui-router/release/angular-ui-router.min.js',
'bower_components/angular-websocket/angular-websocket.min.js',
+ 'bower_components/angular-breadcrumb/dist/angular-breadcrumb.min.js',
'bower_components/bootstrap/dist/js/bootstrap.min.js',
'bower_components/highcharts/highcharts.js',
'bower_components/vis/dist/vis.min.js',
diff --git a/vipra-ui/package.json b/vipra-ui/package.json
index 85743ec7..83d14ec7 100644
--- a/vipra-ui/package.json
+++ b/vipra-ui/package.json
@@ -5,7 +5,8 @@
"author": "Eike Cochu",
"private": true,
"devDependencies": {
- "gulp": "^3.9.0",
+ "bower": "^1.7.7",
+ "gulp": "^3.9.1",
"gulp-angular-templatecache": "^1.8.0",
"gulp-concat": "^2.6.0",
"gulp-cssnano": "^2.1.0",
diff --git a/vipra-util/src/main/java/de/vipra/util/ConvertStream.java b/vipra-util/src/main/java/de/vipra/util/ConvertStream.java
deleted file mode 100644
index f7d926ff..00000000
--- a/vipra-util/src/main/java/de/vipra/util/ConvertStream.java
+++ /dev/null
@@ -1,118 +0,0 @@
-package de.vipra.util;
-
-import java.io.BufferedReader;
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.Queue;
-
-import de.vipra.util.ex.NotImplementedException;
-
-/**
- * The convert stream class is used to create a converting stream of objects
- * from a file resource. The file is read sequentially and objects are
- * deserialized from the file contents, according to some convert method of the
- * convert stream. The amount of lines read by the stream is decided by the
- * convert method.
- *
- * @param <T>
- * object type returned by the stream
- */
-public abstract class ConvertStream<T> implements Closeable, AutoCloseable, Iterator<T>, Iterable<T> {
-
- private final BufferedReader reader;
- private Queue<String> buffer;
-
- public ConvertStream(File file) throws FileNotFoundException {
- this.reader = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
- this.buffer = new LinkedList<>();
- }
-
- @Override
- public void close() throws IOException {
- reader.close();
- }
-
- /**
- * Returns the next file line.
- *
- * @return next file line, if available
- */
- protected String nextLine() {
- if (buffer.isEmpty()) {
- try {
- return reader.readLine();
- } catch (IOException e) {}
- }
- return buffer.poll();
- }
-
- /**
- * Push back line into a line buffer, if not processed
- *
- * @param line
- * line to buffer
- */
- protected void buffer(String line) {
- buffer.offer(line);
- }
-
- /**
- * Returns true if next line available. Reads and buffers the next line of
- * the selected file.
- */
- @Override
- public boolean hasNext() {
- String line = null;
- try {
- line = reader.readLine();
- } catch (IOException e) {}
- if (line != null) {
- buffer.offer(line);
- return true;
- }
- return false;
- }
-
- /**
- * Returns the next object in the stream, converted by the convert method
- *
- * @return converted object
- */
- @Override
- public T next() {
- if (buffer.isEmpty()) {
- try {
- return convert(reader.readLine());
- } catch (IOException e) {}
- }
- return convert(buffer.poll());
- }
-
- @Override
- public void remove() {
- throw new NotImplementedException();
- }
-
- @Override
- public Iterator<T> iterator() {
- return this;
- }
-
- /**
- * Convert method. This method is used to deserialize a file line into an
- * object of type T. If more lines are required for deserialization, they
- * can be requested by using next().
- *
- * @param line
- * the line to be converted
- * @return the converted object
- */
- public abstract T convert(String line);
-
-}
diff --git a/vipra-util/src/main/java/de/vipra/util/CountMap.java b/vipra-util/src/main/java/de/vipra/util/CountMap.java
new file mode 100644
index 00000000..45b3d48d
--- /dev/null
+++ b/vipra-util/src/main/java/de/vipra/util/CountMap.java
@@ -0,0 +1,32 @@
+package de.vipra.util;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+public class CountMap<T> {
+
+ private final Map<T, Integer> map;
+
+ public CountMap() {
+ this.map = new HashMap<>();
+ }
+
+ public CountMap(Map<T, Integer> map) {
+ this.map = map;
+ }
+
+ public void count(T t) {
+ Integer count = map.get(t);
+ if (count == null)
+ map.put(t, 1);
+ else
+ map.put(t, count + 1);
+ }
+
+ public Set<Entry<T, Integer>> entrySet() {
+ return map.entrySet();
+ }
+
+}
diff --git a/vipra-util/src/main/java/de/vipra/util/FrequencyList.java b/vipra-util/src/main/java/de/vipra/util/FrequencyList.java
deleted file mode 100644
index f512248d..00000000
--- a/vipra-util/src/main/java/de/vipra/util/FrequencyList.java
+++ /dev/null
@@ -1,27 +0,0 @@
-package de.vipra.util;
-
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-public class FrequencyList<T> implements Iterable<T> {
-
- private Map<T, Integer> map = new LinkedHashMap<>();
-
- public void add(T t) {
- if (map.containsKey(t))
- map.put(t, map.get(t) + 1);
- else
- map.put(t, 1);
- }
-
- public Integer get(T t) {
- return map.get(t);
- }
-
- @Override
- public Iterator<T> iterator() {
- return map.keySet().iterator();
- }
-
-}
diff --git a/vipra-util/src/main/java/de/vipra/util/Pair.java b/vipra-util/src/main/java/de/vipra/util/Pair.java
deleted file mode 100644
index 23c2e378..00000000
--- a/vipra-util/src/main/java/de/vipra/util/Pair.java
+++ /dev/null
@@ -1,35 +0,0 @@
-package de.vipra.util;
-
-public class Pair<X, Y> {
-
- private X x;
- private Y y;
-
- public Pair() {}
-
- public Pair(X x, Y y) {
- this.x = x;
- this.y = y;
- }
-
- public X x() {
- return x;
- }
-
- public void setX(X x) {
- this.x = x;
- }
-
- public Y y() {
- return y;
- }
-
- public void setY(Y y) {
- this.y = y;
- }
-
- public static <X, Y> Pair<X, Y> pair(X x, Y y) {
- return new Pair<>(x, y);
- }
-
-}
diff --git a/vipra-util/src/main/java/de/vipra/util/Tuple.java b/vipra-util/src/main/java/de/vipra/util/Tuple.java
new file mode 100644
index 00000000..fb7995d3
--- /dev/null
+++ b/vipra-util/src/main/java/de/vipra/util/Tuple.java
@@ -0,0 +1,35 @@
+package de.vipra.util;
+
+public class Tuple<X, Y> {
+
+ private X first;
+ private Y second;
+
+ public Tuple() {}
+
+ public Tuple(X first, Y second) {
+ this.first = first;
+ this.second = second;
+ }
+
+ public X first() {
+ return first;
+ }
+
+ public void setFirst(X first) {
+ this.first = first;
+ }
+
+ public Y second() {
+ return second;
+ }
+
+ public void setSecond(Y second) {
+ this.second = second;
+ }
+
+ public static <X, Y> Tuple<X, Y> pair(X first, Y second) {
+ return new Tuple<>(first, second);
+ }
+
+}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java b/vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java
index b3c6b025..ec0e92cf 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java
@@ -36,7 +36,7 @@ public class ArticleFull extends FileModel<ObjectId> implements Serializable {
public static final Logger log = LoggerFactory.getLogger(ArticleFull.class);
@Id
- private ObjectId id;
+ private ObjectId id = new ObjectId();
@ElasticIndex("title")
private String title;
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java b/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java
index 6dd42ea1..c46f5a25 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java
@@ -22,8 +22,10 @@ import de.vipra.util.an.QueryIgnore;
public class TopicFull implements Model<ObjectId>, Serializable {
@Id
- private ObjectId id;
+ private ObjectId id = new ObjectId();
+
private String name;
+
private Integer index;
@Embedded
@@ -34,6 +36,7 @@ public class TopicFull implements Model<ObjectId>, Serializable {
private List<ArticleFull> articles;
private Date created;
+
private Date modified;
@Override
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TopicRef.java b/vipra-util/src/main/java/de/vipra/util/model/TopicRef.java
index c5de02e6..00837669 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/TopicRef.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/TopicRef.java
@@ -4,26 +4,15 @@ import java.io.Serializable;
import org.mongodb.morphia.annotations.Embedded;
import org.mongodb.morphia.annotations.Reference;
-import org.mongodb.morphia.annotations.Transient;
@SuppressWarnings("serial")
@Embedded
public class TopicRef implements Comparable<TopicRef>, Serializable {
- @Transient
- private String topicIndex;
@Reference(ignoreMissing = true)
private Topic topic;
private Integer count;
- public String getTopicIndex() {
- return topicIndex;
- }
-
- public void setTopicIndex(String index) {
- this.topicIndex = index;
- }
-
public Integer getCount() {
return count;
}
@@ -47,8 +36,7 @@ public class TopicRef implements Comparable<TopicRef>, Serializable {
@Override
public String toString() {
- return TopicRef.class.getSimpleName() + "[topicIndex:" + topicIndex + ", topic: " + topic + ", count:" + count
- + "]";
+ return TopicRef.class.getSimpleName() + "[topic: " + topic + ", count:" + count + "]";
}
}
diff --git a/vipra-util/src/main/java/de/vipra/util/service/MongoService.java b/vipra-util/src/main/java/de/vipra/util/service/MongoService.java
index 2f4a4bc4..c975d865 100644
--- a/vipra-util/src/main/java/de/vipra/util/service/MongoService.java
+++ b/vipra-util/src/main/java/de/vipra/util/service/MongoService.java
@@ -13,7 +13,7 @@ import org.mongodb.morphia.query.UpdateOperations;
import de.vipra.util.Config;
import de.vipra.util.ListUtils;
import de.vipra.util.Mongo;
-import de.vipra.util.Pair;
+import de.vipra.util.Tuple;
import de.vipra.util.an.QueryIgnore;
import de.vipra.util.an.UpdateIgnore;
import de.vipra.util.ex.ConfigException;
@@ -75,7 +75,7 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
}
@Override
- public List<Type> getMultiple(Integer skip, Integer limit, String sortBy, Pair<String, Object> criteria,
+ public List<Type> getMultiple(Integer skip, Integer limit, String sortBy, Tuple<String, Object> criteria,
String... fields) {
return getMultiple(
QueryBuilder.builder().skip(skip).limit(limit).sortBy(sortBy).fields(true, fields).criteria(criteria));
@@ -93,8 +93,8 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
if (builder.getSortBy() != null)
query.order(builder.getSortBy());
if (builder.getCriteria() != null)
- for (Pair<String, Object> criteria : builder.getCriteria())
- query.field(criteria.x()).equal(criteria.y());
+ for (Tuple<String, Object> criteria : builder.getCriteria())
+ query.field(criteria.first()).equal(criteria.second());
if (builder.getFields() != null) {
String[] fields = builder.getFields();
if (builder.isInclude()) {
@@ -219,8 +219,8 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
if (builder.getLimit() != null && builder.getLimit() > 0)
query.limit(builder.getLimit());
if (builder.getCriteria() != null)
- for (Pair<String, Object> criteria : builder.getCriteria())
- query.field(criteria.x()).equal(criteria.y());
+ for (Tuple<String, Object> criteria : builder.getCriteria())
+ query.field(criteria.first()).equal(criteria.second());
return datastore.getCount(query);
}
diff --git a/vipra-util/src/main/java/de/vipra/util/service/Service.java b/vipra-util/src/main/java/de/vipra/util/service/Service.java
index 3e9aff75..136e5758 100644
--- a/vipra-util/src/main/java/de/vipra/util/service/Service.java
+++ b/vipra-util/src/main/java/de/vipra/util/service/Service.java
@@ -3,7 +3,7 @@ package de.vipra.util.service;
import java.util.ArrayList;
import java.util.List;
-import de.vipra.util.Pair;
+import de.vipra.util.Tuple;
import de.vipra.util.model.Model;
/**
@@ -39,7 +39,7 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
/**
* @see {@link Service#getMultiple(QueryBuilder)}
*/
- List<Type> getMultiple(Integer skip, Integer limit, String sortBy, Pair<String, Object> criteria, String... fields)
+ List<Type> getMultiple(Integer skip, Integer limit, String sortBy, Tuple<String, Object> criteria, String... fields)
throws E;
/**
@@ -158,7 +158,7 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
private Integer skip;
private Integer limit;
private String sortBy;
- private List<Pair<String, Object>> criteria;
+ private List<Tuple<String, Object>> criteria;
private String[] fields;
private boolean include;
@@ -218,7 +218,7 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
*/
public QueryBuilder criteria(String field, Object value) {
if (field != null && value != null && !field.isEmpty())
- criteria(Pair.pair(field, value));
+ criteria(Tuple.pair(field, value));
return this;
}
@@ -229,7 +229,7 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
* field value pair to compare
* @return QueryBuilder instance
*/
- public QueryBuilder criteria(Pair<String, Object> pair) {
+ public QueryBuilder criteria(Tuple<String, Object> pair) {
if (pair != null) {
if (criteria == null) {
criteria = new ArrayList<>();
@@ -269,7 +269,7 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
return sortBy.startsWith("+") ? sortBy.substring(1) : sortBy;
}
- public List<Pair<String, Object>> getCriteria() {
+ public List<Tuple<String, Object>> getCriteria() {
return criteria;
}
--
GitLab