diff --git a/build.sh b/build.sh
index 8de1b5776f40819a305762907ad316347afc2718..63f31a0df005942ec80000b28b82125853735837 100755
--- a/build.sh
+++ b/build.sh
@@ -50,6 +50,19 @@ if [ $? -ne 0 ]; then
exit 1
fi
+echo "" >> $LOG
+echo "-------------------------------" >> $LOG
+echo "compiling vipra-ui" | tee -a $LOG
+echo "-------------------------------" >> $LOG
+cd ./vipra-ui
+./build.sh >> $LOG 2>&1
+cd ..
+cp -r ./vipra-ui/public/* ./vipra-backend/src/main/webapp
+if [ $? -ne 0 ]; then
+ echo "error"
+ exit 1
+fi
+
echo "" >> $LOG
echo "-------------------------------" >> $LOG
echo "compiling vipra-backend" | tee -a $LOG
diff --git a/vipra-backend/src/main/java/de/vipra/rest/resource/TopicResource.java b/vipra-backend/src/main/java/de/vipra/rest/resource/TopicResource.java
index acbbf0a9ab25495d457c59b0f4662a8d1cbfe50a..de832f21cf8d821a796ea498249ae96d1bc91a7e 100644
--- a/vipra-backend/src/main/java/de/vipra/rest/resource/TopicResource.java
+++ b/vipra-backend/src/main/java/de/vipra/rest/resource/TopicResource.java
@@ -79,7 +79,6 @@ public class TopicResource {
@GET
@Produces(MediaType.APPLICATION_JSON)
- @Consumes(MediaType.APPLICATION_JSON)
@Path("{id}")
public Response getTopic(@PathParam("id") String id, @QueryParam("fields") String fields)
throws ConfigException, IOException {
@@ -110,7 +109,6 @@ public class TopicResource {
@GET
@Produces(MediaType.APPLICATION_JSON)
- @Consumes(MediaType.APPLICATION_JSON)
@Path("{id}/articles")
public Response getArticles(@PathParam("id") String id, @QueryParam("skip") Integer skip,
@QueryParam("limit") Integer limit, @QueryParam("sort") @DefaultValue("title") String sortBy,
@@ -138,6 +136,26 @@ public class TopicResource {
}
}
+ @GET
+ @Produces(MediaType.APPLICATION_JSON)
+ @Path("{id}/similar/by-words")
+ public Response similarTopicsByWords(@PathParam("id") String id, @QueryParam("skip") Integer skip,
+ @QueryParam("limit") Integer limit, @QueryParam("sort") @DefaultValue("title") String sortBy,
+ @QueryParam("fields") String fields) {
+ // TODO implement
+ return null;
+ }
+
+ @GET
+ @Produces(MediaType.APPLICATION_JSON)
+ @Path("{id}/similar/by-articles")
+ public Response similarTopicsByArticles(@PathParam("id") String id, @QueryParam("skip") Integer skip,
+ @QueryParam("limit") Integer limit, @QueryParam("sort") @DefaultValue("title") String sortBy,
+ @QueryParam("fields") String fields) {
+ // TODO implement
+ return null;
+ }
+
@PUT
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
diff --git a/vipra-backend/src/main/webapp/WEB-INF/web.xml b/vipra-backend/src/main/webapp/WEB-INF/web.xml
index 90458a45e4c3c71a3e1d60ac21de704cec850c0f..d28582b4638dc6242b05aa7ad3d72027a8865922 100644
--- a/vipra-backend/src/main/webapp/WEB-INF/web.xml
+++ b/vipra-backend/src/main/webapp/WEB-INF/web.xml
@@ -1,15 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
-<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://xmlns.jcp.org/xml/ns/javaee" xsi:schemaLocation="http://xmlns.jcp.org/xml/ns/javaee http://xmlns.jcp.org/xml/ns/javaee/web-app_3_1.xsd" version="3.1">
- <servlet>
- <servlet-name>jersey</servlet-name>
- <servlet-class>org.glassfish.jersey.servlet.ServletContainer</servlet-class>
- <init-param>
- <param-name>javax.ws.rs.Application</param-name>
- <param-value>de.vipra.rest.Application</param-value>
- </init-param>
- </servlet>
- <servlet-mapping>
- <servlet-name>jersey</servlet-name>
- <url-pattern>/rest/*</url-pattern>
- </servlet-mapping>
+<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://xmlns.jcp.org/xml/ns/javaee"
+ xsi:schemaLocation="http://xmlns.jcp.org/xml/ns/javaee http://xmlns.jcp.org/xml/ns/javaee/web-app_3_1.xsd"
+ version="3.1">
+ <welcome-file-list>
+ <welcome-file>index.html</welcome-file>
+ </welcome-file-list>
+ <servlet>
+ <servlet-name>jersey</servlet-name>
+ <servlet-class>org.glassfish.jersey.servlet.ServletContainer</servlet-class>
+ <init-param>
+ <param-name>javax.ws.rs.Application</param-name>
+ <param-value>de.vipra.rest.Application</param-value>
+ </init-param>
+ </servlet>
+ <servlet-mapping>
+ <servlet-name>jersey</servlet-name>
+ <url-pattern>/rest/*</url-pattern>
+ </servlet-mapping>
</web-app>
\ No newline at end of file
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/CmdOptions.java b/vipra-cmd/src/main/java/de/vipra/cmd/CmdOptions.java
index 0610a0a427573eb4b68408745ce611f602f955a7..8158313b39dfe0da7a857ddfa62dd7870f83cbcb 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/CmdOptions.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/CmdOptions.java
@@ -41,6 +41,9 @@ public class CmdOptions extends Options {
public static final String OPT_MODELING = "m";
public static final String OPT_MODELING_LONG = "modeling";
+ public static final String OPT_INDEXING = "e";
+ public static final String OPT_INDEXING_LONG = "indexing";
+
public CmdOptions() {
addOption(Option.builder(OPT_HELP).longOpt(OPT_HELP_LONG).desc("print this message").build());
addOption(Option.builder(OPT_SHELL).longOpt(OPT_SHELL_LONG).hasArg(true).argName("name")
@@ -56,6 +59,7 @@ public class CmdOptions extends Options {
addOption(Option.builder(OPT_SILENT).longOpt(OPT_SILENT_LONG).desc("mute all output").build());
addOption(Option.builder(OPT_CONFIG).longOpt(OPT_CONFIG_LONG).desc("show configuration").build());
addOption(Option.builder(OPT_MODELING).longOpt(OPT_MODELING_LONG).desc("regenerate topic model").build());
+ addOption(Option.builder(OPT_INDEXING).longOpt(OPT_INDEXING_LONG).desc("regenerate search index").build());
}
public void printHelp(String cmd) {
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/Main.java b/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
index a883948673f8e76a1c787140d3a2467b70ea3214..70842fdfb932a45b41f6439d519b90e51d75afb4 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
@@ -11,6 +11,7 @@ import static de.vipra.cmd.CmdOptions.OPT_SHELL;
import static de.vipra.cmd.CmdOptions.OPT_SILENT;
import static de.vipra.cmd.CmdOptions.OPT_STATS;
import static de.vipra.cmd.CmdOptions.OPT_TEST;
+import static de.vipra.cmd.CmdOptions.OPT_INDEXING;
import java.util.ArrayList;
import java.util.List;
@@ -37,6 +38,7 @@ import de.vipra.cmd.option.ClearCommand;
import de.vipra.cmd.option.Command;
import de.vipra.cmd.option.ConfigCommand;
import de.vipra.cmd.option.ImportCommand;
+import de.vipra.cmd.option.IndexingCommand;
import de.vipra.cmd.option.ModelingCommand;
import de.vipra.cmd.option.StatsCommand;
import de.vipra.cmd.option.TestCommand;
@@ -112,6 +114,9 @@ public class Main {
if (cline.hasOption(OPT_MODELING))
commands.add(new ModelingCommand());
+ if (cline.hasOption(OPT_INDEXING))
+ commands.add(new IndexingCommand());
+
if (cline.hasOption(OPT_STATS))
commands.add(new StatsCommand());
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/lda/Analyzer.java b/vipra-cmd/src/main/java/de/vipra/cmd/lda/Analyzer.java
index 01cc8d15cddb786f91a45bac51a23b5be7a2eae8..1d8e0f205efadb14922cec6876b737c28ec3fe9a 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/lda/Analyzer.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/lda/Analyzer.java
@@ -1,13 +1,8 @@
package de.vipra.cmd.lda;
-import java.util.List;
-
import de.vipra.cmd.ex.AnalyzerException;
import de.vipra.util.Config;
-import de.vipra.util.ConvertStream;
import de.vipra.util.WordMap;
-import de.vipra.util.model.TopicFull;
-import de.vipra.util.model.TopicRef;
public abstract class Analyzer {
@@ -25,27 +20,6 @@ public abstract class Analyzer {
public abstract void analyze() throws AnalyzerException;
- /**
- * Returns a converting stream of topics, read from the topic definition
- * file. Usually, a topic definition consists of a list of words, that are
- * assigned to that topic with a certain likeliness.
- *
- * @return topic definition stream
- * @throws AnalyzerException
- */
- public abstract ConvertStream<TopicFull> getTopicDefinitions() throws AnalyzerException;
-
- /**
- * Returns a converting stream of lists of topic references. Normally, topic
- * modeling outputs topics for each word of each document. These references
- * are returned by this function.
- *
- * @return stream of lists of topic references per document (ordered by
- * index)
- * @throws AnalyzerException
- */
- public abstract ConvertStream<List<TopicRef>> getTopics() throws AnalyzerException;
-
public static Analyzer getAnalyzer(Config config, WordMap wordMap) throws AnalyzerException {
Analyzer analyzer = null;
switch (config.analyzer) {
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/lda/DTMAnalyzer.java b/vipra-cmd/src/main/java/de/vipra/cmd/lda/DTMAnalyzer.java
index 54d2a8b82ab9fc171cb9551f826da8dd19501242..306ed2726d6d9db08aa9bc2bfa7b36b3a814e9c8 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/lda/DTMAnalyzer.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/lda/DTMAnalyzer.java
@@ -4,27 +4,26 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
-import java.util.List;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import de.vipra.cmd.ex.AnalyzerException;
-import de.vipra.cmd.file.DTMVocabulary;
import de.vipra.util.Config;
import de.vipra.util.Constants;
-import de.vipra.util.ConvertStream;
import de.vipra.util.StringUtils;
import de.vipra.util.WordMap;
import de.vipra.util.ex.ConfigException;
-import de.vipra.util.model.TopicFull;
-import de.vipra.util.model.TopicRef;
public class DTMAnalyzer extends Analyzer {
public static final Logger log = LogManager.getLogger(DTMAnalyzer.class);
public static final String NAME = "dtm";
+ public static final int dynamicMinIter = 100;
+ public static final int dynamicMaxIter = 1000;
+ public static final int staticIter = 100;
+
private String command;
private File modelDir;
private File outDir;
@@ -68,11 +67,11 @@ public class DTMAnalyzer extends Analyzer {
// alpha (default -10)
"--alpha=0.01",
// minimum number if iterations
- "--lda_sequence_min_iter=5",
+ "--lda_sequence_min_iter=" + dynamicMinIter,
// maximum number of iterations
- "--lda_sequence_max_iter=10",
+ "--lda_sequence_max_iter=" + dynamicMaxIter,
// em iter (default 20)
- "--lda_max_em_iter=20",
+ "--lda_max_em_iter=" + staticIter,
// input file prefix
"--corpus_prefix=" + corpusPrefix,
// output directory
@@ -101,32 +100,10 @@ public class DTMAnalyzer extends Analyzer {
in.close();
p.waitFor();
- log.info("done");
+ // TODO save model
} catch (IOException | InterruptedException e) {
throw new AnalyzerException(e);
}
}
- @Override
- public ConvertStream<TopicFull> getTopicDefinitions() throws AnalyzerException {
- DTMVocabulary vocab;
- try {
- vocab = new DTMVocabulary(modelDir, false);
- } catch (IOException e) {
- throw new AnalyzerException(e);
- }
- return null;
- }
-
- @Override
- public ConvertStream<List<TopicRef>> getTopics() throws AnalyzerException {
- DTMVocabulary vocab;
- try {
- vocab = new DTMVocabulary(modelDir, false);
- } catch (IOException e) {
- throw new AnalyzerException(e);
- }
- return null;
- }
-
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbAnalyzer.java b/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbAnalyzer.java
index 8fa94370cfc54739ff0c6b3d6b4803a3fff82dc7..3af056a2d683fd06fb27bc17b88c84b5923bf60d 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbAnalyzer.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbAnalyzer.java
@@ -1,29 +1,38 @@
package de.vipra.cmd.lda;
+import java.io.BufferedReader;
import java.io.File;
-import java.io.FileNotFoundException;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import org.bson.types.ObjectId;
import de.vipra.cmd.ex.AnalyzerException;
+import de.vipra.cmd.file.FilebaseIndex;
import de.vipra.util.Config;
import de.vipra.util.Constants;
-import de.vipra.util.ConvertStream;
-import de.vipra.util.NumberUtils;
-import de.vipra.util.StringUtils;
+import de.vipra.util.CountMap;
+import de.vipra.util.FileUtils;
import de.vipra.util.WordMap;
import de.vipra.util.ex.ConfigException;
+import de.vipra.util.ex.DatabaseException;
+import de.vipra.util.model.ArticleFull;
+import de.vipra.util.model.Topic;
import de.vipra.util.model.TopicFull;
import de.vipra.util.model.TopicRef;
import de.vipra.util.model.TopicWord;
import de.vipra.util.model.Word;
+import de.vipra.util.service.MongoService;
import jgibblda.Estimator;
import jgibblda.LDACmdOption;
@@ -36,7 +45,9 @@ public class JGibbAnalyzer extends Analyzer {
private File modelDir;
private File modelFile;
private LDACmdOption options;
- private WordMap wordMap;
+ private MongoService<ArticleFull, ObjectId> dbArticles;
+ private MongoService<TopicFull, ObjectId> dbTopics;
+ private FilebaseIndex index;
protected JGibbAnalyzer() {
super("JGibb Analyzer");
@@ -64,7 +75,14 @@ public class JGibbAnalyzer extends Analyzer {
options.modelName = NAME;
- this.wordMap = wordMap;
+ try {
+ config = Config.getConfig();
+ dbArticles = MongoService.getDatabaseService(config, ArticleFull.class);
+ dbTopics = MongoService.getDatabaseService(config, TopicFull.class);
+ index = new FilebaseIndex(new File(modelDir, "index"));
+ } catch (Exception e) {
+ throw new AnalyzerException(e);
+ }
}
@Override
@@ -75,103 +93,88 @@ public class JGibbAnalyzer extends Analyzer {
Estimator estimator = new Estimator();
estimator.init(options);
estimator.estimate();
- }
- @Override
- public ConvertStream<TopicFull> getTopicDefinitions() throws AnalyzerException {
+ // read topic definitions and save
+
File twords = new File(modelDir, NAME + ".twords");
+ List<String> lines;
try {
- return new ConvertStream<TopicFull>(twords) {
- @Override
- public TopicFull convert(String line) {
- TopicFull topicDef = new TopicFull();
- List<TopicWord> topicWords = new ArrayList<>();
-
- // get index of topic
- // the index will be used as a temporary id until the topic
- // is created in the database
- Integer index = StringUtils.getFirstNumber(line);
- if (index == null) {
- log.error("could not extract topic index from line: " + line);
- } else {
- topicDef.setIndex(index);
- }
-
- // get all lines that follow until the next topic is
- // discovered (line does not start with a tab). Cut off
- // after k words are gathered.
- String nextLine;
- while ((nextLine = nextLine()) != null) {
- if (nextLine.startsWith("\t")) {
- String[] parts = nextLine.trim().split("\\s+");
- try {
- Word word = wordMap.get(parts[0]);
- // round likeliness precision
- double likeliness = NumberUtils.roundToPrecision(Double.parseDouble(parts[1]),
- Constants.LIKELINESS_PRECISION);
-
- // check if word likely enough to relate to
- // topic
- if (likeliness < Constants.MINIMUM_LIKELINESS)
- continue;
-
- TopicWord topicWord = new TopicWord(word, likeliness);
- topicWords.add(topicWord);
- } catch (NumberFormatException e) {
- log.error("could not parse number in line: " + nextLine);
- } catch (ArrayIndexOutOfBoundsException e) {
- log.error("ill formatted topic definition in line: " + nextLine);
- }
- } else {
- buffer(nextLine);
- break;
- }
- }
- Collections.sort(topicWords, Collections.reverseOrder());
- topicDef.setWords(topicWords);
- topicDef.setName(TopicFull.getNameFromWords(topicWords));
- return topicDef;
- }
- };
- } catch (FileNotFoundException e) {
+ lines = FileUtils.readFile(twords);
+ } catch (IOException e) {
throw new AnalyzerException(e);
}
- }
- @Override
- public ConvertStream<List<TopicRef>> getTopics() throws AnalyzerException {
+ List<TopicFull> newTopics = new ArrayList<>(options.K);
+ Map<Integer, Topic> newTopicsMap = new HashMap<>(options.K);
+
+ // for each topic
+ for (int topicNum = 0; topicNum < options.K; topicNum++) {
+ TopicFull newTopic = new TopicFull();
+ List<TopicWord> topicWords = new ArrayList<>(options.twords);
+
+ // for each word
+ for (int wordNum = 0, lineNum = topicNum * options.K + 1; wordNum < options.twords; wordNum++, lineNum++) {
+ String[] parts = lines.get(lineNum).trim().split("\\s+");
+ TopicWord topicWord = new TopicWord(new Word(parts[0]), Double.parseDouble(parts[1]));
+ topicWords.add(topicWord);
+ }
+
+ newTopic.setWords(topicWords);
+ newTopics.add(newTopic);
+ newTopicsMap.put(topicNum, new Topic(newTopic.getId()));
+ }
+
+ dbTopics.drop();
+ try {
+ dbTopics.createMultiple(newTopics);
+ } catch (DatabaseException e) {
+ throw new AnalyzerException(e);
+ }
+
+ // read documents and reference topics
+
File tassign = new File(modelDir, NAME + ".tassign");
+ Pattern topicIndexPattern = Pattern.compile(":(\\d+)");
+ BufferedReader in = null;
+
try {
- return new ConvertStream<List<TopicRef>>(tassign) {
- @Override
- public List<TopicRef> convert(String line) {
- // count topics
- Map<String, Integer> countMap = new HashMap<>();
- String[] wordList = line.split("\\s+");
- for (String word : wordList) {
- // the file uses a simple integer as topic ids. Use this
- // id as a temporary id until the topic is created in
- // the database
- String topic = word.split(":")[1];
- Integer count = countMap.get(topic);
- countMap.put(topic, count == null ? 1 : count + 1);
- }
-
- // turn into list
- List<TopicRef> topicCount = new ArrayList<>(countMap.size());
- for (Entry<String, Integer> e : countMap.entrySet()) {
- TopicRef tc = new TopicRef();
- tc.setTopicIndex(e.getKey());
- tc.setCount(e.getValue());
- topicCount.add(tc);
- }
-
- Collections.sort(topicCount, Collections.reverseOrder());
-
- return topicCount;
+ in = new BufferedReader(new InputStreamReader(new FileInputStream(tassign)));
+ String line;
+ int articleIndex = 0;
+
+ // each line in the tassign file is a document, formatted with
+ // <word-id>:<topic-id>
+ while ((line = in.readLine()) != null) {
+ // extract topic ids and count them
+ CountMap<String> countMap = new CountMap<>();
+ Matcher matcher = topicIndexPattern.matcher(line);
+ while (matcher.find()) {
+ countMap.count(matcher.group(1));
+ }
+
+ // create list of topics refs referencing topics with counted
+ // occurrences
+ List<TopicRef> newTopicRefs = new ArrayList<>();
+ for (Entry<String, Integer> entry : countMap.entrySet()) {
+ TopicRef ref = new TopicRef();
+ ref.setCount(entry.getValue());
+ ref.setTopic(newTopicsMap.get(Integer.parseInt(entry.getKey())));
+ newTopicRefs.add(ref);
+ }
+
+ // update article with topic references (partial update)
+ ArticleFull article = new ArticleFull();
+ article.setId(index.get(articleIndex++));
+ article.setTopics(newTopicRefs);
+ try {
+ // TODO: using field name here. Hard to refactor
+ dbArticles.updateSingle(article, "topics");
+ } catch (DatabaseException e) {
+ log.error(e);
}
- };
- } catch (FileNotFoundException e) {
+ }
+ in.close();
+ } catch (IOException e) {
throw new AnalyzerException(e);
}
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/IndexingCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/IndexingCommand.java
new file mode 100644
index 0000000000000000000000000000000000000000..29977e57cc16f76126474999bbf89194d79fd2e5
--- /dev/null
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/IndexingCommand.java
@@ -0,0 +1,54 @@
+package de.vipra.cmd.option;
+
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.bson.types.ObjectId;
+import org.elasticsearch.client.Client;
+
+import de.vipra.cmd.file.Filebase;
+import de.vipra.cmd.file.FilebaseIndex;
+import de.vipra.util.Config;
+import de.vipra.util.ESClient;
+import de.vipra.util.ESSerializer;
+import de.vipra.util.MongoUtils;
+import de.vipra.util.model.ArticleFull;
+import de.vipra.util.service.MongoService;
+
+public class IndexingCommand implements Command {
+
+ public static final Logger log = LogManager.getLogger(IndexingCommand.class);
+
+ @Override
+ public void run() throws Exception {
+ Config config = Config.getConfig();
+ MongoService<ArticleFull, ObjectId> dbArticles = MongoService.getDatabaseService(config, ArticleFull.class);
+ Filebase filebase = Filebase.getFilebase(config);
+ FilebaseIndex index = filebase.getIndex();
+ Client elasticClient = ESClient.getClient(config);
+ ESSerializer<ArticleFull> elasticSerializer = new ESSerializer<>(ArticleFull.class);
+
+ // clear index
+ elasticClient.admin().indices().prepareDelete("_all").get();
+
+ Iterator<String> indexIter = index.iterator();
+ while (indexIter.hasNext()) {
+ // get article from database
+ String id = indexIter.next();
+ ArticleFull article = dbArticles.getSingle(MongoUtils.objectId(id), true);
+ if (article == null) {
+ log.error("no article found in db for id " + id);
+ continue;
+ }
+
+ // index article
+ Map<String, Object> source = elasticSerializer.serialize(article);
+ elasticClient.prepareIndex("articles", "article", article.getId().toString()).setSource(source).get();
+ }
+
+ elasticClient.close();
+ }
+
+}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ModelingCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ModelingCommand.java
index 0bb226b5828f0cf59488a70a4ce94c63eb941058..fd5a5a6e68371b7abe63509a21f6487d3d5861c0 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ModelingCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ModelingCommand.java
@@ -1,35 +1,13 @@
package de.vipra.cmd.option;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import org.bson.types.ObjectId;
-import org.elasticsearch.client.Client;
-import de.vipra.cmd.file.Filebase;
-import de.vipra.cmd.file.FilebaseIndex;
import de.vipra.cmd.lda.Analyzer;
import de.vipra.util.Config;
-import de.vipra.util.Constants;
-import de.vipra.util.ConvertStream;
-import de.vipra.util.ESClient;
-import de.vipra.util.ESSerializer;
-import de.vipra.util.MongoUtils;
import de.vipra.util.StringUtils;
import de.vipra.util.Timer;
import de.vipra.util.WordMap;
-import de.vipra.util.ex.DatabaseException;
-import de.vipra.util.model.ArticleFull;
-import de.vipra.util.model.Topic;
-import de.vipra.util.model.TopicFull;
-import de.vipra.util.model.TopicRef;
import de.vipra.util.model.Word;
import de.vipra.util.service.MongoService;
@@ -38,26 +16,16 @@ public class ModelingCommand implements Command {
public static final Logger log = LogManager.getLogger(ModelingCommand.class);
private Config config;
- private MongoService<ArticleFull, ObjectId> dbArticles;
- private MongoService<TopicFull, ObjectId> dbTopics;
private MongoService<Word, String> dbWords;
- private Filebase filebase;
private WordMap wordMap;
private Analyzer analyzer;
- private Client elasticClient;
- private ESSerializer<ArticleFull> elasticSerializer;
@Override
public void run() throws Exception {
config = Config.getConfig();
- dbArticles = MongoService.getDatabaseService(config, ArticleFull.class);
- dbTopics = MongoService.getDatabaseService(config, TopicFull.class);
dbWords = MongoService.getDatabaseService(config, Word.class);
- filebase = Filebase.getFilebase(config);
wordMap = new WordMap(dbWords);
analyzer = Analyzer.getAnalyzer(config, wordMap);
- elasticClient = ESClient.getClient(config);
- elasticSerializer = new ESSerializer<>(ArticleFull.class);
log.info("using analyzer: " + analyzer.getName());
@@ -71,94 +39,9 @@ public class ModelingCommand implements Command {
analyzer.analyze();
timer.lap("topic modeling");
- /*
- * save topic model
- */
- log.info("saving topic definitions");
- int batchSize = 100;
- ConvertStream<TopicFull> topicDefs = analyzer.getTopicDefinitions();
- Map<String, TopicFull> topicIndexMap = new HashMap<>();
- dbTopics.drop();
- List<TopicFull> newTopicDefs = new ArrayList<>(batchSize);
- List<Topic> newTopicRefs = new ArrayList<>();
- Iterator<TopicFull> it = topicDefs.iterator();
- while (it.hasNext()) {
- newTopicDefs.add(it.next());
- if (newTopicDefs.size() == batchSize || !it.hasNext()) {
- dbTopics.createMultiple(newTopicDefs);
- for (TopicFull newTopicDef : newTopicDefs) {
- topicIndexMap.put(Integer.toString(newTopicDef.getIndex()), newTopicDef);
- newTopicRefs.add(new Topic(newTopicDef.getId()));
- }
- newTopicDefs.clear();
- }
- }
- timer.lap("saving topics");
-
- /*
- * save topic refs and index article
- */
- log.info("saving document topics");
- ConvertStream<List<TopicRef>> topicStream = analyzer.getTopics();
- FilebaseIndex index = filebase.getIndex();
- Iterator<String> indexIter = index.iterator();
- Iterator<List<TopicRef>> topicRefsListIter = topicStream.iterator();
- elasticClient.admin().indices().prepareDelete("_all").get();
- while (indexIter.hasNext() && topicRefsListIter.hasNext()) {
- // get article from database
- String id = indexIter.next();
- ArticleFull article = dbArticles.getSingle(MongoUtils.objectId(id), true);
- if (article == null) {
- log.error("no article found in db for id " + id);
- continue;
- }
-
- double wordCount = article.getStats().getWordCount();
-
- // insert topic references into article, ignoring low refs
- List<TopicRef> topicRefs = topicRefsListIter.next();
- for (ListIterator<TopicRef> topicRefsIter = topicRefs.listIterator(); topicRefsIter.hasNext();) {
- TopicRef topicRef = topicRefsIter.next();
- if ((topicRef.getCount() / wordCount) < Constants.TOPIC_THRESHOLD) {
- topicRefsIter.remove();
- continue;
- }
- TopicFull topicFull = topicIndexMap.get(topicRef.getTopicIndex());
- if (topicFull != null) {
- topicRef.setTopic(new Topic(topicFull.getId(), topicFull.getName()));
- } else
- log.error("no object id for topic index " + topicRef.getTopicIndex());
- }
-
- article.setTopics(topicRefs);
-
- try {
- dbArticles.replaceSingle(article);
- } catch (DatabaseException e) {
- log.error("could not update article: " + article.getTitle() + " (" + article.getId() + ")");
- }
-
- // index article
- Map<String, Object> source = elasticSerializer.serialize(article);
- elasticClient.prepareIndex("articles", "article", article.getId().toString()).setSource(source).get();
- }
-
- /*
- * save words
- */
- log.info("saving words");
- Set<Word> importedWords = wordMap.getNewWords();
- timer.lap("saving topic refs and indexing");
- wordMap.create();
- timer.lap("saving words");
-
- elasticClient.close();
-
/*
* run information
*/
- int newWordsCount = importedWords.size();
- log.info("imported " + newWordsCount + " new " + StringUtils.quantity(newWordsCount, "word"));
log.info(timer.toString());
log.info("done in " + StringUtils.timeString(timer.total()));
}
diff --git a/vipra-cmd/src/main/resources/config.properties b/vipra-cmd/src/main/resources/config.properties
index ca312cfdab39ca456a9fd539ec23088496d59dac..01b9444d6837e0520e7f4edf60af96e78bb7c300 100644
--- a/vipra-cmd/src/main/resources/config.properties
+++ b/vipra-cmd/src/main/resources/config.properties
@@ -3,5 +3,4 @@ db.port=27017
db.name=test
tm.processor=corenlp
tm.analyzer=jgibb
-tm.saveallwords=false
-tm.dtmpath=/home/eike/Downloads/dtm_release/dtm/main
\ No newline at end of file
+tm.dtmpath=/home/eike/repos/master/dtm_release/dtm/main
\ No newline at end of file
diff --git a/vipra-ui/app/html/about.html b/vipra-ui/app/html/about.html
index edd025ffe06df95478efccaa5c932c837f3ae770..d84cf03509d831517129befc3076fbca0c7f7df7 100644
--- a/vipra-ui/app/html/about.html
+++ b/vipra-ui/app/html/about.html
@@ -1,4 +1,4 @@
-<div ng-cloak>
+<div ng-cloak ng-hide="$state.current.name !== 'about'">
<div class="page-header">
<h1>About</h1>
</div>
@@ -261,4 +261,6 @@
</tr>
</tbody>
</table>
-</div>
\ No newline at end of file
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/articles/index.html b/vipra-ui/app/html/articles/index.html
index 103bfb96843f75b14e4cf03b0e073e50997fe6f5..f30bfbbc44e085411a99750e1e910c0abd1d3a3f 100644
--- a/vipra-ui/app/html/articles/index.html
+++ b/vipra-ui/app/html/articles/index.html
@@ -1,4 +1,4 @@
-<div ng-cloak>
+<div ng-cloak ng-hide="$state.current.name !== 'articles'">
<div class="text-muted">
Found <ng-pluralize count="articlesTotal||0" when="{0:'no articles',1:'1 article',other:'{} articles'}"></ng-pluralize> in the database.
<span ng-show="articlesTotal">
@@ -26,5 +26,7 @@
</li>
</ol>
- <pagination total="articlesTotal" page="page" limit="limit" change="changePage"/>
-</div>
\ No newline at end of file
+ <pagination total="articlesTotal" page="page" limit="limit"/>
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/articles/show.html b/vipra-ui/app/html/articles/show.html
index 27afa34735a1fe8eba0d27a8b4f4c62782560564..fe66fe1eaafb118c581229df35df1bc67a2f0c22 100644
--- a/vipra-ui/app/html/articles/show.html
+++ b/vipra-ui/app/html/articles/show.html
@@ -1,4 +1,4 @@
-<div ng-cloak>
+<div ng-cloak ng-hide="$state.current.name !== 'articles.show'">
<div class="page-header">
<h1 ng-bind="::article.title"></h1>
@@ -82,4 +82,6 @@
<hr>
<p ng-bind-html="::article.text" class="text-justify"></p>
-</div>
\ No newline at end of file
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/index.html b/vipra-ui/app/html/index.html
index cbbe697bc86ad5b52503860aa688b10e81a4aa4b..833c9fccfcd6fe90b73a8fc4ac6da439044cf08f 100644
--- a/vipra-ui/app/html/index.html
+++ b/vipra-ui/app/html/index.html
@@ -1,4 +1,4 @@
-<div ng-cloak>
+<div ng-cloak ng-hide="$state.current.name !== 'index'">
<div class="row" ng-hide="search">
<div class="col-md-12">
<div class="heading"></div>
@@ -59,4 +59,6 @@
</ul>
</div>
</div>
-</div>
\ No newline at end of file
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/modal.html b/vipra-ui/app/html/modal.html
deleted file mode 100644
index e441d0bf66c3142a7db352366f576f175590b6bf..0000000000000000000000000000000000000000
--- a/vipra-ui/app/html/modal.html
+++ /dev/null
@@ -1,11 +0,0 @@
-<div class="modal-header">
- <h3 class="modal-title">I'm a modal!</h3>
-</div>
-
-<div class="modal-body">
-</div>
-
-<div class="modal-footer">
- <button class="btn btn-primary" type="button" ng-click="ok()">OK</button>
- <button class="btn btn-warning" type="button" ng-click="cancel()">Cancel</button>
-</div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/network.html b/vipra-ui/app/html/network.html
index b05ce3981550f677285fce56df92b31528175799..431188c673e36bff7cbeac9a8f128fe7e4cafeac 100644
--- a/vipra-ui/app/html/network.html
+++ b/vipra-ui/app/html/network.html
@@ -1,4 +1,4 @@
-<div ng-cloak>
+<div ng-cloak ng-hide="$state.current.name !== 'network'">
<div class="fullsize navpadding">
<div class="graph-legend overlay">
<label style="color:{{colors.articles}}">
@@ -13,4 +13,6 @@
</div>
<div class="fullsize navpadding" id="visgraph"></div>
</div>
-</div>
\ No newline at end of file
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/topics/articles.html b/vipra-ui/app/html/topics/articles.html
new file mode 100644
index 0000000000000000000000000000000000000000..2818921cda1a2cf80db1614a12885872bcc802a5
--- /dev/null
+++ b/vipra-ui/app/html/topics/articles.html
@@ -0,0 +1,44 @@
+<div ng-cloak ng-hide="$state.current.name !== 'topics.show.articles'">
+ <div class="page-header">
+ <h1 ng-bind-template="Articles for topic '{{::topic.name}}'"></h1>
+
+ <table class="item-actions">
+ <tr>
+ <td>
+ <a class="btn btn-default" ui-sref="^">Back</a>
+ </td>
+ </tr>
+ </table>
+ </div>
+
+ <div class="text-muted">
+ Found <ng-pluralize count="articlesTotal||0" when="{0:'no articles',1:'1 article',other:'{} articles'}"></ng-pluralize> in the database.
+ <span ng-show="articlesTotal">
+ Sort by
+ <ol class="nya-bs-select nya-bs-condensed" ng-model="sort" ng-model-store="sort" ng-model-default="'date'">
+ <li value="title" class="nya-bs-option"><a>Title</a></li>
+ <li value="date" class="nya-bs-option"><a>Date</a></li>
+ <li value="created" class="nya-bs-option"><a>Added</a></li>
+ </ol>
+ Direction
+ <ol class="nya-bs-select nya-bs-condensed" ng-model="order" ng-model-store="order" ng-model-default="'+'">
+ <li value="+" class="nya-bs-option"><a>Ascending</a></li>
+ <li value="-" class="nya-bs-option"><a>Descending</a></li>
+ </ol>
+ </span>
+ <br>
+ Page <span ng-bind="page||1"></span> of <span ng-bind="maxPage||1"></span>.
+ </div>
+
+ <pagination total="articlesTotal" page="page" limit="limit" change="changePage"/>
+
+ <ol ng-attr-start="{{(page-1)*limit+1}}">
+ <li ng-repeat="article in articles">
+ <a ui-sref="articles.show({id: article.id})" ng-bind="::article.title"></a>
+ </li>
+ </ol>
+
+ <pagination total="articlesTotal" page="page" limit="limit"/>
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/topics/index.html b/vipra-ui/app/html/topics/index.html
index 8d5a249654d34abc170ce3b082260889ad84a9b0..b4273145dfc78a05af30d7a6e22f19a8c7627f48 100644
--- a/vipra-ui/app/html/topics/index.html
+++ b/vipra-ui/app/html/topics/index.html
@@ -1,4 +1,4 @@
-<div ng-cloak>
+<div ng-cloak ng-hide="$state.current.name !== 'topics'">
<div class="text-muted">
Found <ng-pluralize count="topicsTotal||0" when="{0:'no topics',1:'1 topic',other:'{} topics'}"></ng-pluralize> in the database.
<span ng-show="topicsTotal">
@@ -25,5 +25,7 @@
</li>
</ol>
- <pagination total="topicsTotal" page="page" limit="limit" change="changePage"/>
-</div>
\ No newline at end of file
+ <pagination total="topicsTotal" page="page" limit="limit"/>
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/topics/show.html b/vipra-ui/app/html/topics/show.html
index 964d974382856a7a1d3d1c6d998e644e76daf2ec..2875d8c765214019dfe568660fd63c7425078340 100644
--- a/vipra-ui/app/html/topics/show.html
+++ b/vipra-ui/app/html/topics/show.html
@@ -1,4 +1,4 @@
-<div ng-cloak>
+<div ng-cloak ng-hide="$state.current.name !== 'topics.show'">
<div class="page-header">
<h1>
<div ng-bind="topic.name" ng-hide="isRename"></div>
@@ -29,6 +29,15 @@
<td>
<a class="btn btn-default" ui-sref="network({type:'topics', id:topic.id})">Network graph</a>
</td>
+ <td>
+ <a class="btn btn-default" ui-sref="topics.show.articles({id:topic.id})">Articles</a>
+ </td>
+ <td>
+ <bs-dropdown label="Similar Topics">
+ <li><a ui-sref="topics.show.similar({id:topic.id, type:'by-words'})">By word share</a></li>
+ <li><a ui-sref="topics.show.similar({id:topic.id, type:'by-articles'})">By article share</a></li>
+ </bs-dropdown>
+ </td>
</tr>
</table>
</div>
@@ -84,4 +93,6 @@
</table>
</div>
</div>
-</div>
\ No newline at end of file
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/topics/similar.html b/vipra-ui/app/html/topics/similar.html
new file mode 100644
index 0000000000000000000000000000000000000000..e868dc6e1f8aa1b4e8685edf84d8f9826297a6fa
--- /dev/null
+++ b/vipra-ui/app/html/topics/similar.html
@@ -0,0 +1,5 @@
+<div ng-cloak ng-hide="$state.current.name !== 'topics.show.similar'">
+
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/words/index.html b/vipra-ui/app/html/words/index.html
index 5da54c2da1f787f3e4e2ed9686bc80fc63c52923..d89dba089efdb583f7b3145e8762c685205510a7 100644
--- a/vipra-ui/app/html/words/index.html
+++ b/vipra-ui/app/html/words/index.html
@@ -1,4 +1,4 @@
-<div ng-cloak>
+<div ng-cloak ng-hide="$state.current.name !== 'words'">
<div class="text-muted">
Found <ng-pluralize count="wordsTotal||0" when="{0:'no words',1:'1 word',other:'{} words'}"></ng-pluralize> in the database.
<span ng-show="wordsTotal">
@@ -43,5 +43,7 @@
</div>
</div>
- <pagination total="wordsTotal" page="page" limit="limit" change="changePage"/>
-</div>
\ No newline at end of file
+ <pagination total="wordsTotal" page="page" limit="limit"/>
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/words/show.html b/vipra-ui/app/html/words/show.html
index 891b8f3d74e1bf00e2fa551ca1024aa3b633accd..d1ca723b17d991032cbd7df8eb0819cc4689f21c 100644
--- a/vipra-ui/app/html/words/show.html
+++ b/vipra-ui/app/html/words/show.html
@@ -1,4 +1,4 @@
-<div ng-cloak>
+<div ng-cloak ng-hide="$state.current.name !== 'words.show'">
<div class="page-header">
<h1 ng-bind="::word.id"></h1>
</div>
@@ -29,4 +29,6 @@
</ol>
</div>
</div>
-</div>
\ No newline at end of file
+</div>
+
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/index.html b/vipra-ui/app/index.html
index b4995c130191a52e091396f76f60039060c5372f..dbb7057b110438f78add64361f9c41fbd51d8ad7 100644
--- a/vipra-ui/app/index.html
+++ b/vipra-ui/app/index.html
@@ -35,7 +35,7 @@
<script src="js/templates.js"></script>
</head>
<body>
- <nav class="navbar navbar-default navbar-static-top">
+ <nav class="navbar navbar-default navbar-static-top navbar-breadcrumbs">
<div class="container-fluid">
<!-- Brand and toggle get grouped for better mobile display -->
<div class="navbar-header">
@@ -73,6 +73,8 @@
</div><!-- /.container-fluid -->
</nav>
+ <div ncy-breadcrumb ng-hide="$state.current.name === 'index'"></div>
+
<div class="container" ui-view ng-cloak></div>
</body>
</html>
\ No newline at end of file
diff --git a/vipra-ui/app/js/app.js b/vipra-ui/app/js/app.js
index bfb10682a41c91bacb933c2e6ac5c6d34ac1efaf..f348a0cbc29534af05d49b58093e8ee2484b0fa5 100644
--- a/vipra-ui/app/js/app.js
+++ b/vipra-ui/app/js/app.js
@@ -10,6 +10,7 @@
'ngWebSocket',
'ui.router',
'nya.bootstrap.select',
+ 'ncy-angular-breadcrumb',
'vipra.controllers',
'vipra.directives',
'vipra.factories',
@@ -25,37 +26,48 @@
$stateProvider.state('index', {
url: '/',
- templateUrl: Vipra.const.tplBase + '/index.html',
- controller: 'IndexController'
+ templateUrl: 'html/index.html',
+ controller: 'IndexController',
+ ncyBreadcrumb: {
+ label: 'Start'
+ }
});
$stateProvider.state('about', {
url: '/about',
- templateUrl: Vipra.const.tplBase + '/about.html',
- controller: 'AboutController'
+ templateUrl: 'html/about.html',
+ controller: 'AboutController',
+ ncyBreadcrumb: {
+ label: 'About'
+ }
});
$stateProvider.state('network', {
url: '/network/:type/:id',
- templateUrl: Vipra.const.tplBase + '/network.html',
- controller: 'NetworkController'
+ templateUrl: 'html/network.html',
+ controller: 'NetworkController',
+ ncyBreadcrumb: {
+ label: 'Network'
+ }
});
// states: articles
$stateProvider.state('articles', {
url: '/articles',
- templateUrl: Vipra.const.tplBase + '/articles/index.html',
- controller: 'ArticlesIndexController'
+ templateUrl: 'html/articles/index.html',
+ controller: 'ArticlesIndexController',
+ ncyBreadcrumb: {
+ label: 'Articles'
+ }
});
$stateProvider.state('articles.show', {
url: '/:id',
- views: {
- "@": {
- templateUrl: Vipra.const.tplBase + '/articles/show.html',
- controller: 'ArticlesShowController'
- }
+ templateUrl: 'html/articles/show.html',
+ controller: 'ArticlesShowController',
+ ncyBreadcrumb: {
+ label: '{{article.title.ellipsize(30)}}'
}
});
@@ -63,17 +75,37 @@
$stateProvider.state('topics', {
url: '/topics',
- templateUrl: Vipra.const.tplBase + '/topics/index.html',
- controller: 'TopicsIndexController'
+ templateUrl: 'html/topics/index.html',
+ controller: 'TopicsIndexController',
+ ncyBreadcrumb: {
+ label: 'Topics'
+ }
});
$stateProvider.state('topics.show', {
url: '/:id',
- "views": {
- "@": {
- templateUrl: Vipra.const.tplBase + '/topics/show.html',
- controller: 'TopicsShowController'
- }
+ templateUrl: 'html/topics/show.html',
+ controller: 'TopicsShowController',
+ ncyBreadcrumb: {
+ label: '{{topic.name}}'
+ }
+ });
+
+ $stateProvider.state('topics.show.articles', {
+ url: '/articles',
+ templateUrl: 'html/topics/articles.html',
+ controller: 'TopicsArticlesController',
+ ncyBreadcrumb: {
+ label: 'Topic Articles'
+ }
+ });
+
+ $stateProvider.state('topics.show.similar', {
+ url: '/similar/:type',
+ templateUrl: 'html/topics/similar.html',
+ controller: 'TopicsSimilarController',
+ ncyBreadcrumb: {
+ label: 'Similar Topics (by {{typeLabel}})'
}
});
@@ -81,17 +113,19 @@
$stateProvider.state('words', {
url: '/words',
- templateUrl: Vipra.const.tplBase + '/words/index.html',
- controller: 'WordsIndexController'
+ templateUrl: 'html/words/index.html',
+ controller: 'WordsIndexController',
+ ncyBreadcrumb: {
+ label: 'Words'
+ }
});
$stateProvider.state('words.show', {
url: '/:id',
- "views": {
- "@": {
- templateUrl: Vipra.const.tplBase + '/words/show.html',
- controller: 'WordsShowController'
- }
+ templateUrl: 'html/words/show.html',
+ controller: 'WordsShowController',
+ ncyBreadcrumb: {
+ label: '{{word.id}}'
}
});
diff --git a/vipra-ui/app/js/config.js b/vipra-ui/app/js/config.js
index 643b5e335d4b33eaea31940ee2c5ba76725c47b0..e6f400bdc4fce283becb170cc122199f6b995a1c 100644
--- a/vipra-ui/app/js/config.js
+++ b/vipra-ui/app/js/config.js
@@ -4,7 +4,10 @@
Vipra.config = {
restUrl: '//' + location.hostname + ':8000/vipra/rest',
- websocketUrl: 'ws://' + location.hostname + ':8000/vipra/ws'
+ websocketUrl: 'ws://' + location.hostname + ':8000/vipra/ws',
+ latestItems: 3,
+ searchResults: 10,
+ pageSize: 100
};
})();
\ No newline at end of file
diff --git a/vipra-ui/app/js/constants.js b/vipra-ui/app/js/constants.js
deleted file mode 100644
index b60fe8af4e2d41e875fc7bf7a33b7bb6cda0fb65..0000000000000000000000000000000000000000
--- a/vipra-ui/app/js/constants.js
+++ /dev/null
@@ -1,12 +0,0 @@
-(function() {
-
- window.Vipra = window.Vipra || {};
-
- Vipra.const = {
- tplBase: 'html',
- latestItems: 3,
- searchResults: 10,
- pageSize: 100
- };
-
-})();
\ No newline at end of file
diff --git a/vipra-ui/app/js/controllers.js b/vipra-ui/app/js/controllers.js
index 5fa9f2e55136d44d10ea17369a97697399aa06c9..daa5d471b04c7e2f80328aa6d625323760ea07d1 100644
--- a/vipra-ui/app/js/controllers.js
+++ b/vipra-ui/app/js/controllers.js
@@ -21,15 +21,15 @@
$scope.search = $location.search().query;
- ArticleFactory.query({limit:Vipra.const.latestItems, sort:'-created'}, function(data) {
+ ArticleFactory.query({limit:Vipra.config.latestItems, sort:'-created'}, function(data) {
$scope.latestArticles = data;
});
- TopicFactory.query({limit:Vipra.const.latestItems, sort:'-created'}, function(data) {
+ TopicFactory.query({limit:Vipra.config.latestItems, sort:'-created'}, function(data) {
$scope.latestTopics = data;
});
- WordFactory.query({limit:Vipra.const.latestItems, sort:'-created'}, function(data) {
+ WordFactory.query({limit:Vipra.config.latestItems, sort:'-created'}, function(data) {
$scope.latestWords = data;
});
@@ -37,7 +37,7 @@
if($scope.search) {
$location.search('query', $scope.search);
$scope.searching = true;
- SearchFactory.query({limit:Vipra.const.searchResults, query:$scope.search}, function(data) {
+ SearchFactory.query({limit:Vipra.config.searchResults, query:$scope.search}, function(data) {
$scope.searching = false;
$scope.searchResults = data;
});
@@ -266,11 +266,7 @@
function($scope, $state, $location, ArticleFactory, Store) {
$scope.page = Math.max($location.search().page || 1, 1);
- $scope.limit = Vipra.const.pageSize;
-
- $scope.changePage = function(page) {
- $scope.page = page;
- };
+ $scope.limit = Vipra.config.pageSize;
$scope.$watchGroup(['page','sort','order'], function() {
ArticleFactory.query({
@@ -348,14 +344,10 @@
function($scope, $location, Store, TopicFactory) {
$scope.page = Math.max($location.search().page || 1, 1);
- $scope.limit = Vipra.const.pageSize;
+ $scope.limit = Vipra.config.pageSize;
$scope.sort = Store('sorttopics') || 'name';
$scope.order = Store('ordertopics') || '+';
- $scope.changePage = function(page) {
- $scope.page = page;
- };
-
$scope.$watchGroup(['page','sort','order'], function() {
TopicFactory.query({
skip: ($scope.page-1)*$scope.limit,
@@ -423,14 +415,10 @@
function($scope, $stateParams, $location, Store, TopicFactory) {
$scope.page = Math.max($location.search().page || 1, 1);
- $scope.limit = Vipra.const.pageSize;
+ $scope.limit = Vipra.config.pageSize;
$scope.sort = Store('sortarticles') || 'title';
$scope.order = Store('orderarticles') || '+';
- $scope.changePage = function(page) {
- $scope.page = page;
- };
-
$scope.$watchGroup(['page','sort','order'], function() {
TopicFactory.articles({
id: $stateParams.id,
@@ -446,6 +434,16 @@
}]);
+ /**
+ * Topic Show Similar route
+ */
+ app.controller('TopicsSimilarController', ['$scope', '$stateParams', 'TopicFactory',
+ function($scope, $stateParams, TopicFactory) {
+
+ $scope.typeLabel = $stateParams.type.substring(3);
+
+ }]);
+
/****************************************************************************
* Word Controllers
****************************************************************************/
@@ -461,10 +459,6 @@
$scope.sort = Store('sortwords') || 'id';
$scope.order = Store('orderwords') || '+';
- $scope.changePage = function(page) {
- $scope.page = page;
- };
-
$scope.$watchGroup(['page','sort','order'], function() {
WordFactory.query({
skip: ($scope.page-1)*$scope.limit,
@@ -531,11 +525,8 @@
$scope.calculatePages();
- var change = $scope.change() || function() {};
-
$scope.changePage = function(page) {
$scope.page = page;
- change(page);
window.scrollTo(0,0);
};
diff --git a/vipra-ui/app/js/directives.js b/vipra-ui/app/js/directives.js
index 469d9c48f55fa5c8b75dca076836cd5b3d551307..6493dbce115f60b8a6bb7002e431a27d4adabf02 100644
--- a/vipra-ui/app/js/directives.js
+++ b/vipra-ui/app/js/directives.js
@@ -135,14 +135,17 @@
app.directive('bsDropdown', function() {
return {
+ scope: {
+ label: '@',
+ align: '@'
+ },
templateUrl: 'html/directives/dropdown.html',
transclude: true,
replace: true,
- link: function($scope, $elem, $attrs) {
+ link: function($scope) {
$scope.dropdownId = Vipra.randomId();
- $scope.label = $attrs.label;
$scope.align = 'dropdown-menu-left';
- if($attrs.align === 'right')
+ if($scope.align === 'right')
$scope.align = 'dropdown-menu-right';
}
};
diff --git a/vipra-ui/app/less/app.less b/vipra-ui/app/less/app.less
index da98522ab4bc21f5643e493bded0c5ec7b589b85..e48007ed6ae8b5d351f37b99c54d835452d7df8c 100644
--- a/vipra-ui/app/less/app.less
+++ b/vipra-ui/app/less/app.less
@@ -126,7 +126,7 @@ ul.dashed {
.graph-legend {
position: absolute;
- top: 60px;
+ top: 97px;
left: 10px;
font-weight: bold;
padding: 10px;
@@ -174,7 +174,7 @@ ul.dashed {
}
.navpadding {
- padding-top: 50px;
+ padding-top: 87px;
}
.form-control-inline {
@@ -245,6 +245,10 @@ ul.dashed {
margin-top: -2px;
}
+.navbar-breadcrumbs {
+ margin-bottom: 0;
+}
+
revolve-select, [revolve-select] {
.noselect;
cursor: pointer;
diff --git a/vipra-ui/bower.json b/vipra-ui/bower.json
index 02a607dfe4a39d53e7e20146dd11c1846c966cba..d1495f65d49fb7395fcbf9fad62dcccbbcf10930 100644
--- a/vipra-ui/bower.json
+++ b/vipra-ui/bower.json
@@ -28,6 +28,7 @@
"angular-websocket": "^1.0.14",
"moment": "^2.11.2",
"nya-bootstrap-select": "^2.1.3",
- "nprogress": "^0.2.0"
+ "nprogress": "^0.2.0",
+ "angular-breadcrumb": "^0.4.1"
}
}
diff --git a/vipra-ui/build.sh b/vipra-ui/build.sh
new file mode 100755
index 0000000000000000000000000000000000000000..ee2f1c881fd2a425cb23141b67e71594b77bcc08
--- /dev/null
+++ b/vipra-ui/build.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+rm -rf public
+
+npm install
+bower install
+gulp build
+
+exit 0
diff --git a/vipra-ui/gulpfile.js b/vipra-ui/gulpfile.js
index 4492076aeb769beeb4c7d40d78af24d173687b70..3a7fd6005849079257082c03082524e389ae61bf 100644
--- a/vipra-ui/gulpfile.js
+++ b/vipra-ui/gulpfile.js
@@ -15,6 +15,7 @@ var assets = {
'bower_components/angular-sanitize/angular-sanitize.min.js',
'bower_components/angular-ui-router/release/angular-ui-router.min.js',
'bower_components/angular-websocket/angular-websocket.min.js',
+ 'bower_components/angular-breadcrumb/dist/angular-breadcrumb.min.js',
'bower_components/bootstrap/dist/js/bootstrap.min.js',
'bower_components/highcharts/highcharts.js',
'bower_components/vis/dist/vis.min.js',
diff --git a/vipra-ui/package.json b/vipra-ui/package.json
index 85743ec707d7cd144babcb70fa153651f9443f49..83d14ec7469960bd0bd8025638c23662432dad1e 100644
--- a/vipra-ui/package.json
+++ b/vipra-ui/package.json
@@ -5,7 +5,8 @@
"author": "Eike Cochu",
"private": true,
"devDependencies": {
- "gulp": "^3.9.0",
+ "bower": "^1.7.7",
+ "gulp": "^3.9.1",
"gulp-angular-templatecache": "^1.8.0",
"gulp-concat": "^2.6.0",
"gulp-cssnano": "^2.1.0",
diff --git a/vipra-util/src/main/java/de/vipra/util/ConvertStream.java b/vipra-util/src/main/java/de/vipra/util/ConvertStream.java
deleted file mode 100644
index f7d926ffb0cf7f499971c6630f873f49c8dd84ac..0000000000000000000000000000000000000000
--- a/vipra-util/src/main/java/de/vipra/util/ConvertStream.java
+++ /dev/null
@@ -1,118 +0,0 @@
-package de.vipra.util;
-
-import java.io.BufferedReader;
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.Queue;
-
-import de.vipra.util.ex.NotImplementedException;
-
-/**
- * The convert stream class is used to create a converting stream of objects
- * from a file resource. The file is read sequentially and objects are
- * deserialized from the file contents, according to some convert method of the
- * convert stream. The amount of lines read by the stream is decided by the
- * convert method.
- *
- * @param <T>
- * object type returned by the stream
- */
-public abstract class ConvertStream<T> implements Closeable, AutoCloseable, Iterator<T>, Iterable<T> {
-
- private final BufferedReader reader;
- private Queue<String> buffer;
-
- public ConvertStream(File file) throws FileNotFoundException {
- this.reader = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
- this.buffer = new LinkedList<>();
- }
-
- @Override
- public void close() throws IOException {
- reader.close();
- }
-
- /**
- * Returns the next file line.
- *
- * @return next file line, if available
- */
- protected String nextLine() {
- if (buffer.isEmpty()) {
- try {
- return reader.readLine();
- } catch (IOException e) {}
- }
- return buffer.poll();
- }
-
- /**
- * Push back line into a line buffer, if not processed
- *
- * @param line
- * line to buffer
- */
- protected void buffer(String line) {
- buffer.offer(line);
- }
-
- /**
- * Returns true if next line available. Reads and buffers the next line of
- * the selected file.
- */
- @Override
- public boolean hasNext() {
- String line = null;
- try {
- line = reader.readLine();
- } catch (IOException e) {}
- if (line != null) {
- buffer.offer(line);
- return true;
- }
- return false;
- }
-
- /**
- * Returns the next object in the stream, converted by the convert method
- *
- * @return converted object
- */
- @Override
- public T next() {
- if (buffer.isEmpty()) {
- try {
- return convert(reader.readLine());
- } catch (IOException e) {}
- }
- return convert(buffer.poll());
- }
-
- @Override
- public void remove() {
- throw new NotImplementedException();
- }
-
- @Override
- public Iterator<T> iterator() {
- return this;
- }
-
- /**
- * Convert method. This method is used to deserialize a file line into an
- * object of type T. If more lines are required for deserialization, they
- * can be requested by using next().
- *
- * @param line
- * the line to be converted
- * @return the converted object
- */
- public abstract T convert(String line);
-
-}
diff --git a/vipra-util/src/main/java/de/vipra/util/CountMap.java b/vipra-util/src/main/java/de/vipra/util/CountMap.java
new file mode 100644
index 0000000000000000000000000000000000000000..45b3d48d1769a83276a0e126b71ab3502bd465b4
--- /dev/null
+++ b/vipra-util/src/main/java/de/vipra/util/CountMap.java
@@ -0,0 +1,32 @@
+package de.vipra.util;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+public class CountMap<T> {
+
+ private final Map<T, Integer> map;
+
+ public CountMap() {
+ this.map = new HashMap<>();
+ }
+
+ public CountMap(Map<T, Integer> map) {
+ this.map = map;
+ }
+
+ public void count(T t) {
+ Integer count = map.get(t);
+ if (count == null)
+ map.put(t, 1);
+ else
+ map.put(t, count + 1);
+ }
+
+ public Set<Entry<T, Integer>> entrySet() {
+ return map.entrySet();
+ }
+
+}
diff --git a/vipra-util/src/main/java/de/vipra/util/FrequencyList.java b/vipra-util/src/main/java/de/vipra/util/FrequencyList.java
deleted file mode 100644
index f512248d66c7d35192eb9e7f668749ae6d8b1632..0000000000000000000000000000000000000000
--- a/vipra-util/src/main/java/de/vipra/util/FrequencyList.java
+++ /dev/null
@@ -1,27 +0,0 @@
-package de.vipra.util;
-
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-public class FrequencyList<T> implements Iterable<T> {
-
- private Map<T, Integer> map = new LinkedHashMap<>();
-
- public void add(T t) {
- if (map.containsKey(t))
- map.put(t, map.get(t) + 1);
- else
- map.put(t, 1);
- }
-
- public Integer get(T t) {
- return map.get(t);
- }
-
- @Override
- public Iterator<T> iterator() {
- return map.keySet().iterator();
- }
-
-}
diff --git a/vipra-util/src/main/java/de/vipra/util/Pair.java b/vipra-util/src/main/java/de/vipra/util/Pair.java
deleted file mode 100644
index 23c2e378094845c731aa2a7bbf90b1caa50ac8eb..0000000000000000000000000000000000000000
--- a/vipra-util/src/main/java/de/vipra/util/Pair.java
+++ /dev/null
@@ -1,35 +0,0 @@
-package de.vipra.util;
-
-public class Pair<X, Y> {
-
- private X x;
- private Y y;
-
- public Pair() {}
-
- public Pair(X x, Y y) {
- this.x = x;
- this.y = y;
- }
-
- public X x() {
- return x;
- }
-
- public void setX(X x) {
- this.x = x;
- }
-
- public Y y() {
- return y;
- }
-
- public void setY(Y y) {
- this.y = y;
- }
-
- public static <X, Y> Pair<X, Y> pair(X x, Y y) {
- return new Pair<>(x, y);
- }
-
-}
diff --git a/vipra-util/src/main/java/de/vipra/util/Tuple.java b/vipra-util/src/main/java/de/vipra/util/Tuple.java
new file mode 100644
index 0000000000000000000000000000000000000000..fb7995d3b592f1d0e7715d5dd50d1a29dfddcfc5
--- /dev/null
+++ b/vipra-util/src/main/java/de/vipra/util/Tuple.java
@@ -0,0 +1,35 @@
+package de.vipra.util;
+
+public class Tuple<X, Y> {
+
+ private X first;
+ private Y second;
+
+ public Tuple() {}
+
+ public Tuple(X first, Y second) {
+ this.first = first;
+ this.second = second;
+ }
+
+ public X first() {
+ return first;
+ }
+
+ public void setFirst(X first) {
+ this.first = first;
+ }
+
+ public Y second() {
+ return second;
+ }
+
+ public void setSecond(Y second) {
+ this.second = second;
+ }
+
+ public static <X, Y> Tuple<X, Y> pair(X first, Y second) {
+ return new Tuple<>(first, second);
+ }
+
+}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java b/vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java
index b3c6b025faeb103cd2937ca8d73869b53d003d95..ec0e92cf76332c85b4d344942645a2aaef0e05d5 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java
@@ -36,7 +36,7 @@ public class ArticleFull extends FileModel<ObjectId> implements Serializable {
public static final Logger log = LoggerFactory.getLogger(ArticleFull.class);
@Id
- private ObjectId id;
+ private ObjectId id = new ObjectId();
@ElasticIndex("title")
private String title;
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java b/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java
index 6dd42ea1657e720875c51a2cb8591f45057e6a20..c46f5a25597b65cfd09bdd7ee02f004846c5f718 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java
@@ -22,8 +22,10 @@ import de.vipra.util.an.QueryIgnore;
public class TopicFull implements Model<ObjectId>, Serializable {
@Id
- private ObjectId id;
+ private ObjectId id = new ObjectId();
+
private String name;
+
private Integer index;
@Embedded
@@ -34,6 +36,7 @@ public class TopicFull implements Model<ObjectId>, Serializable {
private List<ArticleFull> articles;
private Date created;
+
private Date modified;
@Override
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TopicRef.java b/vipra-util/src/main/java/de/vipra/util/model/TopicRef.java
index c5de02e6b750df507e7d8d0f9b15bc2eed307f21..00837669c2ddee92726a0e63f0f557b418206754 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/TopicRef.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/TopicRef.java
@@ -4,26 +4,15 @@ import java.io.Serializable;
import org.mongodb.morphia.annotations.Embedded;
import org.mongodb.morphia.annotations.Reference;
-import org.mongodb.morphia.annotations.Transient;
@SuppressWarnings("serial")
@Embedded
public class TopicRef implements Comparable<TopicRef>, Serializable {
- @Transient
- private String topicIndex;
@Reference(ignoreMissing = true)
private Topic topic;
private Integer count;
- public String getTopicIndex() {
- return topicIndex;
- }
-
- public void setTopicIndex(String index) {
- this.topicIndex = index;
- }
-
public Integer getCount() {
return count;
}
@@ -47,8 +36,7 @@ public class TopicRef implements Comparable<TopicRef>, Serializable {
@Override
public String toString() {
- return TopicRef.class.getSimpleName() + "[topicIndex:" + topicIndex + ", topic: " + topic + ", count:" + count
- + "]";
+ return TopicRef.class.getSimpleName() + "[topic: " + topic + ", count:" + count + "]";
}
}
diff --git a/vipra-util/src/main/java/de/vipra/util/service/MongoService.java b/vipra-util/src/main/java/de/vipra/util/service/MongoService.java
index 2f4a4bc485c40fc76f7d819f78ee46aaecb7aae5..c975d865029b23dcf1eced9520887bed77c5f0d9 100644
--- a/vipra-util/src/main/java/de/vipra/util/service/MongoService.java
+++ b/vipra-util/src/main/java/de/vipra/util/service/MongoService.java
@@ -13,7 +13,7 @@ import org.mongodb.morphia.query.UpdateOperations;
import de.vipra.util.Config;
import de.vipra.util.ListUtils;
import de.vipra.util.Mongo;
-import de.vipra.util.Pair;
+import de.vipra.util.Tuple;
import de.vipra.util.an.QueryIgnore;
import de.vipra.util.an.UpdateIgnore;
import de.vipra.util.ex.ConfigException;
@@ -75,7 +75,7 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
}
@Override
- public List<Type> getMultiple(Integer skip, Integer limit, String sortBy, Pair<String, Object> criteria,
+ public List<Type> getMultiple(Integer skip, Integer limit, String sortBy, Tuple<String, Object> criteria,
String... fields) {
return getMultiple(
QueryBuilder.builder().skip(skip).limit(limit).sortBy(sortBy).fields(true, fields).criteria(criteria));
@@ -93,8 +93,8 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
if (builder.getSortBy() != null)
query.order(builder.getSortBy());
if (builder.getCriteria() != null)
- for (Pair<String, Object> criteria : builder.getCriteria())
- query.field(criteria.x()).equal(criteria.y());
+ for (Tuple<String, Object> criteria : builder.getCriteria())
+ query.field(criteria.first()).equal(criteria.second());
if (builder.getFields() != null) {
String[] fields = builder.getFields();
if (builder.isInclude()) {
@@ -219,8 +219,8 @@ public class MongoService<Type extends Model<IdType>, IdType> implements Service
if (builder.getLimit() != null && builder.getLimit() > 0)
query.limit(builder.getLimit());
if (builder.getCriteria() != null)
- for (Pair<String, Object> criteria : builder.getCriteria())
- query.field(criteria.x()).equal(criteria.y());
+ for (Tuple<String, Object> criteria : builder.getCriteria())
+ query.field(criteria.first()).equal(criteria.second());
return datastore.getCount(query);
}
diff --git a/vipra-util/src/main/java/de/vipra/util/service/Service.java b/vipra-util/src/main/java/de/vipra/util/service/Service.java
index 3e9aff75b8cd10cf9d73e01c854463d3c1c23cd7..136e57580ddad941af8fb1632c29c4818192ad69 100644
--- a/vipra-util/src/main/java/de/vipra/util/service/Service.java
+++ b/vipra-util/src/main/java/de/vipra/util/service/Service.java
@@ -3,7 +3,7 @@ package de.vipra.util.service;
import java.util.ArrayList;
import java.util.List;
-import de.vipra.util.Pair;
+import de.vipra.util.Tuple;
import de.vipra.util.model.Model;
/**
@@ -39,7 +39,7 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
/**
* @see {@link Service#getMultiple(QueryBuilder)}
*/
- List<Type> getMultiple(Integer skip, Integer limit, String sortBy, Pair<String, Object> criteria, String... fields)
+ List<Type> getMultiple(Integer skip, Integer limit, String sortBy, Tuple<String, Object> criteria, String... fields)
throws E;
/**
@@ -158,7 +158,7 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
private Integer skip;
private Integer limit;
private String sortBy;
- private List<Pair<String, Object>> criteria;
+ private List<Tuple<String, Object>> criteria;
private String[] fields;
private boolean include;
@@ -218,7 +218,7 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
*/
public QueryBuilder criteria(String field, Object value) {
if (field != null && value != null && !field.isEmpty())
- criteria(Pair.pair(field, value));
+ criteria(Tuple.pair(field, value));
return this;
}
@@ -229,7 +229,7 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
* field value pair to compare
* @return QueryBuilder instance
*/
- public QueryBuilder criteria(Pair<String, Object> pair) {
+ public QueryBuilder criteria(Tuple<String, Object> pair) {
if (pair != null) {
if (criteria == null) {
criteria = new ArrayList<>();
@@ -269,7 +269,7 @@ public interface Service<Type extends Model<IdType>, IdType, E extends Exception
return sortBy.startsWith("+") ? sortBy.substring(1) : sortBy;
}
- public List<Pair<String, Object>> getCriteria() {
+ public List<Tuple<String, Object>> getCriteria() {
return criteria;
}