diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000000000000000000000000000000000000..522d49d72c2e8e4e9fde2bf7e9161a2110d45138
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1 @@
+*.ser.gz -crlf
diff --git a/HypernymReader/src/de/vipra/hypernym/HypernymReader.java b/HypernymReader/src/de/vipra/hypernym/HypernymReader.java
index 8275bc6fd403bb89294729b49ca3cc5fee1d1ee9..fc7f68d32b9741e7eddb03d17c8bcdf2340b5f18 100644
--- a/HypernymReader/src/de/vipra/hypernym/HypernymReader.java
+++ b/HypernymReader/src/de/vipra/hypernym/HypernymReader.java
@@ -21,7 +21,7 @@ import org.apache.commons.lang3.StringEscapeUtils;
public class HypernymReader {
public static final String IN_FILE_PATH = "/home/eike/Downloads/en.lhd.extension.2015-10.nt";
- public static final String OUT_FILE_PATH = "/home/eike/Downloads/hypernyms.gz";
+ public static final String OUT_FILE_PATH = "/home/eike/Downloads/hypernyms.ser.gz";
public static final Pattern LINE_PATTERN = Pattern
.compile("<http://dbpedia.org/resource/([^>]+)>\\s+<([^>]+)>\\s+<http://dbpedia.org/resource/([^>]+)>");
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 4259dcf87ead2ef2104dc508a6281bd158cf8370..4bd762bbba675c1b5697dcd63b4b6a518ca31fcd 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -20,5 +20,5 @@ COPY elasticsearch.yml /elasticsearch/config/elasticsearch.yml
COPY nginx.conf /etc/nginx/nginx.conf
COPY webapps /tomcat/webapps/
COPY webroot /webroot/
-EXPOSE 80 9300 27017
+EXPOSE 80 9200 9300 27017
CMD ["/usr/bin/supervisord"]
\ No newline at end of file
diff --git a/docker/docker-run.sh b/docker/docker-run.sh
index 9dc49a2ce0761f14db0df9376cc6d58c221ae4cd..645fd946e3bb5fb83258edb73092ad193368ab4a 100755
--- a/docker/docker-run.sh
+++ b/docker/docker-run.sh
@@ -5,6 +5,7 @@
# set host machine ports
HOST_NGINX=80
+HOST_ELASTICSEARCH_REST=9200
HOST_ELASTICSEARCH_API=9300
HOST_MONGODB=27017
@@ -15,7 +16,7 @@ REPLACE_WEBROOT=1
#######################################################################################
DIR="$(dirname "$(readlink -f "$0")")"
-PORT_MAPPING="-p $HOST_NGINX:80 -p $HOST_ELASTICSEARCH_API:9300 -p $HOST_MONGODB:27017"
+PORT_MAPPING="-p $HOST_NGINX:80 -p $HOST_ELASTICSEARCH_REST:9200 -p $HOST_ELASTICSEARCH_API:9300 -p $HOST_MONGODB:27017"
if [ $REPLACE_WEBAPPS -eq 1 ]; then
VOLUME_WEBAPPS="-v $DIR/webapps:/tomcat/webapps"
diff --git a/docker/elasticsearch.yml b/docker/elasticsearch.yml
index ba6a829ef79a6245c2d2c9e09f3ea963d8f911cb..863e544762a4e46fd8d35ff8bf2c6efee8866018 100644
--- a/docker/elasticsearch.yml
+++ b/docker/elasticsearch.yml
@@ -93,4 +93,5 @@
#
# action.destructive_requires_name: true
-network.host: 0.0.0.0
\ No newline at end of file
+network.host: 0.0.0.0
+http.cors.enabled: true
\ No newline at end of file
diff --git a/docker/nginx.conf b/docker/nginx.conf
index 30f7582560fe99069235ae99779f9ab3fd730cee..264192aed86531b45ac9f761002eaa1d416151fa 100644
--- a/docker/nginx.conf
+++ b/docker/nginx.conf
@@ -46,5 +46,13 @@ http {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass http://localhost:8080/rest/;
}
+
+ location /es/ {
+ proxy_buffering off;
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_pass http://localhost:9200/;
+ }
}
}
\ No newline at end of file
diff --git a/vipra-backend/src/main/java/de/vipra/rest/resource/ArticleResource.java b/vipra-backend/src/main/java/de/vipra/rest/resource/ArticleResource.java
index 3fafdd4b752c4211196a323198be3356283055be..43a83f7ad6b834d3636c9301c5c3522a8313c3bd 100644
--- a/vipra-backend/src/main/java/de/vipra/rest/resource/ArticleResource.java
+++ b/vipra-backend/src/main/java/de/vipra/rest/resource/ArticleResource.java
@@ -1,6 +1,8 @@
package de.vipra.rest.resource;
import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
import java.util.Date;
import java.util.List;
@@ -140,13 +142,18 @@ public class ArticleResource {
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@Path("{id}")
- public Response getArticle(@PathParam("id") final String id, @QueryParam("fields") final String fields,
- @QueryParam("excerpt") final String excerpt) {
+ public Response getArticle(@PathParam("id") String id, @QueryParam("fields") final String fields, @QueryParam("excerpt") final String excerpt) {
final ResponseWrapper<ArticleFull> res = new ResponseWrapper<>();
if (id == null || id.trim().length() == 0) {
res.addError(new APIError(Response.Status.BAD_REQUEST, "ID is empty", String.format(Messages.BAD_REQUEST, "id cannot be empty")));
return res.badRequest();
}
+ try {
+ id = URLDecoder.decode(id, "UTF-8");
+ } catch (final UnsupportedEncodingException e1) {
+ res.addError(new APIError(Response.Status.BAD_REQUEST, "Invalid ID", String.format(Messages.BAD_REQUEST, "id could not be decoded")));
+ return res.badRequest();
+ }
ArticleFull article;
try {
diff --git a/vipra-backend/src/main/java/de/vipra/rest/resource/InfoResource.java b/vipra-backend/src/main/java/de/vipra/rest/resource/InfoResource.java
index a856ff01e781f95b834d8ff827f90a28838bdf96..0c8a2490c1313195b1e83ad9649d6f4cffb8ace1 100644
--- a/vipra-backend/src/main/java/de/vipra/rest/resource/InfoResource.java
+++ b/vipra-backend/src/main/java/de/vipra/rest/resource/InfoResource.java
@@ -64,7 +64,10 @@ public class InfoResource {
// constants
info.put("const.windowres", Constants.WINDOW_RESOLUTION);
- info.put("const.procmode", Constants.PROCESSOR_MODE);
+ info.put("const.proctext", Constants.PROCESSOR_USE_TEXT);
+ info.put("const.procentity", Constants.PROCESSOR_USE_ENTITIES);
+ info.put("const.proctype", Constants.PROCESSOR_USE_ENTITY_TYPES);
+ info.put("const.prochypernym", Constants.PROCESSOR_USE_HYPERNYMS);
info.put("const.importbuf", Constants.IMPORT_BUFFER_MAX);
info.put("const.esboosttopics", Constants.ES_BOOST_TOPICS);
info.put("const.esboosttitles", Constants.ES_BOOST_TITLES);
diff --git a/vipra-backend/src/main/java/de/vipra/rest/resource/SequenceResource.java b/vipra-backend/src/main/java/de/vipra/rest/resource/SequenceResource.java
index 4b678a93b58dec9a51d84e2ddb50ae7e897acced..c33fafe4407e398b7ed01d9f20f6c69dedd9b579 100644
--- a/vipra-backend/src/main/java/de/vipra/rest/resource/SequenceResource.java
+++ b/vipra-backend/src/main/java/de/vipra/rest/resource/SequenceResource.java
@@ -1,6 +1,8 @@
package de.vipra.rest.resource;
import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
import java.util.List;
import javax.servlet.ServletContext;
@@ -90,13 +92,19 @@ public class SequenceResource {
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("{id}")
- public Response getSequence(@PathParam("id") final String id, @QueryParam("fields") final String fields,
- @QueryParam("topWords") final Integer topWords) throws ConfigException, IOException {
+ public Response getSequence(@PathParam("id") String id, @QueryParam("fields") final String fields, @QueryParam("topWords") final Integer topWords)
+ throws ConfigException, IOException {
final ResponseWrapper<SequenceFull> res = new ResponseWrapper<>();
if (id == null || id.trim().length() == 0) {
res.addError(new APIError(Response.Status.BAD_REQUEST, "ID is empty", String.format(Messages.BAD_REQUEST, "id cannot be empty")));
return res.badRequest();
}
+ try {
+ id = URLDecoder.decode(id, "UTF-8");
+ } catch (final UnsupportedEncodingException e1) {
+ res.addError(new APIError(Response.Status.BAD_REQUEST, "Invalid ID", String.format(Messages.BAD_REQUEST, "id could not be decoded")));
+ return res.badRequest();
+ }
SequenceFull sequence;
try {
diff --git a/vipra-backend/src/main/java/de/vipra/rest/resource/TextEntityResource.java b/vipra-backend/src/main/java/de/vipra/rest/resource/TextEntityResource.java
index 28512af1bc23fd03d1c0e658578a0021f130049e..9a5ffbd4a37ceca23274260106d6e94cb89ce86f 100644
--- a/vipra-backend/src/main/java/de/vipra/rest/resource/TextEntityResource.java
+++ b/vipra-backend/src/main/java/de/vipra/rest/resource/TextEntityResource.java
@@ -1,6 +1,8 @@
package de.vipra.rest.resource;
import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
import java.util.List;
import javax.servlet.ServletContext;
@@ -99,7 +101,7 @@ public class TextEntityResource {
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@Path("{id}")
- public Response getEntity(@PathParam("id") final String id, @QueryParam("topicModel") final String topicModel,
+ public Response getEntity(@PathParam("id") String id, @QueryParam("topicModel") final String topicModel,
@QueryParam("fields") final String fields) {
final ResponseWrapper<TextEntityFull> res = new ResponseWrapper<>();
if (id == null || id.trim().length() == 0) {
@@ -111,6 +113,12 @@ public class TextEntityResource {
String.format(Messages.BAD_REQUEST, "topic model cannot be empty")));
return res.badRequest();
}
+ try {
+ id = URLDecoder.decode(id.replaceAll("~", "%"), "UTF-8");
+ } catch (final UnsupportedEncodingException e1) {
+ res.addError(new APIError(Response.Status.BAD_REQUEST, "Invalid ID", String.format(Messages.BAD_REQUEST, "id could not be decoded")));
+ return res.badRequest();
+ }
TextEntityFull entity;
try {
@@ -124,7 +132,7 @@ public class TextEntityResource {
topicModelId = topicModelFull.getId();
}
- entity = dbEntities.getSingle(id + "-" + topicModelId, StringUtils.getFields(fields));
+ entity = dbEntities.getSingle(id.toLowerCase() + "-" + topicModelId, StringUtils.getFields(fields));
} catch (final Exception e) {
e.printStackTrace();
res.addError(new APIError(Response.Status.BAD_REQUEST, "Error", e.getMessage()));
diff --git a/vipra-backend/src/main/java/de/vipra/rest/resource/TopicModelResource.java b/vipra-backend/src/main/java/de/vipra/rest/resource/TopicModelResource.java
index 6d6251922b51b656c4e9e6e48a0441279c2d3d44..4b819eff3be8500699e880abf11312a146bd6c36 100644
--- a/vipra-backend/src/main/java/de/vipra/rest/resource/TopicModelResource.java
+++ b/vipra-backend/src/main/java/de/vipra/rest/resource/TopicModelResource.java
@@ -1,6 +1,8 @@
package de.vipra.rest.resource;
import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
import java.util.List;
import javax.servlet.ServletContext;
@@ -65,12 +67,18 @@ public class TopicModelResource {
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("{id}")
- public Response getTopicModel(@PathParam("id") final String id, @QueryParam("fields") final String fields) {
+ public Response getTopicModel(@PathParam("id") String id, @QueryParam("fields") final String fields) {
final ResponseWrapper<TopicModelFull> res = new ResponseWrapper<>();
if (id == null || id.trim().length() == 0) {
res.addError(new APIError(Response.Status.BAD_REQUEST, "ID is empty", String.format(Messages.BAD_REQUEST, "id cannot be empty")));
return res.badRequest();
}
+ try {
+ id = URLDecoder.decode(id, "UTF-8");
+ } catch (final UnsupportedEncodingException e1) {
+ res.addError(new APIError(Response.Status.BAD_REQUEST, "Invalid ID", String.format(Messages.BAD_REQUEST, "id could not be decoded")));
+ return res.badRequest();
+ }
TopicModelFull topicModel;
try {
diff --git a/vipra-backend/src/main/java/de/vipra/rest/resource/TopicResource.java b/vipra-backend/src/main/java/de/vipra/rest/resource/TopicResource.java
index c87de6c8644e82d60b40aacd09f344ceea5e779d..ae08b1f28bda30c49f6dc8886fcc62104c85b17f 100644
--- a/vipra-backend/src/main/java/de/vipra/rest/resource/TopicResource.java
+++ b/vipra-backend/src/main/java/de/vipra/rest/resource/TopicResource.java
@@ -1,6 +1,8 @@
package de.vipra.rest.resource;
import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
import java.util.Date;
import java.util.List;
@@ -109,12 +111,18 @@ public class TopicResource {
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("{id}")
- public Response getTopic(@PathParam("id") final String id, @QueryParam("fields") final String fields) throws ConfigException, IOException {
+ public Response getTopic(@PathParam("id") String id, @QueryParam("fields") final String fields) throws ConfigException, IOException {
final ResponseWrapper<TopicFull> res = new ResponseWrapper<>();
if (id == null || id.trim().length() == 0) {
res.addError(new APIError(Response.Status.BAD_REQUEST, "ID is empty", String.format(Messages.BAD_REQUEST, "id cannot be empty")));
return res.badRequest();
}
+ try {
+ id = URLDecoder.decode(id, "UTF-8");
+ } catch (final UnsupportedEncodingException e1) {
+ res.addError(new APIError(Response.Status.BAD_REQUEST, "Invalid ID", String.format(Messages.BAD_REQUEST, "id could not be decoded")));
+ return res.badRequest();
+ }
TopicFull topic;
try {
diff --git a/vipra-backend/src/main/java/de/vipra/rest/resource/WindowResource.java b/vipra-backend/src/main/java/de/vipra/rest/resource/WindowResource.java
index 5e11fdb10df7fa943e1d1a9a41171839a36acbd9..7e89a6d1cd8379e2d70bb54e9cf9c72258732fbf 100644
--- a/vipra-backend/src/main/java/de/vipra/rest/resource/WindowResource.java
+++ b/vipra-backend/src/main/java/de/vipra/rest/resource/WindowResource.java
@@ -1,6 +1,8 @@
package de.vipra.rest.resource;
import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
import java.util.List;
import javax.servlet.ServletContext;
@@ -89,7 +91,7 @@ public class WindowResource {
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@Path("{id}")
- public Response getWindow(@PathParam("id") final String id, @QueryParam("topicModel") final String topicModel,
+ public Response getWindow(@PathParam("id") String id, @QueryParam("topicModel") final String topicModel,
@QueryParam("fields") final String fields) {
final ResponseWrapper<WindowFull> res = new ResponseWrapper<>();
if (id == null) {
@@ -101,6 +103,12 @@ public class WindowResource {
String.format(Messages.BAD_REQUEST, "topic model cannot be empty")));
return res.badRequest();
}
+ try {
+ id = URLDecoder.decode(id, "UTF-8");
+ } catch (final UnsupportedEncodingException e1) {
+ res.addError(new APIError(Response.Status.BAD_REQUEST, "Invalid ID", String.format(Messages.BAD_REQUEST, "id could not be decoded")));
+ return res.badRequest();
+ }
WindowFull window;
try {
@@ -114,7 +122,7 @@ public class WindowResource {
topicModelId = topicModelFull.getId();
}
- window = dbWindows.getSingle(id + "-" + topicModelId, StringUtils.getFields(fields));
+ window = dbWindows.getSingle(id.toLowerCase() + "-" + topicModelId, StringUtils.getFields(fields));
} catch (final Exception e) {
e.printStackTrace();
res.addError(new APIError(Response.Status.BAD_REQUEST, "Error", e.getMessage()));
diff --git a/vipra-backend/src/main/java/de/vipra/rest/resource/WordResource.java b/vipra-backend/src/main/java/de/vipra/rest/resource/WordResource.java
index ccf0650b253bf8b8249aaf3585029d12ed4d747e..9f13b3ac7550d8c0a455e114bb48dfb675fa225a 100644
--- a/vipra-backend/src/main/java/de/vipra/rest/resource/WordResource.java
+++ b/vipra-backend/src/main/java/de/vipra/rest/resource/WordResource.java
@@ -1,6 +1,8 @@
package de.vipra.rest.resource;
import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
import java.util.List;
import javax.servlet.ServletContext;
@@ -95,7 +97,7 @@ public class WordResource {
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@Path("{id}")
- public Response getWord(@PathParam("id") final String id, @QueryParam("topicModel") final String topicModel,
+ public Response getWord(@PathParam("id") String id, @QueryParam("topicModel") final String topicModel,
@QueryParam("fields") final String fields) {
final ResponseWrapper<WordFull> res = new ResponseWrapper<>();
if (id == null || id.trim().isEmpty()) {
@@ -107,6 +109,12 @@ public class WordResource {
String.format(Messages.BAD_REQUEST, "topic model cannot be empty")));
return res.badRequest();
}
+ try {
+ id = URLDecoder.decode(id, "UTF-8");
+ } catch (final UnsupportedEncodingException e1) {
+ res.addError(new APIError(Response.Status.BAD_REQUEST, "Invalid ID", String.format(Messages.BAD_REQUEST, "id could not be decoded")));
+ return res.badRequest();
+ }
WordFull word;
try {
@@ -120,7 +128,7 @@ public class WordResource {
topicModelId = topicModelFull.getId();
}
- word = dbWords.getSingle(id + "-" + topicModelId, StringUtils.getFields(fields));
+ word = dbWords.getSingle(id.toLowerCase() + "-" + topicModelId, StringUtils.getFields(fields));
} catch (final Exception e) {
e.printStackTrace();
res.addError(new APIError(Response.Status.BAD_REQUEST, "Error", e.getMessage()));
diff --git a/vipra-cmd/.classpath b/vipra-cmd/.classpath
index 4b6b27664bf27624189421ebc42d984b75d9e8d1..a2cbc0d348384398004328b24b9d9195d28a426e 100644
--- a/vipra-cmd/.classpath
+++ b/vipra-cmd/.classpath
@@ -11,6 +11,7 @@
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
+ <classpathentry kind="src" path="src/main/hypernyms"/>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
diff --git a/vipra-cmd/.settings/org.eclipse.wst.common.component b/vipra-cmd/.settings/org.eclipse.wst.common.component
index ec7635d9ec5720876c9b0971d2e37846e10ffad3..ffc0821a6c1fe46e8f41fb53d2299a2d65bd944d 100644
--- a/vipra-cmd/.settings/org.eclipse.wst.common.component
+++ b/vipra-cmd/.settings/org.eclipse.wst.common.component
@@ -2,5 +2,6 @@
<wb-module deploy-name="vipra-cmd">
<wb-resource deploy-path="/" source-path="/src/main/java"/>
<wb-resource deploy-path="/" source-path="/src/main/resources"/>
+ <wb-resource deploy-path="/" source-path="/src/main/hypernyms"/>
</wb-module>
</project-modules>
diff --git a/vipra-cmd/pom.xml b/vipra-cmd/pom.xml
index 193991c3f1e1bd029c375d901913c0a1f93e38ae..02a54aece9e21ddcc223a78999682fa09ce06bca 100644
--- a/vipra-cmd/pom.xml
+++ b/vipra-cmd/pom.xml
@@ -31,6 +31,11 @@
<artifactId>commons-cli</artifactId>
<version>1.3.1</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>3.4</version>
+ </dependency>
<!-- JsonSimple -->
<dependency>
@@ -88,7 +93,7 @@
<skipTests>true</skipTests>
</configuration>
</plugin>
-
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
diff --git a/vipra-cmd/runcfg/CMD.launch b/vipra-cmd/runcfg/CMD.launch
index c7c1d7de9494f2b762f4890a4baa0c75ce38cd52..aa82fd1f18579b76af999f8f01e357b7932fcc3e 100644
--- a/vipra-cmd/runcfg/CMD.launch
+++ b/vipra-cmd/runcfg/CMD.launch
@@ -11,7 +11,7 @@
</listAttribute>
<stringAttribute key="org.eclipse.jdt.launching.CLASSPATH_PROVIDER" value="org.eclipse.m2e.launchconfig.classpathProvider"/>
<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="de.vipra.cmd.Main"/>
-<stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="-Ar"/>
+<stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="-cS test -I /home/eike/repos/master/ma-impl/docker/data/test-1.json"/>
<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="vipra-cmd"/>
<stringAttribute key="org.eclipse.jdt.launching.SOURCE_PATH_PROVIDER" value="org.eclipse.m2e.launchconfig.sourcepathProvider"/>
<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-ea"/>
diff --git a/vipra-cmd/src/main/hypernyms/hypernyms.ser.gz b/vipra-cmd/src/main/hypernyms/hypernyms.ser.gz
new file mode 100644
index 0000000000000000000000000000000000000000..b21d7089a99cdd1c9a80749854ac815157c3019e
Binary files /dev/null and b/vipra-cmd/src/main/hypernyms/hypernyms.ser.gz differ
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/CommandLineOptions.java b/vipra-cmd/src/main/java/de/vipra/cmd/CommandLineOptions.java
index e849abee437260c7ac1bf1c26410cabd72a8580c..30736fa1f5fb300981746b3ce192ed2d4a90cb62 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/CommandLineOptions.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/CommandLineOptions.java
@@ -12,7 +12,7 @@ import org.apache.commons.cli.ParseException;
public class CommandLineOptions {
- public static final Option CLEAR = Option.builder("c").longOpt("clear").desc("clear the database and models").build();
+ public static final Option ERASE = Option.builder("e").longOpt("erase").desc("erase the database and models").build();
public static final Option DEBUG = Option.builder("d").longOpt("debug").desc("show debug information").build();
public static final Option HELP = Option.builder("h").longOpt("help").desc("show this help").build();
public static final Option LIST = Option.builder("l").longOpt("list").desc("list available models").build();
@@ -39,6 +39,8 @@ public class CommandLineOptions {
.optionalArg(true).build();
public static final Option RENAME = Option.builder("n").longOpt("rename").desc("rename selected models").hasArg().argName("[models...]")
.optionalArg(true).build();
+ public static final Option CLEAR = Option.builder("c").longOpt("clear").desc("clear selected models").hasArg().argName("[models...]")
+ .optionalArg(true).build();
public static final Option SELECT = Option.builder("S").longOpt("select").desc("select models").hasArgs().argName("models...").build();
public static final Option BACKUP = Option.builder("B").longOpt("backup").desc("backup data/filebase").hasArg().argName("path").build();
public static final Option RESTORE = Option.builder("R").longOpt("restore").desc("restore data/filebase").hasArg().argName("path").build();
@@ -49,8 +51,8 @@ public class CommandLineOptions {
private final String cmdName = "vipra";
public CommandLineOptions() {
- final Option[] optionsArray = { CLEAR, DEBUG, HELP, INDEX, LIST, REREAD, SILENT, TEST, ALL, CREATE, DELETE, EDIT, PRINT, IMPORT, MODEL,
- RENAME, SELECT, BACKUP, RESTORE, VERSION, LOG };
+ final Option[] optionsArray = { ERASE, DEBUG, HELP, INDEX, LIST, REREAD, SILENT, TEST, ALL, CREATE, DELETE, EDIT, PRINT, IMPORT, MODEL,
+ RENAME, CLEAR, SELECT, BACKUP, RESTORE, VERSION, LOG };
options = new Options();
for (final Option option : optionsArray)
options.addOption(option);
@@ -85,8 +87,8 @@ public class CommandLineOptions {
return cmd.getOptionValues(opt.getOpt());
}
- public boolean isClear() {
- return hasOption(CLEAR);
+ public boolean isErase() {
+ return hasOption(ERASE);
}
public boolean isDebug() {
@@ -254,6 +256,17 @@ public class CommandLineOptions {
return selectedModels();
}
+ public boolean isClear() {
+ return hasOption(CLEAR);
+ }
+
+ public String[] modelsToClear() {
+ final String[] models = getOptionValues(CLEAR);
+ if (models != null && models.length > 0)
+ return models;
+ return selectedModels();
+ }
+
public boolean isLog() {
return hasOption(LOG);
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/Main.java b/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
index d2a67882a4517e362423f76702150f0baa5ce820..34a78f76d74b32a79285f01b17deb4937a1da64d 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
@@ -10,11 +10,12 @@ import org.mongodb.morphia.logging.MorphiaLoggerFactory;
import org.mongodb.morphia.logging.slf4j.SLF4JLoggerImplFactory;
import de.vipra.cmd.option.BackupCommand;
-import de.vipra.cmd.option.ClearCommand;
+import de.vipra.cmd.option.ClearModelCommand;
import de.vipra.cmd.option.Command;
import de.vipra.cmd.option.CreateModelCommand;
import de.vipra.cmd.option.DeleteModelCommand;
import de.vipra.cmd.option.EditModelCommand;
+import de.vipra.cmd.option.EraseCommand;
import de.vipra.cmd.option.ImportCommand;
import de.vipra.cmd.option.IndexingCommand;
import de.vipra.cmd.option.ListModelsCommand;
@@ -87,7 +88,7 @@ public class Main {
else
ConsoleUtils.setLogFile(new File(file));
}
- } catch (IOException e) {
+ } catch (final IOException e) {
ConsoleUtils.error("cannot use log file: " + e.getMessage());
return;
}
@@ -108,8 +109,11 @@ public class Main {
if (opts.isRestore())
commands.add(new RestoreCommand(opts.restorePath()));
+ if (opts.isErase())
+ commands.add(new EraseCommand());
+
if (opts.isClear())
- commands.add(new ClearCommand());
+ commands.add(new ClearModelCommand(opts.modelsToClear()));
if (opts.isDelete())
commands.add(new DeleteModelCommand(opts.modelsToDelete()));
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/file/Filebase.java b/vipra-cmd/src/main/java/de/vipra/cmd/file/Filebase.java
index daa810c06f5bb38ae4f2e34ec0ca1b40644c1900..cdfcb282d6d3d9780c4a34cfc349d25c6aeaf94c 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/file/Filebase.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/file/Filebase.java
@@ -67,16 +67,21 @@ public class Filebase {
for (final FilebaseIDDateIndexEntry entry : idDateIndex) {
if (entry.isNew()) {
final ArticleFull newArticle = newArticles.get(entry.getId());
- switch (modelConfig.getProcessorMode()) {
- case TEXT_WITH_ENTITIES:
- outModel.write(wordIndex.transform(newArticle.getProcessedText(), true));
- outModel.write(" ");
- case ENTITIES:
- outModel.write(wordIndex.transform(newArticle.entitiesWithTypes(), false));
- break;
- case TEXT:
- outModel.write(wordIndex.transform(newArticle.getProcessedText(), true));
- }
+ if (modelConfig.isProcessorUseText())
+ outModel.write(wordIndex.transform(newArticle.getProcessedText(), true));
+
+ outModel.write(" ");
+
+ if (modelConfig.isProcessorUseEntities())
+ outModel.write(wordIndex.transform(newArticle.entities(), false));
+ else if (modelConfig.isProcessorUseHypernyms())
+ outModel.write(wordIndex.transform(newArticle.hypernyms(), false));
+
+ outModel.write(" ");
+
+ if (modelConfig.isProcessorUseEntityTypes())
+ outModel.write(wordIndex.transform(newArticle.types(), false));
+
outModel.write(Constants.LINE_SEP);
} else {
if (in == null) {
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearModelCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearModelCommand.java
new file mode 100644
index 0000000000000000000000000000000000000000..d9b8ad9122fea1d727a0f331376523b6c6e95152
--- /dev/null
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearModelCommand.java
@@ -0,0 +1,77 @@
+package de.vipra.cmd.option;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.bson.types.ObjectId;
+
+import de.vipra.util.Config;
+import de.vipra.util.ConsoleUtils;
+import de.vipra.util.model.ArticleFull;
+import de.vipra.util.model.SequenceFull;
+import de.vipra.util.model.TextEntityFull;
+import de.vipra.util.model.TopicFull;
+import de.vipra.util.model.TopicModel;
+import de.vipra.util.model.TopicModelFull;
+import de.vipra.util.model.WindowFull;
+import de.vipra.util.model.WordFull;
+import de.vipra.util.service.MongoService;
+import de.vipra.util.service.QueryBuilder;
+
+public class ClearModelCommand implements Command {
+
+ private final String[] names;
+
+ public ClearModelCommand(final String[] names) {
+ this.names = names;
+ }
+
+ @Override
+ public void run() throws Exception {
+ final Config config = Config.getConfig();
+ final MongoService<TopicModelFull, ObjectId> dbTopicModels = MongoService.getDatabaseService(config, TopicModelFull.class);
+ final MongoService<ArticleFull, ObjectId> dbArticles = MongoService.getDatabaseService(config, ArticleFull.class);
+ final MongoService<TopicFull, ObjectId> dbTopics = MongoService.getDatabaseService(config, TopicFull.class);
+ final MongoService<SequenceFull, ObjectId> dbSequences = MongoService.getDatabaseService(config, SequenceFull.class);
+ final MongoService<WordFull, String> dbWords = MongoService.getDatabaseService(config, WordFull.class);
+ final MongoService<TextEntityFull, String> dbEntities = MongoService.getDatabaseService(config, TextEntityFull.class);
+ final MongoService<WindowFull, String> dbWindows = MongoService.getDatabaseService(config, WindowFull.class);
+
+ final List<TopicModelFull> topicModels = dbTopicModels.getMultiple(QueryBuilder.builder().in("name", Arrays.asList(names)).allFields());
+
+ for (final TopicModelFull topicModel : topicModels) {
+ final File modelDir = new File(config.getDataDirectory(), topicModel.getName());
+
+ final QueryBuilder builder = QueryBuilder.builder().eq("topicModel", new TopicModel(topicModel.getId()));
+
+ dbArticles.deleteMultiple(builder);
+ dbTopics.deleteMultiple(builder);
+ dbSequences.deleteMultiple(builder);
+ dbWords.deleteMultiple(builder);
+ dbEntities.deleteMultiple(builder);
+ dbWindows.deleteMultiple(builder);
+
+ if (modelDir.exists()) {
+ FileUtils.cleanDirectory(modelDir);
+ topicModel.getModelConfig().saveToFile(modelDir);
+ }
+
+ topicModel.setArticleCount(0L);
+ topicModel.setTopicCount(0L);
+ topicModel.setEntityCount(0L);
+ topicModel.setWordCount(0L);
+
+ ConsoleUtils.info("model cleared: " + topicModel.getName());
+ }
+
+ dbTopicModels.replaceMultiple(topicModels);
+ }
+
+ @Override
+ public boolean requiresLock() {
+ return true;
+ }
+
+}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteModelCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteModelCommand.java
index cbd9f7130453320b62f8fd3a2b9ea3634cdeb849..0d33b269c5c126e69c14915d7633789194bd9bd2 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteModelCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteModelCommand.java
@@ -46,6 +46,7 @@ public class DeleteModelCommand implements Command {
final QueryBuilder builder = QueryBuilder.builder().eq("topicModel", new TopicModel(topicModel.getId()));
+ // delete from database
long deleted = 0;
deleted += dbArticles.deleteMultiple(builder);
deleted += dbTopics.deleteMultiple(builder);
@@ -54,11 +55,15 @@ public class DeleteModelCommand implements Command {
deleted += dbEntities.deleteMultiple(builder);
deleted += dbWindows.deleteMultiple(builder);
+ // delete from filebase
if (modelDir.exists()) {
org.apache.commons.io.FileUtils.deleteDirectory(modelDir);
deleted++;
}
+ // delete from config
+ config.deleteTopicModelConfig(topicModel.getName());
+
if (deleted > 0)
ConsoleUtils.info("model deleted: " + topicModel.getName());
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/EditModelCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/EditModelCommand.java
index dec571d5af7d3d1603ae90e8c13269ba4d8dbb94..ec0a84a265cedda0f96ccc5cf5c69d8e456984e5 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/EditModelCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/EditModelCommand.java
@@ -9,7 +9,6 @@ import com.fasterxml.jackson.databind.JsonMappingException;
import de.vipra.util.Config;
import de.vipra.util.ConsoleUtils;
-import de.vipra.util.Constants.ProcessorMode;
import de.vipra.util.Constants.WindowResolution;
import de.vipra.util.ex.ConfigException;
import de.vipra.util.ex.DatabaseException;
@@ -56,9 +55,13 @@ public class EditModelCommand implements Command {
ConsoleUtils.readDouble("max similar documents divergence", topicModelConfig.getMaxSimilarDocumentsDivergence(), 0.0, 1.0, true));
topicModelConfig.setMaxSimilarTopicsDivergence(
ConsoleUtils.readDouble("max similar topics divergence", topicModelConfig.getMaxSimilarTopicsDivergence(), 0.0, 1.0, true));
+ topicModelConfig.setProcessorUseText(ConsoleUtils.readBoolean("processor use text", topicModelConfig.isProcessorUseText()));
+ topicModelConfig.setProcessorUseEntities(ConsoleUtils.readBoolean("processor use entities", topicModelConfig.isProcessorUseEntities()));
+ topicModelConfig
+ .setProcessorUseEntityTypes(ConsoleUtils.readBoolean("processor use entity types", topicModelConfig.isProcessorUseEntityTypes()));
+ topicModelConfig.setProcessorUseHypernyms(ConsoleUtils.readBoolean("processor use hypernyms", topicModelConfig.isProcessorUseHypernyms()));
topicModelConfig
.setWindowResolution(ConsoleUtils.readEnum(WindowResolution.class, "window resolution", topicModelConfig.getWindowResolution()));
- topicModelConfig.setProcessorMode(ConsoleUtils.readEnum(ProcessorMode.class, "processor mode", topicModelConfig.getProcessorMode()));
final TopicModelFull topicModel = dbTopicModels.getSingle(QueryBuilder.builder().eq("name", topicModelConfig.getName()));
topicModel.setModelConfig(topicModelConfig);
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/EraseCommand.java
similarity index 92%
rename from vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java
rename to vipra-cmd/src/main/java/de/vipra/cmd/option/EraseCommand.java
index 1894e665bcf36e08720af5b65e51caabfe5b5427..29ffaed73d7742e9b3ab6585a6d4d4d0b23da119 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/EraseCommand.java
@@ -11,7 +11,7 @@ import de.vipra.util.ConsoleUtils;
import de.vipra.util.ESClient;
import de.vipra.util.service.MongoService;
-public class ClearCommand implements Command {
+public class EraseCommand implements Command {
private Config config;
private Client elasticClient;
@@ -33,7 +33,7 @@ public class ClearCommand implements Command {
config.clearTopicModelConfigs();
- ConsoleUtils.info("cleared");
+ ConsoleUtils.info("erased");
}
@Override
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
index 4f7a5b3fe1e457d9465bac192a6828cf7daac66b..007db4ea6edf4a7a537af7eaee9bda3e933b9272 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
@@ -22,6 +22,7 @@ import de.vipra.cmd.file.Filebase;
import de.vipra.cmd.file.FilebaseException;
import de.vipra.cmd.file.FilebaseWindowIndex;
import de.vipra.cmd.file.FilebaseWordIndex;
+import de.vipra.cmd.text.HypernymAnalyzer;
import de.vipra.cmd.text.ProcessedText;
import de.vipra.cmd.text.Processor;
import de.vipra.cmd.text.ProcessorException;
@@ -30,13 +31,13 @@ import de.vipra.cmd.text.SpotlightResponse;
import de.vipra.util.Config;
import de.vipra.util.ConsoleUtils;
import de.vipra.util.Constants;
-import de.vipra.util.Constants.ProcessorMode;
import de.vipra.util.StringUtils;
import de.vipra.util.Timer;
import de.vipra.util.ex.ConfigException;
import de.vipra.util.ex.DatabaseException;
import de.vipra.util.model.ArticleFull;
import de.vipra.util.model.ArticleStats;
+import de.vipra.util.model.TextEntity;
import de.vipra.util.model.TextEntityCount;
import de.vipra.util.model.TextEntityFull;
import de.vipra.util.model.TopicModel;
@@ -164,8 +165,7 @@ public class ImportCommand implements Command {
// preprocess text
final ProcessedText processedText = processor.process(modelConfig, article.getText());
- if (modelConfig.getProcessorMode() != ProcessorMode.ENTITIES
- && processedText.getReducedWordCount() < modelConfig.getDocumentMinimumLength()) {
+ if (processedText.getReducedWordCount() < modelConfig.getDocumentMinimumLength()) {
ConsoleUtils.info(ConsoleUtils.positionString(current, max) + " skipped \"" + object.get("title") + "\" ("
+ processedText.getReducedWordCount() + ")");
} else {
@@ -177,20 +177,39 @@ public class ImportCommand implements Command {
final List<TextEntityCount> textEntitiesCounts = spotlightResponse.getEntities();
if (textEntitiesCounts != null) {
+ // replace entities with hypernyms
+ if (modelConfig.isProcessorUseHypernyms()) {
+ final HypernymAnalyzer hypernymAnalyzer = HypernymAnalyzer.getInstance();
+ for (final TextEntityCount textEntityCount : textEntitiesCounts) {
+ final TextEntity textEntity = textEntityCount.getEntity();
+ final String hypernym = hypernymAnalyzer.getHypernym(textEntity.realEntity());
+ if (hypernym != null) {
+ textEntity.setUrl(hypernymAnalyzer.getURL(hypernym));
+ textEntity.setIsHypernym(true);
+ }
+ }
+ }
+
+ // insert entities into text
for (final TextEntityCount textEntityCount : textEntitiesCounts) {
// get new text entity
final TextEntityFull newTextEntity = new TextEntityFull(textEntityCount.getEntity(), topicModel);
+ if (newTextEntity.getEntity() == null || newTextEntity.getEntity().isEmpty())
+ continue;
newTextEntities.add(newTextEntity);
// insert entity into text
articleText = articleText.replaceAll(
"(?i)\\b" + Pattern.quote(textEntityCount.getEntity().getEntity()) + "\\b(?![^<]*>|[^<>]*</)",
- Matcher.quoteReplacement(textEntityCount.getEntity().aTag()));
+ Matcher.quoteReplacement(newTextEntity.aTag(textEntityCount.getEntity().getEntity())));
+
+ // replace entity surface form by resource form
+ textEntityCount.getEntity().setEntity(newTextEntity.getEntity());
}
- }
- article.setEntities(textEntitiesCounts);
- article.setText(articleText);
+ article.setEntities(textEntitiesCounts);
+ article.setText(articleText);
+ }
}
article.setProcessedText(processedText.getWords());
@@ -219,6 +238,8 @@ public class ImportCommand implements Command {
ConsoleUtils.error("could not save processed article in the filebase '" + article.getTitle() + "'");
} catch (final IOException e) {
ConsoleUtils.error("io error");
+ } catch (final ClassNotFoundException e) {
+ ConsoleUtils.error("could not initialize hypernym analyzer");
}
}
@@ -269,9 +290,9 @@ public class ImportCommand implements Command {
if (config.getSpotlightUrl() != null)
spotlightAnalyzer = new SpotlightAnalyzer(modelConfig);
- if ((modelConfig.getProcessorMode() == ProcessorMode.ENTITIES || modelConfig.getProcessorMode() == ProcessorMode.TEXT_WITH_ENTITIES)
+ if ((modelConfig.isProcessorUseEntities() || modelConfig.isProcessorUseEntityTypes() || modelConfig.isProcessorUseHypernyms())
&& spotlightAnalyzer == null)
- throw new ConfigException("spotlight url is empty, but processor mode is set to " + modelConfig.getProcessorMode());
+ throw new ConfigException("spotlight url is empty, but processor requires entity informations");
buffer = new ArticleBuffer(dbArticles);
filebase = new Filebase(modelConfig, config.getDataDirectory());
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/text/HypernymAnalyzer.java b/vipra-cmd/src/main/java/de/vipra/cmd/text/HypernymAnalyzer.java
new file mode 100644
index 0000000000000000000000000000000000000000..4f9f56421ae44ada493d6e434cb4a367af4a4614
--- /dev/null
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/text/HypernymAnalyzer.java
@@ -0,0 +1,45 @@
+package de.vipra.cmd.text;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.ObjectInputStream;
+import java.util.Map;
+import java.util.zip.GZIPInputStream;
+
+import org.apache.commons.lang3.StringEscapeUtils;
+
+import de.vipra.util.FileUtils;
+
+public class HypernymAnalyzer {
+
+ private static HypernymAnalyzer instance;
+
+ private final Map<String, String> hypernyms;
+
+ @SuppressWarnings("unchecked")
+ private HypernymAnalyzer() throws IOException, ClassNotFoundException {
+ final InputStream in = FileUtils.getResource("hypernyms.ser.gz");
+ final ObjectInputStream ois = new ObjectInputStream(new BufferedInputStream(new GZIPInputStream(in)));
+ hypernyms = (Map<String, String>) ois.readObject();
+ ois.close();
+ }
+
+ public boolean containsEntity(final String entity) {
+ return hypernyms.containsKey(StringEscapeUtils.unescapeJava(entity));
+ }
+
+ public String getHypernym(final String entity) {
+ return hypernyms.get(StringEscapeUtils.unescapeJava(entity));
+ }
+
+ public String getURL(final String hypernym) {
+ return "http://dbpedia.org/resource/" + hypernym;
+ }
+
+ public static HypernymAnalyzer getInstance() throws IOException, ClassNotFoundException {
+ if (instance == null)
+ instance = new HypernymAnalyzer();
+ return instance;
+ }
+}
diff --git a/vipra-cmd/src/main/resources/config.json b/vipra-cmd/src/main/resources/config.json
index 24eefc91e093151cb9f216cd855df8917f0f2a79..b0b589f81342502719816096f32cd32508bf5a5a 100644
--- a/vipra-cmd/src/main/resources/config.json
+++ b/vipra-cmd/src/main/resources/config.json
@@ -15,14 +15,17 @@
"staticIterations": 100,
"topicAutoNamingWords": 4,
"maxSimilarDocuments": 10,
- "documentMinimumLength": 10,
- "documentMinimumWordFrequency": 5,
- "spotlightSupport": 0,
+ "documentMinimumLength": 1,
+ "documentMinimumWordFrequency": 1,
+ "spotlightSupport": 200,
"spotlightConfidence": 0.5,
"minRelativeProbability": 0.01,
"risingDecayLambda": 0.0,
"maxSimilarDocumentsDivergence": 0.25,
- "windowResolution": "YEAR",
- "processorMode": "TEXT"
+ "windowResolution": "MONTH",
+ "processorUseText": true,
+ "processorUseEntities": true,
+ "processorUseEntityTypes": true,
+ "processorUseHypernyms": true
}
}
\ No newline at end of file
diff --git a/vipra-cmd/src/main/resources/hypernyms.map b/vipra-cmd/src/main/resources/hypernyms.map
deleted file mode 100644
index 54477d8e0a4743ded6172b3d2c323cfafb6c677c..0000000000000000000000000000000000000000
Binary files a/vipra-cmd/src/main/resources/hypernyms.map and /dev/null differ
diff --git a/vipra-ui/app/html/about.html b/vipra-ui/app/html/about.html
index 33c1c7c7c42b8cc4206e27beff12d9e1765a0f9e..72c510d43aa5835de9fbf30ad08c488288ec3771 100644
--- a/vipra-ui/app/html/about.html
+++ b/vipra-ui/app/html/about.html
@@ -93,12 +93,24 @@
</td>
</tr>
<tr>
- <th>Processor mode</th>
- <td ng-bind-template="{{::info.const.procmode}}"></td>
+ <th>Processor use text</th>
+ <td ng-bind-template="{{::info.const.proctext}}"></td>
+ </tr>
+ <tr>
+ <th>Processor use entities</th>
+ <td ng-bind-template="{{::info.const.procentity}}"></td>
+ </tr>
+ <tr>
+ <th>Processor use entity types</th>
+ <td ng-bind-template="{{::info.const.proctype}}"></td>
+ </tr>
+ <tr>
+ <th>Processor use hypernyms</th>
+ <td ng-bind-template="{{::info.const.prochypernym}}"></td>
</tr>
<tr class="well">
<td colspan="2">
- The processor mode defines the processed text output. In text mode, the text is trimmed down, in entity mode, the text is scanned for entities. In mixed mode, the found entities are inserted into the trimmed text.
+ These variables define the processor behavior when transforming imported text into the filebase form. The text is transformed according to these settings as a preparation for the topic model generation.
</td>
</tr>
<tr>
diff --git a/vipra-ui/app/html/articles/show.html b/vipra-ui/app/html/articles/show.html
index 7fcb0c155b6f743c064b2bc7254b06c18c383223..7840ea927e73ac9cb8e63fef5ed96ee12641f072 100644
--- a/vipra-ui/app/html/articles/show.html
+++ b/vipra-ui/app/html/articles/show.html
@@ -28,14 +28,8 @@
<table class="table table-bordered table-condensed table-fixed nomargin">
<tbody>
<tr>
- <th class="infocol">ID</th>
- <td>
- <span class="ellipsis" ng-bind="::article.id"></span>
- </td>
- </tr>
- <tr>
- <th>Date</th>
- <td ng-bind="::articleDate"></td>
+ <th class="infocol">Date</th>
+ <td ng-bind-template="{{$root.Vipra.formatDateTime(article.date)}}"></td>
</tr>
<tr ng-if="article.url">
<th>URL</th>
@@ -108,6 +102,9 @@
</div>
<div role="tabpanel" class="tab-pane tab-entities">
<br>
+ <div class="row">
+ <div class="col-md-12" highcharts="entityDistribution"></div>
+ </div>
<div class="row">
<div class="col-md-12">
<div class="panel panel-default">
@@ -141,6 +138,9 @@
</div>
<div role="tabpanel" class="tab-pane tab-words">
<br>
+ <div class="row">
+ <div class="col-md-12" highcharts="wordDistribution"></div>
+ </div>
<div class="row">
<div class="col-md-12">
<div class="panel panel-default">
diff --git a/vipra-ui/app/html/entities/articles.html b/vipra-ui/app/html/entities/articles.html
index 6232bdd52f138e8cf61fff57ad3471fae5afa1f7..ad8512714b77c12497bb52ab0247408ac98ca1c0 100644
--- a/vipra-ui/app/html/entities/articles.html
+++ b/vipra-ui/app/html/entities/articles.html
@@ -1,61 +1,67 @@
<div class="container" ng-cloak ng-hide="!rootModels.topicModel || state.name !== 'entities.show.articles'">
- <div class="row">
- <div class="col-md-12">
- <div class="page-header">
- <h1 ng-bind-template="Articles for entity '{{::entity}}'"></h1>
- <table class="item-actions">
- <tr>
- <td>
- <a class="btn btn-default" ng-click="goBack()" ng-show="oldState.name && oldState.name !== state.name" ng-cloak>Back</a>
- </td>
- </tr>
- </table>
- </div>
- </div>
- </div>
- <div class="row">
- <div class="col-md-12 text-center">
- <pagination total="articlesTotal" page="entitiesArticlesModels.page" limit="entitiesArticlesModels.limit" />
- </div>
+ <div class="page-header no-border">
+ <span class="label label-default">Entity</span>
+ <h1 ng-bind="::entity"></h1>
</div>
- <div class="row">
- <div class="col-md-12">
- <div class="panel panel-default">
- <div class="panel-heading">
- Found
- <ng-pluralize count="articlesTotal||0" when="{0:'no articles',1:'1 article',other:'{} articles'}"></ng-pluralize> in the database.
- <span ng-show="articlesTotal" ng-cloak>
- Sort by
- <ol class="nya-bs-select nya-bs-condensed" ng-model="entitiesArticlesModels.sortkey">
- <li value="title" class="nya-bs-option"><a>Title</a></li>
- <li value="date" class="nya-bs-option"><a>Date</a></li>
- <li value="created" class="nya-bs-option"><a>Added</a></li>
- </ol>
- <sort-dir ng-model="entitiesArticlesModels.sortdir" />
- </span>
+ <div>
+ <ul class="nav nav-tabs" role="tablist">
+ <li>
+ <a ui-sref="entities.show({id:entity})"><i class="fa fa-file-text-o"></i></a>
+ </li>
+ <li class="active">
+ <a>
+ Articles
+ </a>
+ </li>
+ </ul>
+ <div class="tab-content">
+ <div role="tabpanel" class="tab-pane active">
+ <div class="row">
+ <br>
+ <div class="col-md-12">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Found
+ <ng-pluralize count="articlesTotal||0" when="{0:'no articles',1:'1 article',other:'{} articles'}"></ng-pluralize> in the database.
+ <span ng-show="articlesTotal" ng-cloak>
+ Sort by
+ <div class="input-group inline">
+ <select class="form-control" ng-model="entitiesArticlesModels.sortkey">
+ <option value="title">Title</option>
+ <option value="date">Date</option>
+ <option value="created">Added</option>
+ </select>
+ <span class="input-group-btn">
+ <sort-dir ng-model="entitiesArticlesModels.sortdir" />
+ </span>
+ </div>
+ </span>
+ </div>
+ <table class="table table-hover table-condensed table-fixed">
+ <tbody>
+ <tr ng-repeat="article in articles">
+ <td>
+ <article-link article="::article"/>
+ </td>
+ </tr>
+ <tr ng-show="loadingArticles">
+ <td>Loading...</td>
+ </tr>
+ </tbody>
+ </table>
+ <div class="panel-footer">
+ Page <span ng-bind="entitiesArticlesModels.page||1"></span> of <span ng-bind="maxPage||1"></span>
+ </div>
+ </div>
+ </div>
</div>
- <table class="table table-hover table-condensed table-fixed">
- <tbody>
- <tr ng-repeat="article in articles">
- <td>
- <article-link article="::article"/>
- </td>
- </tr>
- <tr ng-show="loadingArticles">
- <td>Loading...</td>
- </tr>
- </tbody>
- </table>
- <div class="panel-footer">
- Page <span ng-bind="entitiesArticlesModels.page||1"></span> of <span ng-bind="maxPage||1"></span>
+ <div class="row">
+ <div class="col-md-12 text-center">
+ <pagination total="articlesTotal" page="entitiesArticlesModels.page" limit="entitiesArticlesModels.limit" />
+ </div>
</div>
</div>
</div>
</div>
- <div class="row">
- <div class="col-md-12 text-center">
- <pagination total="articlesTotal" page="entitiesArticlesModels.page" limit="entitiesArticlesModels.limit" />
- </div>
- </div>
</div>
<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/entities/show.html b/vipra-ui/app/html/entities/show.html
index 26ea9a294677bfc02d0b77599a1845e33e86e019..0f8eb97c75b2e0d0bd2cdf1b0b859c183c25739f 100644
--- a/vipra-ui/app/html/entities/show.html
+++ b/vipra-ui/app/html/entities/show.html
@@ -19,12 +19,8 @@
<h3>Info</h3>
<table class="table table-bordered table-condensed table-fixed">
<tbody>
- <tr>
- <th class="infocol">ID</th>
- <td ng-bind="::entity.id"></td>
- </tr>
<tr ng-if="entity.url">
- <th>URL</th>
+ <th class="infocol">URL</th>
<td>
<a ng-href="{{::entity.url}}" target="_blank">
<i class="fa fa-link"></i>
@@ -32,22 +28,20 @@
</a>
</td>
</tr>
- </tbody>
- </table>
- <h3>Types</h3>
- <table class="table table-bordered table-condensed table-fixed" ng-show="entity.types.length" ng-cloak>
- <thead>
<tr>
- <th>Name</th>
+ <th class="infocol">Hypernym</th>
+ <td ng-bind-template="{{ entity.isHypernym ? 'yes' : 'no' }}"></td>
</tr>
- </thead>
- <tbody>
- <tr ng-repeat="type in entity.types">
- <td ng-bind="type"></td>
+ <tr>
+ <th class="infocol">Types</th>
+ <td ng-bind-template="{{entity.types.join(', ') || 'No types'}}" ng-class="{'text-muted':!entity.types}"></td>
+ </tr>
+ <tr>
+ <th class="infocol">Abstract</th>
+ <td ng-bind-template="{{abstract || 'No abstract'}}" ng-class="{'text-muted':!abstract}"></td>
</tr>
</tbody>
</table>
- <p class="text-muted" ng-hide="entity.types.length">No types</p>
</div>
</div>
<div class="loading" ng-hide="entity">Loading...</div>
diff --git a/vipra-ui/app/html/help/articles.html b/vipra-ui/app/html/help/articles.html
new file mode 100644
index 0000000000000000000000000000000000000000..7a012ab25219235985f64df395f14a15209c6e98
--- /dev/null
+++ b/vipra-ui/app/html/help/articles.html
@@ -0,0 +1,61 @@
+<p>The articles page shows all articles that were imported into this topic model. The articles are paginated to reduce page load and they can be filtered by name, initial and date. The article sorting can be changed and for each article, an excerpt and a topic list can be shown by activating the article dropdown.</p>
+
+<table class="help-listing">
+ <tr>
+ <td><kbd>1</kbd></td>
+ <td>Article title</td>
+ </tr>
+ <tr>
+ <td><kbd>2</kbd></td>
+ <td>Article menu dropdown</td>
+ </tr>
+ <tr>
+ <td><kbd>3</kbd></td>
+ <td>Article excerpt text</td>
+ </tr>
+ <tr>
+ <td><kbd>4</kbd></td>
+ <td>Article topics with topic menu dropdowns</td>
+ </tr>
+ <tr>
+ <td><kbd>5</kbd></td>
+ <td>Page number</td>
+ </tr>
+ <tr>
+ <td><kbd>6</kbd></td>
+ <td>Article date</td>
+ </tr>
+ <tr>
+ <td><kbd>7</kbd></td>
+ <td>Article topic count</td>
+ </tr>
+ <tr>
+ <td><kbd>8</kbd></td>
+ <td>Article details dropdown</td>
+ </tr>
+ <tr>
+ <td><kbd>9</kbd></td>
+ <td>Pagination</td>
+ </tr>
+ <tr>
+ <td><kbd>10</kbd></td>
+ <td>Filter articles by name</td>
+ </tr>
+ <tr>
+ <td><kbd>11</kbd></td>
+ <td>Filter articles by initial of name</td>
+ </tr>
+ <tr>
+ <td><kbd>12</kbd></td>
+ <td>Sort articles by various attributes, sort direction</td>
+ </tr>
+ <tr>
+ <td><kbd>13</kbd></td>
+ <td>Filter articles by date range</td>
+ </tr>
+</table>
+
+<div class="image">
+ <img class="default" src="img/ui-articles.png">
+ <img class="hover" src="img/ui-articles-hover.png">
+</div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/help/articles.show.html b/vipra-ui/app/html/help/articles.show.html
new file mode 100644
index 0000000000000000000000000000000000000000..544589a321ccae548b0ac4487587a9a622542329
--- /dev/null
+++ b/vipra-ui/app/html/help/articles.show.html
@@ -0,0 +1,37 @@
+<p>This page shows a single article with all of its meta informations in the system. The page is divided into four tabs: into, the start tab, shows general article info, the topic distribution of this article, a list of most similar articles and the article text, the entities tab shows a list of entities detected in the text, the words tab shows a list of words used in this article and the network tab is a link to the network display with this article as a start node.</p>
+
+<table class="help-listing">
+ <tr>
+ <td><kbd>1</kbd></td>
+ <td>Article title</td>
+ </tr>
+ <tr>
+ <td><kbd>2</kbd></td>
+ <td>Info tab</td>
+ </tr>
+ <tr>
+ <td><kbd>3</kbd></td>
+ <td>Entities tab</td>
+ </tr>
+ <tr>
+ <td><kbd>4</kbd></td>
+ <td>Words tab</td>
+ </tr>
+ <tr>
+ <td><kbd>5</kbd></td>
+ <td>Network link with article as start node</td>
+ </tr>
+ <tr>
+ <td><kbd>6</kbd></td>
+ <td>Topic share of this article</td>
+ </tr>
+ <tr>
+ <td><kbd>7</kbd></td>
+ <td>Similar articles of this article</td>
+ </tr>
+</table>
+
+<div class="image">
+ <img class="default" src="img/ui-article.png">
+ <img class="hover" src="img/ui-article-hover.png">
+</div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/help/entities.html b/vipra-ui/app/html/help/entities.html
new file mode 100644
index 0000000000000000000000000000000000000000..948f327efcce694c1f0ab9d63d1848a874a0d135
--- /dev/null
+++ b/vipra-ui/app/html/help/entities.html
@@ -0,0 +1,33 @@
+<p>The entities page shows all entities that were detected in article texts. The entities are paginated to reduce page load and they can be filtered by name and initial. The entity sorting can be changed.</p>
+
+<table class="help-listing">
+ <tr>
+ <td><kbd>1</kbd></td>
+ <td>Entity name</td>
+ </tr>
+ <tr>
+ <td><kbd>2</kbd></td>
+ <td>Entity menu dropdown</td>
+ </tr>
+ <tr>
+ <td><kbd>3</kbd></td>
+ <td>Pagination</td>
+ </tr>
+ <tr>
+ <td><kbd>4</kbd></td>
+ <td>Filter entities by name</td>
+ </tr>
+ <tr>
+ <td><kbd>5</kbd></td>
+ <td>Filter entities by initial of name</td>
+ </tr>
+ <tr>
+ <td><kbd>6</kbd></td>
+ <td>Sort entities by various attributes, sort direction</td>
+ </tr>
+</table>
+
+<div class="image">
+ <img class="default" src="img/ui-entities.png">
+ <img class="hover" src="img/ui-entities-hover.png">
+</div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/help/entities.show.html b/vipra-ui/app/html/help/entities.show.html
new file mode 100644
index 0000000000000000000000000000000000000000..73bc41c05113f2f836761e397ef50c0e88ccb6c0
--- /dev/null
+++ b/vipra-ui/app/html/help/entities.show.html
@@ -0,0 +1,25 @@
+<p>This page shows a single entity in this topic model that was detected in one or more article texts. The info tab shows a list of associated entity types that were extracted from DBPedia and the articles tab shows a list of articles that use this entity.</p>
+
+<table class="help-listing">
+ <tr>
+ <td><kbd>1</kbd></td>
+ <td>Entity name</td>
+ </tr>
+ <tr>
+ <td><kbd>2</kbd></td>
+ <td>Info tab</td>
+ </tr>
+ <tr>
+ <td><kbd>3</kbd></td>
+ <td>Articles tab</td>
+ </tr>
+ <tr>
+ <td><kbd>4</kbd></td>
+ <td>Entity types</td>
+ </tr>
+</table>
+
+<div class="image">
+ <img class="default" src="img/ui-entity.png">
+ <img class="hover" src="img/ui-entity-hover.png">
+</div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/help/explorer.html b/vipra-ui/app/html/help/explorer.html
new file mode 100644
index 0000000000000000000000000000000000000000..3b402b6caafb0e03eb3f05f1b0bd7851cef58182
--- /dev/null
+++ b/vipra-ui/app/html/help/explorer.html
@@ -0,0 +1,57 @@
+<p>The explorer is the core of the VIPRA application. Here, you can explore the topics in the current topic model in a visual way. The page is divided into four areas: top left is the list of available topics, top right shows the selected topics in an area chart, bottom left shows the selected topics and allows the selection of one topic for further examination and bottom right shows a detailed view on a selected topic.</p>
+
+<table class="help-listing">
+ <tr>
+ <td><kbd>1</kbd></td>
+ <td>Topic selection, select all topics, select none, toggle selection</td>
+ </tr>
+ <tr>
+ <td><kbd>2</kbd></td>
+ <td>Sort topics by multiple values, change sort direction</td>
+ </tr>
+ <tr>
+ <td><kbd>3</kbd></td>
+ <td>Filter topics by name</td>
+ </tr>
+ <tr>
+ <td><kbd>4</kbd></td>
+ <td>Avaiable topics</td>
+ </tr>
+ <tr>
+ <td><kbd>5</kbd></td>
+ <td>Chart value display</td>
+ </tr>
+ <tr>
+ <td><kbd>6</kbd></td>
+ <td>Chart draw style</td>
+ </tr>
+ <tr>
+ <td><kbd>7</kbd></td>
+ <td>Chart value stacking</td>
+ </tr>
+ <tr>
+ <td><kbd>8</kbd></td>
+ <td>Reset chart zoom</td>
+ </tr>
+ <tr>
+ <td><kbd>9</kbd></td>
+ <td>Topic relevance chart of selected topics</td>
+ </tr>
+ <tr>
+ <td><kbd>10</kbd></td>
+ <td>Selected topics for refined analysis</td>
+ </tr>
+ <tr>
+ <td><kbd>11</kbd></td>
+ <td>Topic words, sorted by importance</td>
+ </tr>
+ <tr>
+ <td><kbd>12</kbd></td>
+ <td>Word evolution chart of selected words</td>
+ </tr>
+</table>
+
+<div class="image">
+ <img class="default" src="img/ui-explorer.png">
+ <img class="hover" src="img/ui-explorer-hover.png">
+</div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/help/index.html b/vipra-ui/app/html/help/index.html
new file mode 100644
index 0000000000000000000000000000000000000000..77032f93f78b420c734ce7c8cfc2d4236b21bbe3
--- /dev/null
+++ b/vipra-ui/app/html/help/index.html
@@ -0,0 +1,33 @@
+<p>This is the start page of VIPRA. It provides a search mask with an optional advanced search that can limit the results to a date range. Below the search box some database quick stats are shown.</p>
+
+<table class="help-listing">
+ <tr>
+ <td><kbd>1</kbd></td>
+ <td>Search box</td>
+ </tr>
+ <tr>
+ <td><kbd>2</kbd></td>
+ <td>Advanced search toggle</td>
+ </tr>
+ <tr>
+ <td><kbd>3</kbd></td>
+ <td>From date</td>
+ </tr>
+ <tr>
+ <td><kbd>4</kbd></td>
+ <td>To date</td>
+ </tr>
+ <tr>
+ <td><kbd>5</kbd></td>
+ <td>Database quick stats</td>
+ </tr>
+ <tr>
+ <td><kbd>6</kbd></td>
+ <td>Introduction slides</td>
+ </tr>
+</table>
+
+<div class="image">
+ <img class="default" src="img/ui-start.png">
+ <img class="hover" src="img/ui-start-hover.png">
+</div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/help/network.html b/vipra-ui/app/html/help/network.html
new file mode 100644
index 0000000000000000000000000000000000000000..1207e60321e8fb8b317d2ea65532aaa101036d8b
--- /dev/null
+++ b/vipra-ui/app/html/help/network.html
@@ -0,0 +1,47 @@
+<table class="help-listing">
+ <tr>
+ <td><kbd>1</kbd></td>
+ <td>Filter visible nodes (requires reset)</td>
+ </tr>
+ <tr>
+ <td><kbd>2</kbd></td>
+ <td>Highlight neighbors on node selection, load node relations on click</td>
+ </tr>
+ <tr>
+ <td><kbd>3</kbd></td>
+ <td>Filter visible nodes by name</td>
+ </tr>
+ <tr>
+ <td><kbd>4</kbd></td>
+ <td>Reset network to initial state</td>
+ </tr>
+ <tr>
+ <td><kbd>5</kbd></td>
+ <td>Zoom network to fit window</td>
+ </tr>
+ <tr>
+ <td><kbd>6</kbd></td>
+ <td>Pause/resume network physics simulation</td>
+ </tr>
+ <tr>
+ <td><kbd>7</kbd></td>
+ <td>Load all non-loaded relations of visible nodes</td>
+ </tr>
+ <tr>
+ <td><kbd>8</kbd></td>
+ <td>Article details when selecting an article node</td>
+ </tr>
+ <tr>
+ <td><kbd>9</kbd></td>
+ <td>Article node, square</td>
+ </tr>
+ <tr>
+ <td><kbd>10</kbd></td>
+ <td>Topic node, triangle</td>
+ </tr>
+</table>
+
+<div class="image">
+ <img class="default" src="img/ui-network.png">
+ <img class="hover" src="img/ui-network-hover.png">
+</div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/help/topics.html b/vipra-ui/app/html/help/topics.html
new file mode 100644
index 0000000000000000000000000000000000000000..2ff4966961f06f2c5275da99b00af5b8b0209565
--- /dev/null
+++ b/vipra-ui/app/html/help/topics.html
@@ -0,0 +1,37 @@
+<p>The topics page shows all generated topics in this topic model. The topics are paginated to reduce page load and they can be filtered by name and initial. The topic sorting can be changed.</p>
+
+<table class="help-listing">
+ <tr>
+ <td><kbd>1</kbd></td>
+ <td>Topic title</td>
+ </tr>
+ <tr>
+ <td><kbd>2</kbd></td>
+ <td>Topic menu dropdown</td>
+ </tr>
+ <tr>
+ <td><kbd>3</kbd></td>
+ <td>Topic article count</td>
+ </tr>
+ <tr>
+ <td><kbd>4</kbd></td>
+ <td>Pagination</td>
+ </tr>
+ <tr>
+ <td><kbd>5</kbd></td>
+ <td>Filter topics by name</td>
+ </tr>
+ <tr>
+ <td><kbd>6</kbd></td>
+ <td>Filter topics by initial of name</td>
+ </tr>
+ <tr>
+ <td><kbd>7</kbd></td>
+ <td>Sort topics by various attributes, sort direction</td>
+ </tr>
+</table>
+
+<div class="image">
+ <img class="default" src="img/ui-topics.png">
+ <img class="hover" src="img/ui-topics-hover.png">
+</div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/help/topics.show.html b/vipra-ui/app/html/help/topics.show.html
new file mode 100644
index 0000000000000000000000000000000000000000..5f071b616a110e96fd4795bab5acf96dfd6ff096
--- /dev/null
+++ b/vipra-ui/app/html/help/topics.show.html
@@ -0,0 +1,49 @@
+<p>This page shows a single topic that was generated in the current topic model. The info tab shows the topic relevance evolution, like in the explorer but limited to this topic, below that is the word evolution of the most important words in this topic. The sequences tab shows a list of sequence words and allows the comparison of two sequences by selecting a sequence for comparison on the top right. The articles tab shows a list of articles that reference this topic and the network tab links to the network display with this topic as a start node.</p>
+
+<table class="help-listing">
+ <tr>
+ <td><kbd>1</kbd></td>
+ <td>Topic name, can be renamed by selecting Actions > Rename</td>
+ </tr>
+ <tr>
+ <td><kbd>2</kbd></td>
+ <td>Info tab</td>
+ </tr>
+ <tr>
+ <td><kbd>3</kbd></td>
+ <td>Sequences tab</td>
+ </tr>
+ <tr>
+ <td><kbd>4</kbd></td>
+ <td>Articles tab</td>
+ </tr>
+ <tr>
+ <td><kbd>5</kbd></td>
+ <td>Network link with article as start node</td>
+ </tr>
+ <tr>
+ <td><kbd>6</kbd></td>
+ <td>Topic actions</td>
+ </tr>
+ <tr>
+ <td><kbd>7</kbd></td>
+ <td>Chart value display</td>
+ </tr>
+ <tr>
+ <td><kbd>8</kbd></td>
+ <td>Chart draw style</td>
+ </tr>
+ <tr>
+ <td><kbd>9</kbd></td>
+ <td>Reset chart zoom</td>
+ </tr>
+ <tr>
+ <td><kbd>10</kbd></td>
+ <td>Topic relevance chart limited to this topic</td>
+ </tr>
+</table>
+
+<div class="image">
+ <img class="default" src="img/ui-topic.png">
+ <img class="hover" src="img/ui-topic-hover.png">
+</div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/help/words.html b/vipra-ui/app/html/help/words.html
new file mode 100644
index 0000000000000000000000000000000000000000..dd1f71859f9103f735db47cd278d9e5cc665c64d
--- /dev/null
+++ b/vipra-ui/app/html/help/words.html
@@ -0,0 +1,33 @@
+<p>The words page shows all words that were used in any imported article. The words are paginated to reduce page load and they can be filtered by name and initial. The word sorting can be changed.</p>
+
+<table class="help-listing">
+ <tr>
+ <td><kbd>1</kbd></td>
+ <td>Word name</td>
+ </tr>
+ <tr>
+ <td><kbd>2</kbd></td>
+ <td>Word menu dropdown</td>
+ </tr>
+ <tr>
+ <td><kbd>3</kbd></td>
+ <td>Pagination</td>
+ </tr>
+ <tr>
+ <td><kbd>4</kbd></td>
+ <td>Filter words by name</td>
+ </tr>
+ <tr>
+ <td><kbd>5</kbd></td>
+ <td>Filter words by initial of name</td>
+ </tr>
+ <tr>
+ <td><kbd>6</kbd></td>
+ <td>Sort words by various attributes, sort direction</td>
+ </tr>
+</table>
+
+<div class="image">
+ <img class="default" src="img/ui-words.png">
+ <img class="hover" src="img/ui-words-hover.png">
+</div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/help/words.show.html b/vipra-ui/app/html/help/words.show.html
new file mode 100644
index 0000000000000000000000000000000000000000..a9999523469e2c444cbabcecb6a5db34992f1c1f
--- /dev/null
+++ b/vipra-ui/app/html/help/words.show.html
@@ -0,0 +1,25 @@
+<p>This page shows a single word used in one or multiple articles in this topic model. The topics tab shows a list of topics that reference this word and the articles tab shows a list of articles that contain this word.</p>
+
+<table class="help-listing">
+ <tr>
+ <td><kbd>1</kbd></td>
+ <td>Entity name</td>
+ </tr>
+ <tr>
+ <td><kbd>2</kbd></td>
+ <td>Info tab</td>
+ </tr>
+ <tr>
+ <td><kbd>3</kbd></td>
+ <td>Topics tab</td>
+ </tr>
+ <tr>
+ <td><kbd>4</kbd></td>
+ <td>Articles tab</td>
+ </tr>
+</table>
+
+<div class="image">
+ <img class="default" src="img/ui-word.png">
+ <img class="hover" src="img/ui-word-hover.png">
+</div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/index.html b/vipra-ui/app/html/index.html
index 08e17e3b6a66ef785d4c5193f5a87ccb274982a7..136e642100664fc8e3ae8a728d538b685dfa3093 100644
--- a/vipra-ui/app/html/index.html
+++ b/vipra-ui/app/html/index.html
@@ -54,18 +54,9 @@
<div class="container-fluid" ng-show="!searching && !search">
<div class="container">
<div class="row">
- <div class="col-xs-offset-1 col-xs-5 text-center">
- <h3>Screencast</h3>
- <div class="embed-responsive embed-responsive-16by9 media-box">
- <video controls muted poster="img/vipra-screencast.png">
- <source src="//ftp.cochu.io/vipra/screencast.mp4" type="video/mp4">
- <source src="//ftp.cochu.io/vipra/screencast.webm" type="video/webm">
- Your browser does not support the video tag.
- </video>
- </div>
- </div>
- <div class="col-xs-5 text-center">
- <h3>Slides</h3>
+ <div class="col-xs-offset-4 col-xs-4 text-center">
+ <br>
+ <h3>Introduction</h3>
<div class="embed-responsive embed-responsive-16by9 media-box">
<a ui-sref="slides">
<img src="img/vipra-slides.png" class="slides-thumb">
diff --git a/vipra-ui/app/html/network.html b/vipra-ui/app/html/network.html
index d410777f47cd522ce4b3a0fe7245b00db93e4944..d17c63a3a8429c85ae994387a78f45394c59b430 100644
--- a/vipra-ui/app/html/network.html
+++ b/vipra-ui/app/html/network.html
@@ -24,6 +24,10 @@
<input type="checkbox" id="highlightEnabled" ng-model="highlightEnabled">
<label for="highlightEnabled" analytics-on analytics-event="Network Highlight Enable" analytics-category="Network actions">Neighborhood highlight</label>
</div>
+ <div class="checkbox">
+ <input type="checkbox" id="loadOnClick" ng-model="loadOnClick">
+ <label for="loadOnClick" analytics-on analytics-event="Network Click Load Enable" analytics-category="Network actions">Load on click</label>
+ </div>
<div style="display:none">
Window
<window-dropdown ng-model="selectedWindow" windows="windows" />
diff --git a/vipra-ui/app/html/partials/topicmodel-popover.html b/vipra-ui/app/html/partials/topicmodel-popover.html
deleted file mode 100644
index 585b8ee94eb9e5f10cbb96691054b4806ccf3133..0000000000000000000000000000000000000000
--- a/vipra-ui/app/html/partials/topicmodel-popover.html
+++ /dev/null
@@ -1,30 +0,0 @@
-<table class="td-padded">
- <tr>
- <th>Articles</th>
- <td ng-bind="::topicModel.articleCount || 0"></td>
- </tr>
- <tr>
- <th>Topics</th>
- <td ng-bind="::topicModel.topicCount || 0"></td>
- </tr>
- <tr>
- <th>Words</th>
- <td ng-bind="::topicModel.wordCount || 0"></td>
- </tr>
- <tr>
- <th>Entities</th>
- <td ng-bind="::topicModel.entityCount || 0"></td>
- </tr>
- <tr>
- <th>Windows</th>
- <td ng-bind="::topicModel.windowCount || 0"></td>
- </tr>
- <tr>
- <th>Last generated</th>
- <td ng-bind-template="{{::topicModel.lastGeneratedString}}"></td>
- </tr>
- <tr>
- <th>Last indexed</th>
- <td ng-bind-template="{{::topicModel.lastIndexedString}}"></td>
- </tr>
-</table>
\ No newline at end of file
diff --git a/vipra-ui/app/html/words/articles.html b/vipra-ui/app/html/words/articles.html
index 14d2247e70b3332e824d482e9bb6914f3abd34bf..fa3e7a219b9b9ffd9e77ea2a9ca30b99dbe6f69d 100644
--- a/vipra-ui/app/html/words/articles.html
+++ b/vipra-ui/app/html/words/articles.html
@@ -1,58 +1,69 @@
<div class="container" ng-cloak ng-hide="!rootModels.topicModel || state.name !== 'words.show.articles'">
- <div class="row">
- <div class="col-md-12">
- <div class="page-header">
- <h1 ng-bind-template="Articles for word '{{::word}}'"></h1>
- <table class="item-actions">
- <tr>
- <td>
- <a class="btn btn-default" ng-click="goBack()" ng-show="oldState.name && oldState.name !== state.name" ng-cloak>Back</a>
- </td>
- </tr>
- </table>
- </div>
- </div>
- </div>
- <div class="row">
- <div class="col-md-12 text-center">
- <pagination total="articlesTotal" page="wordsArticlesModels.page" limit="wordsArticlesModels.limit" />
- </div>
+ <div class="page-header no-border">
+ <span class="label label-default">Word</span>
+ <h1 ng-bind="::word"></h1>
</div>
- <div class="row">
- <div class="col-md-12">
- <div class="panel panel-default">
- <div class="panel-heading">
- Found
- <ng-pluralize count="articlesTotal||0" when="{0:'no articles',1:'1 article',other:'{} articles'}"></ng-pluralize> in the database.
- <span ng-show="articlesTotal" ng-cloak>
- Sort by
- <ol class="nya-bs-select nya-bs-condensed" ng-model="wordsArticlesModels.sortkey">
- <li value="title" class="nya-bs-option"><a>Title</a></li>
- <li value="date" class="nya-bs-option"><a>Date</a></li>
- <li value="created" class="nya-bs-option"><a>Added</a></li>
- </ol>
- <sort-dir ng-model="wordsArticlesModels.sortdir" />
- </span>
+ <div>
+ <ul class="nav nav-tabs" role="tablist">
+ <li>
+ <a ui-sref="words.show({id:word})"><i class="fa fa-file-text-o"></i></a>
+ </li>
+ <li>
+ <a ui-sref="words.show.topics({id:word})">
+ Topics
+ </a>
+ </li>
+ <li class="active">
+ <a>
+ Articles
+ </a>
+ </li>
+ </ul>
+ <div class="tab-content">
+ <div role="tabpanel" class="tab-pane active">
+ <div class="row">
+ <br>
+ <div class="col-md-12">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Found
+ <ng-pluralize count="articlesTotal||0" when="{0:'no articles',1:'1 article',other:'{} articles'}"></ng-pluralize> in the database.
+ <span ng-show="articlesTotal" ng-cloak>
+ Sort by
+ <div class="input-group inline">
+ <select class="form-control" ng-model="wordsArticlesModels.sortkey">
+ <option value="title">Title</option>
+ <option value="date">Date</option>
+ <option value="created">Added</option>
+ </select>
+ <span class="input-group-btn">
+ <sort-dir ng-model="wordsArticlesModels.sortdir" />
+ </span>
+ </div>
+ </span>
+ </div>
+ <table class="table table-hover table-condensed table-fixed">
+ <tbody>
+ <tr ng-repeat="article in articles">
+ <td>
+ <article-link article="::article"/>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ <div class="panel-footer">
+ Page <span ng-bind="wordsArticlesModels.page||1"></span> of <span ng-bind="maxPage||1"></span>
+ </div>
+ </div>
+ </div>
</div>
- <table class="table table-hover table-condensed table-fixed">
- <tbody>
- <tr ng-repeat="article in articles">
- <td>
- <article-link article="::article"/>
- </td>
- </tr>
- </tbody>
- </table>
- <div class="panel-footer">
- Page <span ng-bind="wordsArticlesModels.page||1"></span> of <span ng-bind="maxPage||1"></span>
+ <div class="row">
+ <div class="col-md-12 text-center">
+ <pagination total="articlesTotal" page="wordsArticlesModels.page" limit="wordsArticlesModels.limit" />
+ </div>
</div>
</div>
</div>
</div>
- <div class="row">
- <div class="col-md-12 text-center">
- <pagination total="articlesTotal" page="wordsArticlesModels.page" limit="wordsArticlesModels.limit" />
- </div>
- </div>
</div>
<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/html/words/show.html b/vipra-ui/app/html/words/show.html
index c6eafbb8d45ceda3b4375bf2453c849faf8241e7..183a37d0fc0e2f5ea933a2fd80e3cb3cb3dc1abb 100644
--- a/vipra-ui/app/html/words/show.html
+++ b/vipra-ui/app/html/words/show.html
@@ -25,8 +25,8 @@
<table class="table table-bordered table-condensed table-fixed">
<tbody>
<tr>
- <th class="infocol">ID</th>
- <td ng-bind="::word.id"></td>
+ <th class="infocol">Word</th>
+ <td ng-bind="::word.word"></td>
</tr>
</tbody>
</table>
diff --git a/vipra-ui/app/html/words/topics.html b/vipra-ui/app/html/words/topics.html
index a4fe2e10835cb788dcb82bf3f6a70f84122d686d..69482210a13b7b01505c295600822677cf77aa9d 100644
--- a/vipra-ui/app/html/words/topics.html
+++ b/vipra-ui/app/html/words/topics.html
@@ -1,57 +1,68 @@
<div class="container" ng-cloak ng-hide="!rootModels.topicModel || state.name !== 'words.show.topics'">
- <div class="row">
- <div class="col-md-12">
- <div class="page-header">
- <h1 ng-bind-template="Topics for word '{{::word}}'"></h1>
- <table class="item-actions">
- <tr>
- <td>
- <a class="btn btn-default" ng-click="goBack()" ng-show="oldState.name && oldState.name !== state.name" ng-cloak>Back</a>
- </td>
- </tr>
- </table>
- </div>
- </div>
- </div>
- <div class="row">
- <div class="col-md-12 text-center">
- <pagination total="topicsTotal" page="wordsTopicsModels.page" limit="wordsTopicsModels.limit" />
- </div>
+ <div class="page-header no-border">
+ <span class="label label-default">Word</span>
+ <h1 ng-bind="::word"></h1>
</div>
- <div class="row">
- <div class="col-md-12">
- <div class="panel panel-default">
- <div class="panel-heading">
- Found
- <ng-pluralize count="topicsTotal||0" when="{0:'no topics',1:'1 topic',other:'{} topics'}"></ng-pluralize> in the database.
- <span ng-show="topicsTotal" ng-cloak>
- Sort by
- <ol class="nya-bs-select nya-bs-condensed" ng-model="wordsTopicsModels.sortkey">
- <li value="name" class="nya-bs-option"><a>Name</a></li>
- <li value="created" class="nya-bs-option"><a>Added</a></li>
- </ol>
- <sort-dir ng-model="wordsTopicsModels.sortdir" />
- </span>
+ <div>
+ <ul class="nav nav-tabs" role="tablist">
+ <li>
+ <a ui-sref="words.show({id:word})"><i class="fa fa-file-text-o"></i></a>
+ </li>
+ <li class="active">
+ <a>
+ Topics
+ </a>
+ </li>
+ <li>
+ <a ui-sref="words.show.articles({id:word})">
+ Articles
+ </a>
+ </li>
+ </ul>
+ <div class="tab-content">
+ <div role="tabpanel" class="tab-pane active">
+ <div class="row">
+ <br>
+ <div class="col-md-12">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Found
+ <ng-pluralize count="topicsTotal||0" when="{0:'no topics',1:'1 topic',other:'{} topics'}"></ng-pluralize> in the database.
+ <span ng-show="topicsTotal" ng-cloak>
+ Sort by
+ <div class="input-group inline">
+ <select class="form-control" ng-model="wordsTopicsModels.sortkey">
+ <option value="name">Name</option>
+ <option value="created">Added</option>
+ </select>
+ <span class="input-group-btn">
+ <sort-dir ng-model="wordsTopicsModels.sortdir" />
+ </span>
+ </div>
+ </span>
+ </div>
+ <table class="table table-hover table-condensed table-fixed">
+ <tbody>
+ <tr ng-repeat="topic in topics">
+ <td>
+ <topic-link topic="::topic" />
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ <div class="panel-footer">
+ Page <span ng-bind="wordsTopicsModels.page||1"></span> of <span ng-bind="maxPage||1"></span>
+ </div>
+ </div>
+ </div>
</div>
- <table class="table table-hover table-condensed table-fixed">
- <tbody>
- <tr ng-repeat="topic in topics">
- <td>
- <topic-link topic="::topic" />
- </td>
- </tr>
- </tbody>
- </table>
- <div class="panel-footer">
- Page <span ng-bind="wordsTopicsModels.page||1"></span> of <span ng-bind="maxPage||1"></span>
+ <div class="row">
+ <div class="col-md-12 text-center">
+ <pagination total="topicsTotal" page="wordsTopicsModels.page" limit="wordsTopicsModels.limit" />
+ </div>
</div>
</div>
</div>
</div>
- <div class="row">
- <div class="col-md-12 text-center">
- <pagination total="topicsTotal" page="wordsTopicsModels.page" limit="wordsTopicsModels.limit" />
- </div>
- </div>
</div>
-<div ng-cloak ui-view></div>
+<div ng-cloak ui-view></div>
\ No newline at end of file
diff --git a/vipra-ui/app/img/ui-article-hover.png b/vipra-ui/app/img/ui-article-hover.png
new file mode 100644
index 0000000000000000000000000000000000000000..d559761daa72e5688db79bd6cb78afb9110ff428
Binary files /dev/null and b/vipra-ui/app/img/ui-article-hover.png differ
diff --git a/vipra-ui/app/img/ui-article.png b/vipra-ui/app/img/ui-article.png
new file mode 100644
index 0000000000000000000000000000000000000000..1729d6217c0090bdaddb160029fc987a182e302c
Binary files /dev/null and b/vipra-ui/app/img/ui-article.png differ
diff --git a/vipra-ui/app/img/ui-articles-hover.png b/vipra-ui/app/img/ui-articles-hover.png
new file mode 100644
index 0000000000000000000000000000000000000000..22faabcccb8c0ddfc38be806a7ee3f038715b7c3
Binary files /dev/null and b/vipra-ui/app/img/ui-articles-hover.png differ
diff --git a/vipra-ui/app/img/ui-articles.png b/vipra-ui/app/img/ui-articles.png
new file mode 100644
index 0000000000000000000000000000000000000000..ff027a78596e0493da0fba829ab1428715cd93f3
Binary files /dev/null and b/vipra-ui/app/img/ui-articles.png differ
diff --git a/vipra-ui/app/img/ui-entities-hover.png b/vipra-ui/app/img/ui-entities-hover.png
new file mode 100644
index 0000000000000000000000000000000000000000..56092fdca9f61d34ea82efa22441d9c16e0b7c28
Binary files /dev/null and b/vipra-ui/app/img/ui-entities-hover.png differ
diff --git a/vipra-ui/app/img/ui-entities.png b/vipra-ui/app/img/ui-entities.png
new file mode 100644
index 0000000000000000000000000000000000000000..3e7d181ee12992517a52559eec9be03923e18666
Binary files /dev/null and b/vipra-ui/app/img/ui-entities.png differ
diff --git a/vipra-ui/app/img/ui-entity-hover.png b/vipra-ui/app/img/ui-entity-hover.png
new file mode 100644
index 0000000000000000000000000000000000000000..4331895b334bec6f751b1721294d771941144806
Binary files /dev/null and b/vipra-ui/app/img/ui-entity-hover.png differ
diff --git a/vipra-ui/app/img/ui-entity.png b/vipra-ui/app/img/ui-entity.png
new file mode 100644
index 0000000000000000000000000000000000000000..907e2df263c95ac17ea02e3594d211884b9d3c7e
Binary files /dev/null and b/vipra-ui/app/img/ui-entity.png differ
diff --git a/vipra-ui/app/img/ui-explorer-hover.png b/vipra-ui/app/img/ui-explorer-hover.png
new file mode 100644
index 0000000000000000000000000000000000000000..6c91409a7d1141387f49ec2610bcafa9a7b81e71
Binary files /dev/null and b/vipra-ui/app/img/ui-explorer-hover.png differ
diff --git a/vipra-ui/app/img/ui-explorer.png b/vipra-ui/app/img/ui-explorer.png
new file mode 100644
index 0000000000000000000000000000000000000000..ceb5ee4c67e189d471ce4c131737a1f8d2dac6a8
Binary files /dev/null and b/vipra-ui/app/img/ui-explorer.png differ
diff --git a/vipra-ui/app/img/ui-network-hover.png b/vipra-ui/app/img/ui-network-hover.png
new file mode 100644
index 0000000000000000000000000000000000000000..c7216ae5d21c01d4fed64d2c6c00e7dd55583a46
Binary files /dev/null and b/vipra-ui/app/img/ui-network-hover.png differ
diff --git a/vipra-ui/app/img/ui-network.png b/vipra-ui/app/img/ui-network.png
new file mode 100644
index 0000000000000000000000000000000000000000..17061c8dcf9c1a4ab804c1477cf89256799a3767
Binary files /dev/null and b/vipra-ui/app/img/ui-network.png differ
diff --git a/vipra-ui/app/img/ui-start-hover.png b/vipra-ui/app/img/ui-start-hover.png
new file mode 100644
index 0000000000000000000000000000000000000000..3c302933eda5ca90b9e3ef8d1e207c81acd7baec
Binary files /dev/null and b/vipra-ui/app/img/ui-start-hover.png differ
diff --git a/vipra-ui/app/img/ui-start.png b/vipra-ui/app/img/ui-start.png
new file mode 100644
index 0000000000000000000000000000000000000000..5b243bb69e8aba448f8220c9e699a81e3d80216e
Binary files /dev/null and b/vipra-ui/app/img/ui-start.png differ
diff --git a/vipra-ui/app/img/ui-topic-hover.png b/vipra-ui/app/img/ui-topic-hover.png
new file mode 100644
index 0000000000000000000000000000000000000000..4e58ab38fd2c78f5e1a7fe1d684f83583ede755a
Binary files /dev/null and b/vipra-ui/app/img/ui-topic-hover.png differ
diff --git a/vipra-ui/app/img/ui-topic.png b/vipra-ui/app/img/ui-topic.png
new file mode 100644
index 0000000000000000000000000000000000000000..c64f637ff388de08bd5ef618327dc2de8b76a63d
Binary files /dev/null and b/vipra-ui/app/img/ui-topic.png differ
diff --git a/vipra-ui/app/img/ui-topics-hover.png b/vipra-ui/app/img/ui-topics-hover.png
new file mode 100644
index 0000000000000000000000000000000000000000..4018877801bd8b2b4d7f7f7bae32feaa0f240027
Binary files /dev/null and b/vipra-ui/app/img/ui-topics-hover.png differ
diff --git a/vipra-ui/app/img/ui-topics.png b/vipra-ui/app/img/ui-topics.png
new file mode 100644
index 0000000000000000000000000000000000000000..1bb2ac12dbefbba872a04e65100d3b15280ddbbf
Binary files /dev/null and b/vipra-ui/app/img/ui-topics.png differ
diff --git a/vipra-ui/app/img/ui-word-hover.png b/vipra-ui/app/img/ui-word-hover.png
new file mode 100644
index 0000000000000000000000000000000000000000..840cd7ee65ca68a9c960dfa23abb19d1c5e61ce1
Binary files /dev/null and b/vipra-ui/app/img/ui-word-hover.png differ
diff --git a/vipra-ui/app/img/ui-word.png b/vipra-ui/app/img/ui-word.png
new file mode 100644
index 0000000000000000000000000000000000000000..286aff0490cb9eac6e7890a9e7cfc82db61b230f
Binary files /dev/null and b/vipra-ui/app/img/ui-word.png differ
diff --git a/vipra-ui/app/img/ui-words-hover.png b/vipra-ui/app/img/ui-words-hover.png
new file mode 100644
index 0000000000000000000000000000000000000000..1696d928d3e988a711a19381fe7764bee92b0ac5
Binary files /dev/null and b/vipra-ui/app/img/ui-words-hover.png differ
diff --git a/vipra-ui/app/img/ui-words.png b/vipra-ui/app/img/ui-words.png
new file mode 100644
index 0000000000000000000000000000000000000000..76e3ecd5b974de8488ba23ada3134379bc9d22a3
Binary files /dev/null and b/vipra-ui/app/img/ui-words.png differ
diff --git a/vipra-ui/app/index.html b/vipra-ui/app/index.html
index a675f363832fb10673a44ffa23d62c81f273ed15..5286a53a7f863c41e0e197d48072377d8607a77f 100644
--- a/vipra-ui/app/index.html
+++ b/vipra-ui/app/index.html
@@ -56,6 +56,11 @@
<li ng-class="{'text-italic active':rootModels.topicModel}">
<a class="active-topicmodel" tabindex="0" ng-click="chooseTopicModel()" ng-bind-template="{{rootModels.topicModel ? rootModels.topicModel.name : 'Models'}}" ng-attr-title="{{rootModels.topicModel.name}} {{rootModels.topicModel.modelConfig.description}}" analytics-on analytics-event="Menu Item (Topic Models)" analytics-category="Menu actions"></a>
</li>
+ <li ng-class="{disabled:!rootModels.helpEnabled}" ng-attr-title="{{ rootModels.helpEnabled ? 'Help' : 'No help available' }}">
+ <a ng-click="showHelp()">
+ <i class="fa fa-question-circle"></i>
+ </a>
+ </li>
<li title="Keyboard cheatsheet">
<a tabindex="0" ng-click="showCheatSheet()" analytics-on analytics-event="Menu Item (Cheatsheet)" analytics-category="Menu actions">
<i class="fa fa-keyboard-o"></i>
@@ -63,7 +68,7 @@
</li>
<li ui-sref-active="active" title="About">
<a tabindex="0" ui-sref="about">
- <i class="fa fa-question-circle"></i>
+ <i class="fa fa-info-circle"></i>
</a>
</li>
<li title="Feedback">
@@ -89,7 +94,7 @@
<i class="form-control-feedback glyphicon glyphicon-search text-muted"></i>
</div>
<div class="list-group nomargin" ng-show="topicModels.length" ng-cloak>
- <button type="button" class="list-group-item topic-model" ng-repeat="topicModel in topicModels | filter:{name:rootModels.topicModelFilter}" ng-click="changeTopicModel(topicModel)" ng-class="{'active selected-model':rootModels.topicModel.id===topicModel.id}" analytics-on analytics-event="Topic model" analytics-category="Topic model actions" bs-popover popover-title="{{::topicModel.name}}" popover-template="partials/topicmodel-popover.html" popover-placement="bottom">
+ <button type="button" class="list-group-item topic-model" ng-repeat="topicModel in topicModels | filter:{name:rootModels.topicModelFilter}" ng-click="changeTopicModel(topicModel)" ng-class="{'active selected-model':rootModels.topicModel.id===topicModel.id}" analytics-on analytics-event="Topic model" analytics-category="Topic model actions">
<span class="badge badge-group">
<span class="badge-part" ng-if="!topicModel.lastGenerated" title="Model was never generated">Non-generated</span>
<span class="badge-part" ng-bind="::topicModel.articleCount" ng-show="topicModel.articleCount" ng-attr-title="{{::topicModel.articleCount + ' article(s)'}}" ng-cloak></span>
@@ -140,6 +145,17 @@
</div>
</div>
</div>
+ <div id="helpModal" class="modal" tabindex="-1" role="dialog" data-backdrop="static">
+ <div class="modal-dialog modal-lg">
+ <div class="modal-content">
+ <div class="modal-header">
+ <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">×</span></button>
+ <h4 class="modal-title">Help</h4>
+ </div>
+ <div class="modal-body help-body" help-content enabled="rootModels.helpEnabled"></div>
+ </div>
+ </div>
+ </div>
<div class="alerts"></div>
<div class="overlay-message" ng-show="fatal">
<div class="message" ng-bind="fatal"></div>
diff --git a/vipra-ui/app/js/app.js b/vipra-ui/app/js/app.js
index c73784eed3a9bc1763bbc267a04bfdb19f795957..e9fe0924746ff2b55fa044451d684a08fc8a629a 100644
--- a/vipra-ui/app/js/app.js
+++ b/vipra-ui/app/js/app.js
@@ -223,12 +223,16 @@
$rootScope.Vipra = window.Vipra;
- $rootScope.$on('$stateChangeError', function() {
+ $rootScope.error = function(code) {
$state.transitionTo('error', {
- code: 404
+ code: code
}, {
location: 'replace'
});
+ };
+
+ $rootScope.$on('$stateChangeError', function() {
+ $rootScope.error(404);
});
$rootScope.$on('$stateChangeSuccess', function(event, toState, toParams, fromState) {
diff --git a/vipra-ui/app/js/controllers.js b/vipra-ui/app/js/controllers.js
index 1d6d78582108d11935e774ccbf6b8ebf2af61839..f1b3f83f37de44f6c878d634b1c1b9554783fce9 100644
--- a/vipra-ui/app/js/controllers.js
+++ b/vipra-ui/app/js/controllers.js
@@ -9,13 +9,14 @@
var app = angular.module('vipra.controllers', []);
- app.controller('RootController', ['$scope', '$rootScope', '$q', '$state', '$window', 'hotkeys', 'TopicModelFactory',
- function($scope, $rootScope, $q, $state, $window, hotkeys, TopicModelFactory) {
+ app.controller('RootController', ['$scope', '$rootScope', '$q', '$state', '$window', '$timeout', 'hotkeys', 'TopicModelFactory',
+ function($scope, $rootScope, $q, $state, $window, $timeout, hotkeys, TopicModelFactory) {
$scope.rootModels = {
topicModel: null,
search: null,
- title: 'Home'
+ title: 'Home',
+ helpEnabled: false
};
var prevTopicModelLoading = false;
@@ -72,7 +73,6 @@
$scope.changeTopicModel = function(topicModel) {
$scope.rootModels.topicModel = topicModel;
$('#topicModelModal').modal('hide');
- $('.popover').popover('hide');
localStorage.tm = topicModel.id;
if($state.current.name != 'index')
$state.transitionTo('index');
@@ -168,7 +168,7 @@
});
hotkeys.add({
- combo: 'i',
+ combo: 't',
description: 'Go to entities',
callback: function() {
if ($scope.rootModels.topicModel && $state.current.name !== 'entities')
@@ -186,14 +186,20 @@
});
hotkeys.add({
- combo: 'm',
- description: 'Choose a topic model',
+ combo: 'h',
+ description: 'Show help informations',
callback: function() {
- $scope.chooseTopicModel();
+ $scope.showHelp();
}
});
$scope.showCheatSheet = hotkeys.toggleCheatSheet;
+
+ $scope.showHelp = function() {
+ if($scope.rootModels.helpEnabled) {
+ $('#helpModal').modal();
+ }
+ };
}
]);
@@ -294,6 +300,7 @@
$scope.physicsEnabled = true;
$scope.highlightEnabled = true;
+ $scope.loadOnClick = true;
var level2Color = 'rgba(130,130,130,0.8)';
var level3Color = 'rgba(220,220,220,0.4)';
@@ -354,7 +361,7 @@
words: false
};
- var newNode = function(title, type, show, dbid, color, shape, loader, x, y) {
+ var newNode = function(title, type, show, dbid, color, shape, loader) {
ids[dbid] = ++id;
return {
id: id,
@@ -470,6 +477,7 @@
// on node select
var nodeLoader = null;
$scope.select = function(props) {
+ if(!$scope.loadOnClick) return;
var node = $scope.nodes.get(props.nodes[0]);
if (node) {
nodeLoader = node.loader(node, props);
@@ -873,7 +881,7 @@
$timeout(function() {
$('#outer').layout({
applyDefaultStyles: true,
- south__size: '50%',
+ south__size: '45%',
center__childOptions: {
applyDefaultStyles: true,
center__onresize: function() {
@@ -1189,9 +1197,6 @@
}, function(data) {
$scope.article = data;
$scope.article.text = $scope.prepareText($scope.article.text);
- $scope.articleDate = Vipra.formatDate($scope.article.date);
- $scope.articleCreated = Vipra.formatDateTime($scope.article.created);
- $scope.articleModified = Vipra.formatDateTime($scope.article.modified);
$scope.rootModels.title = $scope.article.title;
// calculate share from divergence
@@ -1234,6 +1239,8 @@
data: topicShareSeries
}]);
}, 0);
+ }, function(e) {
+ $scope.error(e.status);
});
$scope.openTabWords = function() {
@@ -1252,11 +1259,13 @@
$scope.showMoreWords = function() {
wordsCount += 20;
$scope.words = $scope.allWords.slice(0, wordsCount);
+ $scope.refreshWordDistribution();
};
$scope.showAllWords = function() {
wordsCount = $scope.allWords.length;
$scope.words = $scope.allWords;
+ $scope.refreshWordDistribution();
};
$scope.openTabEntities = function() {
@@ -1275,16 +1284,18 @@
$scope.showMoreEntities = function() {
entitiesCount += 20;
$scope.entities = $scope.allEntities.slice(0, entitiesCount);
+ $scope.refreshEntityDistribution();
};
$scope.showAllEntities = function() {
entitiesCount = $scope.allEntities.length;
$scope.entities = $scope.allEntities;
+ $scope.refreshEntityDistribution();
};
$scope.prepareText = function(text) {
var base = $state.href('entities', {}, {absolute: true});
- return text.replace(/href="[^>]+>([^<]+)/g, 'href="' + base + '/$1">$1');
+ return text.replace(/href="[^"]*" data-entity="([^"]*)"/g, 'href="' + base + '/$1"');
};
var topicShareChartElement = $('#topic-share');
@@ -1301,6 +1312,32 @@
highcharts.tooltip.hide();
}
};
+
+ $scope.refreshEntityDistribution = function() {
+ var series = [];
+ if($scope.entities && $scope.entities.length) {
+ for(var i = 0; i < $scope.entities.length; i++) {
+ series.push({
+ name: $scope.entities[i].entity.entity,
+ data: [$scope.entities[i].count]
+ });
+ }
+ }
+ $scope.entityDistribution = itemCountChart(series, ['Entities']);
+ };
+
+ $scope.refreshWordDistribution = function() {
+ var series = [];
+ if($scope.words && $scope.words.length) {
+ for(var i = 0; i < $scope.words.length; i++) {
+ series.push({
+ name: $scope.words[i].word,
+ data: [$scope.words[i].count]
+ });
+ }
+ }
+ $scope.wordDistribution = itemCountChart(series, ['Words']);
+ };
}
]);
@@ -1375,8 +1412,6 @@
id: $stateParams.id
}, function(data) {
$scope.topic = data;
- $scope.topicCreated = Vipra.formatDateTime($scope.topic.created);
- $scope.topicModified = Vipra.formatDateTime($scope.topic.modified);
$scope.rootModels.title = $scope.topic.name;
// take topic model from topic
@@ -1394,7 +1429,9 @@
$scope.topicsShowModels.sequence = $scope.topic.sequences[0];
$scope.redrawRelevanceGraph();
- });
+ }, function(e) {
+ $scope.error(e.status);
+ });
$scope.redrawRelevanceGraph = function() {
if (!$scope.topic || !$scope.topic.sequences) return;
@@ -1515,6 +1552,14 @@
$('.tab-sequences').on('mouseenter', '.compare-row', function() {
$('[word="' + $(this).attr('word') + '"]').addClass('highlight');
});
+
+ $scope.$on('$stateChangeSuccess', function(e, toState) {
+ if(toState.name === 'topics.show') {
+ $timeout(function() {
+ $scope.redrawRelevanceGraph();
+ }, 100);
+ }
+ });
}
]);
@@ -1541,6 +1586,8 @@
$scope.articles = data;
$scope.articlesTotal = headers("V-Total");
$scope.maxPage = Math.ceil($scope.articlesTotal / $scope.topicsArticlesModels.limit);
+ }, function(e) {
+ $scope.error(e.status);
});
});
@@ -1596,8 +1643,8 @@
}
]);
- app.controller('EntitiesShowController', ['$scope', '$state', '$stateParams', 'EntityFactory',
- function($scope, $state, $stateParams, EntityFactory) {
+ app.controller('EntitiesShowController', ['$scope', '$state', '$stateParams', 'EntityFactory', 'DBPediaFactory',
+ function($scope, $state, $stateParams, EntityFactory, DBPediaFactory) {
$scope.rootModels.title = 'Entity';
$scope.checkTopicModel('entities.show', function() {
@@ -1609,6 +1656,13 @@
$scope.entityCreated = Vipra.formatDateTime($scope.entity.created);
$scope.entityModified = Vipra.formatDateTime($scope.entity.modified);
$scope.rootModels.title = $scope.entity.entity;
+ DBPediaFactory.get($scope.entity.url, function(data) {
+ try {
+ $scope.abstract = data.results.bindings[0].abstract.value;
+ } catch(e) {}
+ });
+ }, function(e) {
+ $scope.error(e.status);
});
});
}
@@ -1707,6 +1761,8 @@
}, function(data) {
$scope.word = data;
$scope.rootModels.title = $scope.word.word;
+ }, function(e) {
+ $scope.error(e.status);
});
});
}
@@ -2021,7 +2077,7 @@
spacingBottom: 0,
spacingTop: 0,
spacingLeft: 0,
- spacingRight: 0,
+ spacingRight: 0
},
credits: {
enabled: false
@@ -2052,4 +2108,42 @@
};
}
+ function itemCountChart(series, categories) {
+ return {
+ chart: {
+ type: 'column',
+ spacingLeft: 0,
+ spacingRight: 0,
+ height: 300
+ },
+ title: {
+ text: null
+ },
+ credits: {
+ enabled: false
+ },
+ xAxis: {
+ categories: categories
+ },
+ yAxis: {
+ title: null
+ },
+ tooltip: {
+ headerFormat: '',
+ pointFormat: '{series.name}: <b>{point.y}</b>'
+ },
+ legend: {
+ enabled: false
+ },
+ noData: {
+ style: {
+ fontSize: "14px",
+ color: "#777",
+ fontWeight: "normal"
+ }
+ },
+ series: series
+ };
+ }
+
})();
\ No newline at end of file
diff --git a/vipra-ui/app/js/directives.js b/vipra-ui/app/js/directives.js
index 90613587bcc59b3b51eb037e3ac3c91d41779bff..4ec8f13ed28de010f235afd022f8273786e5a332 100644
--- a/vipra-ui/app/js/directives.js
+++ b/vipra-ui/app/js/directives.js
@@ -203,7 +203,10 @@
}
var hc = $($elem.data('target')).find('[highcharts]');
if(hc.length) {
- hc.highcharts().reflow();
+ hc = hc.highcharts();
+ if(hc) {
+ hc.reflow();
+ }
}
});
}
@@ -562,4 +565,44 @@
};
}]);
+ app.directive('helpContent', ['$templateRequest', '$sce', '$compile', '$state', function($templateRequest, $sce, $compile, $state) {
+ return {
+ scope: {
+ enabled: '='
+ },
+ link: function($scope, $elem) {
+ $scope.$on('$stateChangeSuccess', function() {
+ if(!$state.current.name) {
+ $scope.enabled = false;
+ return;
+ }
+ var templateUrl = $sce.getTrustedResourceUrl('html/help/' + $state.current.name + '.html');
+ try {
+ $templateRequest(templateUrl, true).then(function(template) {
+ if(template.startsWith('<!DOCTYPE')) {
+ $scope.enabled = false;
+ return;
+ }
+ $compile($elem.html(template).contents())($scope);
+ $scope.enabled = true;
+ }, function() {});
+ } catch(e) {}
+ });
+ }
+ };
+ }]);
+
+ app.directive('compileHtml', ['$compile', function($compile) {
+ return {
+ link: function($scope, $elem, $attrs) {
+ $scope.$watch(function () {
+ return $scope.$eval($attrs.compileHtml);
+ }, function (value) {
+ $elem.html(value);
+ $compile($elem.contents())($scope);
+ });
+ }
+ };
+ }]);
+
})();
\ No newline at end of file
diff --git a/vipra-ui/app/js/factories.js b/vipra-ui/app/js/factories.js
index 49c0546dfed7b99b76585eaadc8bad979e37b5df..485c747d28b9eb9f0926e8563b5a3d3ced0a8d28 100644
--- a/vipra-ui/app/js/factories.js
+++ b/vipra-ui/app/js/factories.js
@@ -88,4 +88,15 @@
return $myResource(Vipra.config.restUrl + '/windows/:id');
}]);
+ app.factory('DBPediaFactory', ['$myResource', function($myResource) {
+ var q1 = 'http://dbpedia.org/sparql?default-graph-uri=http%3A%2F%2Fdbpedia.org&query=select+%3Fabstract+where+%7B+%3C';
+ var q2 = '%3E+%3Chttp%3A%2F%2Fdbpedia.org%2Fontology%2Fabstract%3E+%3Fabstract+filter%28langMatches%28lang%28%3Fabstract%29%2C%22en%22%29%29+%7D';
+ return {
+ get: function(url, success, error) {
+ url = q1 + encodeURIComponent(url) + q2;
+ return $myResource(url).get({}, success, error);
+ }
+ };
+ }]);
+
})();
\ No newline at end of file
diff --git a/vipra-ui/app/js/helpers.js b/vipra-ui/app/js/helpers.js
index 8a33c7a25a5712cae94c396a2824f492bf29f27a..a79de3785b613bb52e461ab97390e69e609b0fa3 100644
--- a/vipra-ui/app/js/helpers.js
+++ b/vipra-ui/app/js/helpers.js
@@ -14,15 +14,15 @@
*/
Vipra.formatDate = function(date) {
- return new Date(date).toLocaleDateString();
+ return date ? new Date(date).toLocaleDateString() : '';
};
Vipra.formatTime = function(date) {
- return new Date(date).toLocaleTimeString();
+ return date ? new Date(date).toLocaleTimeString() : '';
};
Vipra.formatDateTime = function(date) {
- return new Date(date).toLocaleString();
+ return date ? new Date(date).toLocaleString() : '';
};
Vipra.toPercent = function(input, nums) {
diff --git a/vipra-ui/app/less/app.less b/vipra-ui/app/less/app.less
index 1adb2ddb803705549ab31f4aa2522a3b3096dbd8..a49479dfb558d8fda104e816b4e9f6d1850e4a1f 100644
--- a/vipra-ui/app/less/app.less
+++ b/vipra-ui/app/less/app.less
@@ -144,9 +144,6 @@ a:hover {
td {
overflow: hidden;
}
-a {
- word-break: break-all;
-}
.initial {
float: left;
@@ -393,7 +390,7 @@ a {
bottom: 0;
top: 0;
left: @sidebar-padding;
- right: @sidebar-padding;
+ right: 1px;
overflow-y: auto;
margin-bottom: 0;
> li {
@@ -415,7 +412,7 @@ a {
}
}
&.no-offset {
- top: 5px;
+ top: 0;
}
.popover-area {
position: absolute;
@@ -438,7 +435,6 @@ a {
}
.colorbox {
-
display: inline-block;
width: 5px;
height: 21px;
@@ -1025,6 +1021,57 @@ entity-menu {
margin-top: -2px;
}
+.help-body {
+ div {
+ padding: 10px;
+ width: 100%;
+ img {
+ width: 100%;
+ border: 1px solid #ddd;
+ padding: 5px;
+ border-radius: 5px;
+ }
+ &:not(:hover) {
+ .hover {
+ display:none;
+ }
+ }
+ &:hover {
+ .default {
+ display: none;
+ }
+
+ }
+ }
+
+ .image:after {
+ content: 'Hover image for details';
+ font-size: 12px;
+ }
+}
+
+.help-listing {
+ td + td {
+ padding-left: 5px;
+ vertical-align: bottom
+ }
+
+ td:first-child {
+ text-align: right;
+ }
+}
+
+.input-group.inline {
+ display: inline-table;
+ vertical-align: middle;
+
+ .input-group-addon,
+ .input-group-btn,
+ .form-control {
+ width: auto !important;
+ }
+}
+
[ng\:cloak], [ng-cloak], .ng-cloak {
display: none !important;
}
@@ -1131,7 +1178,7 @@ entity-menu {
}
}
-@media (max-width: 1050px) {
+@media (max-width: 1059px) {
.search-form {
display: none;
}
diff --git a/vipra-util/src/main/java/de/vipra/util/ColorUtils.java b/vipra-util/src/main/java/de/vipra/util/ColorUtils.java
index 534d7783e39b77b0a9eae741cd34151f6dd03f0f..ab4631d1e038eab641f5bbc74f8eb6d3f120a157 100644
--- a/vipra-util/src/main/java/de/vipra/util/ColorUtils.java
+++ b/vipra-util/src/main/java/de/vipra/util/ColorUtils.java
@@ -16,17 +16,17 @@ public class ColorUtils {
/**
* Generate a random color, using an optional seed.
- *
+ *
* @param seed
* optional random seed, null for default seed
* @return generated random color
*/
public static Color generateRandomColor(final Long seed) {
- Random random = seed != null ? new Random(seed) : new Random();
- int red = (random.nextInt(256) + 255) / 2;
- int green = (random.nextInt(256) + 255) / 2;
- int blue = (random.nextInt(256) + 255) / 2;
- Color color = new Color(red, green, blue);
+ final Random random = seed != null ? new Random(seed) : new Random();
+ final int red = (random.nextInt(256) + 255) / 2;
+ final int green = (random.nextInt(256) + 255) / 2;
+ final int blue = (random.nextInt(256) + 255) / 2;
+ final Color color = new Color(red, green, blue);
return color;
}
@@ -39,7 +39,7 @@ public class ColorUtils {
/**
* Generate random colors, using an optional seed.
- *
+ *
* @param number
* how many colors to generate
* @param seed
@@ -47,12 +47,12 @@ public class ColorUtils {
* @return generated random colors
*/
public static List<Color> generateRandomColors(final int number, final Long seed) {
- Random random = seed != null ? new Random(seed) : new Random();
+ final Random random = seed != null ? new Random(seed) : new Random();
final List<Color> colors = new ArrayList<>(number);
for (int i = 0; i < number; i++) {
- int red = (random.nextInt(256) + 255) / 2;
- int green = (random.nextInt(256) + 255) / 2;
- int blue = (random.nextInt(256) + 255) / 2;
+ final int red = (random.nextInt(256) + 255) / 2;
+ final int green = (random.nextInt(256) + 255) / 2;
+ final int blue = (random.nextInt(256) + 255) / 2;
colors.add(new Color(red, green, blue));
}
return colors;
@@ -60,7 +60,7 @@ public class ColorUtils {
/**
* Turns a Color object into a hexadecimal string of format #ABCDEF.
- *
+ *
* @param color
* Color to transform into hex string
* @return hex string
@@ -71,7 +71,7 @@ public class ColorUtils {
/**
* Parses a hexadecimal string into a Color object.
- *
+ *
* @param hex
* hexadecimal string
* @return Color object
diff --git a/vipra-util/src/main/java/de/vipra/util/Config.java b/vipra-util/src/main/java/de/vipra/util/Config.java
index 2fbf5ab715604abb39e3b327196b7ce7cace76e4..ee37dab8697d3a6a7e31c56c1a9cc4fd488b6d30 100644
--- a/vipra-util/src/main/java/de/vipra/util/Config.java
+++ b/vipra-util/src/main/java/de/vipra/util/Config.java
@@ -132,6 +132,10 @@ public class Config {
return modelConfig;
}
+ public void deleteTopicModelConfig(final String name) {
+ topicModelConfigs.remove(name);
+ }
+
private void setTopicModelConfigs(final Map<String, TopicModelConfig> topicModelConfigs) {
this.topicModelConfigs = topicModelConfigs;
}
@@ -197,7 +201,7 @@ public class Config {
/**
* Returns a generic logging directory
- *
+ *
* @return generic logging directory
*/
public static File getGenericLogFile() {
diff --git a/vipra-util/src/main/java/de/vipra/util/ConsoleUtils.java b/vipra-util/src/main/java/de/vipra/util/ConsoleUtils.java
index 42240f46ff279c6eb497c7c69fc7536f4b1077a6..34ce26db7ad32b8a06f4441b56b32d9ca922a5f9 100644
--- a/vipra-util/src/main/java/de/vipra/util/ConsoleUtils.java
+++ b/vipra-util/src/main/java/de/vipra/util/ConsoleUtils.java
@@ -235,6 +235,33 @@ public class ConsoleUtils {
}
}
+ public static boolean readBoolean(String message, final Boolean def) {
+ final BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
+ if (def != null)
+ message += " [" + def + "]";
+ message += ": ";
+ while (true) {
+ System.out.print(message);
+ try {
+ final String line = in.readLine().toLowerCase().trim();
+ if (line.equals("true") || line.equals("yes") || line.equals("1") || line.equals("on") || line.equals("enable")
+ || line.equals("enabled"))
+ return true;
+ if (line.equals("false") || line.equals("no") || line.equals("0") || line.equals("off") || line.equals("disable")
+ || line.equals("disabled"))
+ return false;
+ if (def != null)
+ return def;
+ } catch (final IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ public static boolean readBoolean(final String message) {
+ return readBoolean(message, null);
+ }
+
public static String positionString(final int current, final int max) {
String positionStr = "(" + StringUtils.pad(Integer.toString(current), Integer.toString(max).length(), " ", true) + "/" + max + ")";
if (max > 1) {
diff --git a/vipra-util/src/main/java/de/vipra/util/Constants.java b/vipra-util/src/main/java/de/vipra/util/Constants.java
index d23af9f1c498a1d756607045ab69e88ede9535c0..484fdb4b7d52bbc734ec224ec5fd357fe3d4ef49 100644
--- a/vipra-util/src/main/java/de/vipra/util/Constants.java
+++ b/vipra-util/src/main/java/de/vipra/util/Constants.java
@@ -32,6 +32,7 @@ public class Constants {
public static final String CONFIG_FILE = "config.json";
public static final String BUILDNUMBER_FILE = "buildNumber.properties";
public static final String LOG_FILE = "vipra.log";
+ public static final String HYPERNYMS_FILE = "hypernyms.map";
/*
* DATABASE
@@ -167,11 +168,13 @@ public class Constants {
public static final WindowResolution WINDOW_RESOLUTION = WindowResolution.YEAR;
/**
- * The processor mode defines the processed text output. In text mode, the
- * text is trimmed down, in entity mode, the text is scanned for entities.
- * In mixed mode, the found entities are inserted into the trimmed text.
+ * Processor modes for transforming text as topic model generation
+ * preparation.
*/
- public static final ProcessorMode PROCESSOR_MODE = ProcessorMode.TEXT;
+ public static final boolean PROCESSOR_USE_TEXT = true;
+ public static final boolean PROCESSOR_USE_ENTITIES = true;
+ public static final boolean PROCESSOR_USE_ENTITY_TYPES = true;
+ public static final boolean PROCESSOR_USE_HYPERNYMS = true;
/**
* Stopwords list. Extensive list of stopwords used to clean imported
@@ -263,7 +266,7 @@ public class Constants {
public static final String REGEX_URL = "(?:^|[\\W])((ht|f)tp(s?):\\/\\/|www\\.)(([\\w\\-]+\\.){1,}?([\\w\\-.~]+\\/?)*[\\p{Alnum}.,%_=?&#\\-+()\\[\\]\\*$~@!:/{};']*)";
/**
- * Regular expressiong to find and remove numbers from text.
+ * Regular expression to find and remove numbers from text.
*/
public static final String REGEX_NUMBER = "\\b\\w*\\d+\\w*\\b";
@@ -272,6 +275,11 @@ public class Constants {
*/
public static final String REGEX_SINGLECHAR = "\\b\\w\\b";
+ /**
+ * Regular expression to match forbidden characters in an entity name.
+ */
+ public static final String REGEX_ENTITY_DISALLOWED = "[/\\.]+";
+
/*
* INDEX
*/
@@ -403,10 +411,4 @@ public class Constants {
}
}
- public static enum ProcessorMode {
- TEXT,
- ENTITIES,
- TEXT_WITH_ENTITIES
- }
-
}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/Article.java b/vipra-util/src/main/java/de/vipra/util/model/Article.java
index 70905010f3cc51998c925ceddbec90c87bcc8096..c71e49ea2cae138362ea6e9f321340c4c8a3362b 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/Article.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/Article.java
@@ -1,6 +1,7 @@
package de.vipra.util.model;
import java.io.Serializable;
+import java.util.Date;
import org.bson.types.ObjectId;
import org.mongodb.morphia.annotations.Entity;
@@ -22,6 +23,8 @@ public class Article implements Model<ObjectId>, Serializable {
private String title;
+ private Date date;
+
private Integer topicsCount;
public Article() {}
@@ -48,6 +51,14 @@ public class Article implements Model<ObjectId>, Serializable {
this.title = title;
}
+ public Date getDate() {
+ return date;
+ }
+
+ public void setDate(final Date date) {
+ this.date = date;
+ }
+
public Integer getTopicsCount() {
return topicsCount;
}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java b/vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java
index 45c590db657c72cf32e89a2e6e5ad346fe776cf2..9ef0ba0e6cb279a4b8beae5225eb5abe0f63999b 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java
@@ -277,17 +277,26 @@ public class ArticleFull implements Model<ObjectId>, Serializable {
meta.put(key, value);
}
- public String[] entitiesWithTypes() {
- int size = 0;
- for (final TextEntityCount textEntity : entities) {
- size++;
- if (textEntity.getEntity().getTypes() != null)
- size += textEntity.getEntity().getTypes().size();
- }
- final List<String> entitiesWithTypes = new ArrayList<>(size);
+ public String[] entities() {
+ final List<String> strEntities = new ArrayList<>(entities.size());
+ for (final TextEntityCount textEntity : entities)
+ strEntities.add(textEntity.getEntity().getEntity());
+ return strEntities.toArray(new String[strEntities.size()]);
+ }
+
+ public String[] types() {
+ final List<String> strTypes = new ArrayList<>(entities.size() * 3);
+ for (final TextEntityCount textEntity : entities)
+ strTypes.addAll(textEntity.getEntity().getTypes());
+ return strTypes.toArray(new String[strTypes.size()]);
+ }
+
+ public String[] hypernyms() {
+ final List<String> strHypernyms = new ArrayList<>(entities.size());
for (final TextEntityCount textEntity : entities)
- entitiesWithTypes.addAll(textEntity.getEntity().entityWithTypes());
- return entitiesWithTypes.toArray(new String[size]);
+ if (textEntity.getEntity().getIsHypernym())
+ strHypernyms.add(textEntity.getEntity().getEntity());
+ return strHypernyms.toArray(new String[strHypernyms.size()]);
}
@PrePersist
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TextEntity.java b/vipra-util/src/main/java/de/vipra/util/model/TextEntity.java
index 2299e1859befd1911a98d1e2dafbf1e7e094a0f8..3a0be98c7dc6eadfe467182205d512927b43e731 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/TextEntity.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/TextEntity.java
@@ -16,6 +16,8 @@ public class TextEntity implements Serializable {
private String entity;
+ private Boolean isHypernym;
+
private String url;
private List<String> types;
@@ -35,6 +37,14 @@ public class TextEntity implements Serializable {
this.entity = entity;
}
+ public Boolean getIsHypernym() {
+ return isHypernym;
+ }
+
+ public void setIsHypernym(final Boolean isHypernym) {
+ this.isHypernym = isHypernym;
+ }
+
public String getUrl() {
return url;
}
@@ -55,6 +65,10 @@ public class TextEntity implements Serializable {
return "<a href=\"" + url + "\">" + entity + "</a>";
}
+ public String realEntity() {
+ return TextEntityFull.realEntity(url);
+ }
+
public List<String> entityWithTypes() {
final List<String> entityWithTypes = new ArrayList<>(types.size() + 1);
entityWithTypes.add(entity.toLowerCase());
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TextEntityFull.java b/vipra-util/src/main/java/de/vipra/util/model/TextEntityFull.java
index 15ce5346c9a0c888c6ca035fe013d2a8964e0d23..277d9a6ec1c4b3468e74ffa60814c0f9d3c5309f 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/TextEntityFull.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/TextEntityFull.java
@@ -3,6 +3,8 @@ package de.vipra.util.model;
import java.io.Serializable;
import java.util.Date;
import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.mongodb.morphia.annotations.Entity;
import org.mongodb.morphia.annotations.Id;
@@ -13,6 +15,7 @@ import org.mongodb.morphia.annotations.Reference;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import de.vipra.util.Constants;
import de.vipra.util.an.QueryIgnore;
@JsonIgnoreProperties(ignoreUnknown = true)
@@ -28,6 +31,9 @@ public class TextEntityFull implements Model<String>, Serializable, Comparable<T
private String entity;
+ @QueryIgnore(multi = true)
+ private Boolean isHypernym;
+
@Reference
@QueryIgnore(multi = true)
private TopicModel topicModel;
@@ -46,11 +52,12 @@ public class TextEntityFull implements Model<String>, Serializable, Comparable<T
public TextEntityFull() {}
public TextEntityFull(final TextEntity textEntity, final TopicModel topicModel) {
- entity = textEntity.getEntity();
+ entity = realEntity(textEntity.getUrl());
url = textEntity.getUrl();
types = textEntity.getTypes();
+ isHypernym = textEntity.getIsHypernym();
this.topicModel = topicModel;
- id = entity + "-" + topicModel.getId();
+ id = entity.toLowerCase() + "-" + topicModel.getId();
}
@Override
@@ -71,6 +78,14 @@ public class TextEntityFull implements Model<String>, Serializable, Comparable<T
this.entity = entity;
}
+ public Boolean getIsHypernym() {
+ return isHypernym;
+ }
+
+ public void setIsHypernym(final Boolean isHypernym) {
+ this.isHypernym = isHypernym;
+ }
+
public TopicModel getTopicModel() {
return topicModel;
}
@@ -112,7 +127,23 @@ public class TextEntityFull implements Model<String>, Serializable, Comparable<T
}
public String aTag() {
- return "<a href=\"" + url + "\">" + id + "</a>";
+ return aTag(entity);
+ }
+
+ public String aTag(final String entity) {
+ return "<a href=\"" + url + "\" data-entity=\"" + this.entity + "\">" + entity + "</a>";
+ }
+
+ public String realEntity() {
+ return realEntity(url);
+ }
+
+ public static String realEntity(final String url) {
+ final Pattern p = Pattern.compile("resource/\\.*(.*)");
+ final Matcher m = p.matcher(url);
+ if (!m.find())
+ return null;
+ return m.group(1).replaceAll(Constants.REGEX_ENTITY_DISALLOWED, "_").replaceAll("_+", "_").replaceAll("^_+|_+$", "").trim();
}
@Override
@@ -147,6 +178,8 @@ public class TextEntityFull implements Model<String>, Serializable, Comparable<T
@PrePersist
public void prePersist() {
+ if (isHypernym == null)
+ isHypernym = false;
modified = new Date();
if (created == null)
created = modified;
diff --git a/vipra-util/src/main/java/de/vipra/util/model/Topic.java b/vipra-util/src/main/java/de/vipra/util/model/Topic.java
index 2f6e0b880e8faa63c2275b9d0784e6fbb331c4c1..c511879c0e42bb4e2ebe890b3bca321d7147c1a5 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/Topic.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/Topic.java
@@ -67,7 +67,7 @@ public class Topic implements Model<ObjectId>, Serializable {
return color;
}
- public void setColor(String color) {
+ public void setColor(final String color) {
this.color = color;
}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java b/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java
index 2e61b2743564e0da1dfe499899c9f582a7bf682e..5fbd3cf20aea15808eb2b7596747dce4a1a9a4f7 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java
@@ -180,7 +180,7 @@ public class TopicFull implements Model<ObjectId>, Serializable {
return color;
}
- public void setColor(String color) {
+ public void setColor(final String color) {
this.color = color;
}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TopicModelConfig.java b/vipra-util/src/main/java/de/vipra/util/model/TopicModelConfig.java
index 01ff903a50a10d869da2de15d03fd6be7a000212..4354c0243fadecc6fb16a25e72e70d15e159702e 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/TopicModelConfig.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/TopicModelConfig.java
@@ -14,7 +14,6 @@ import com.fasterxml.jackson.databind.JsonMappingException;
import de.vipra.util.Config;
import de.vipra.util.Constants;
-import de.vipra.util.Constants.ProcessorMode;
import de.vipra.util.Constants.WindowResolution;
@JsonIgnoreProperties(ignoreUnknown = true)
@@ -44,8 +43,11 @@ public class TopicModelConfig implements Serializable {
private double risingDecayLambda = Constants.RISING_DECAY_LAMBDA;
private double maxSimilarDocumentsDivergence = Constants.MAX_SIMILAR_DOCUMENTS_DIVERGENCE;
private double maxSimilarTopicsDivergence = Constants.MAX_SIMILAR_TOPICS_DIVERGENCE;
+ private boolean processorUseText = Constants.PROCESSOR_USE_TEXT;
+ private boolean processorUseEntities = Constants.PROCESSOR_USE_ENTITIES;
+ private boolean processorUseEntityTypes = Constants.PROCESSOR_USE_ENTITY_TYPES;
+ private boolean processorUseHypernyms = Constants.PROCESSOR_USE_HYPERNYMS;
private WindowResolution windowResolution = Constants.WINDOW_RESOLUTION;
- private ProcessorMode processorMode = Constants.PROCESSOR_MODE;
public TopicModelConfig() {}
@@ -67,8 +69,11 @@ public class TopicModelConfig implements Serializable {
risingDecayLambda = topicModelConfig.getRisingDecayLambda();
maxSimilarDocumentsDivergence = topicModelConfig.getMaxSimilarDocumentsDivergence();
maxSimilarTopicsDivergence = topicModelConfig.getMaxSimilarTopicsDivergence();
+ processorUseText = topicModelConfig.isProcessorUseText();
+ processorUseEntities = topicModelConfig.isProcessorUseEntities();
+ processorUseEntityTypes = topicModelConfig.isProcessorUseEntityTypes();
+ processorUseHypernyms = topicModelConfig.isProcessorUseHypernyms();
windowResolution = topicModelConfig.getWindowResolution();
- processorMode = topicModelConfig.getProcessorMode();
}
public String getName() {
@@ -223,20 +228,44 @@ public class TopicModelConfig implements Serializable {
this.maxSimilarTopicsDivergence = maxSimilarTopicsDivergence;
}
- public WindowResolution getWindowResolution() {
- return windowResolution;
+ public boolean isProcessorUseText() {
+ return processorUseText;
}
- public void setWindowResolution(final WindowResolution windowResolution) {
- this.windowResolution = windowResolution;
+ public void setProcessorUseText(final boolean processorUseText) {
+ this.processorUseText = processorUseText;
+ }
+
+ public boolean isProcessorUseEntities() {
+ return processorUseEntities;
+ }
+
+ public void setProcessorUseEntities(final boolean processorUseEntities) {
+ this.processorUseEntities = processorUseEntities;
+ }
+
+ public boolean isProcessorUseEntityTypes() {
+ return processorUseEntityTypes;
}
- public ProcessorMode getProcessorMode() {
- return processorMode;
+ public void setProcessorUseEntityTypes(final boolean processorUseEntityTypes) {
+ this.processorUseEntityTypes = processorUseEntityTypes;
}
- public void setProcessorMode(final ProcessorMode processorMode) {
- this.processorMode = processorMode;
+ public boolean isProcessorUseHypernyms() {
+ return processorUseHypernyms;
+ }
+
+ public void setProcessorUseHypernyms(final boolean processorUseHypernyms) {
+ this.processorUseHypernyms = processorUseHypernyms;
+ }
+
+ public WindowResolution getWindowResolution() {
+ return windowResolution;
+ }
+
+ public void setWindowResolution(final WindowResolution windowResolution) {
+ this.windowResolution = windowResolution;
}
public File getModelDir(final File dataDir) {
@@ -258,8 +287,8 @@ public class TopicModelConfig implements Serializable {
@Override
public String toString() {
- return "[window=" + windowResolution + ", mode=" + processorMode + ", k=" + kTopics + ", iter=" + staticIterations + "/"
- + dynamicMinIterations + "-" + dynamicMaxIterations + "]";
+ return "[window=" + windowResolution + ", k=" + kTopics + ", iter=" + staticIterations + "/" + dynamicMinIterations + "-"
+ + dynamicMaxIterations + "]";
}
public String toPrettyString() {
@@ -270,7 +299,7 @@ public class TopicModelConfig implements Serializable {
+ "\n spotlightConfidence: " + spotlightConfidence + "\n minTopicShare: " + minTopicShare + "\n minRelativeProbability: "
+ minRelativeProbability + "\n risingDecayLambda: " + risingDecayLambda + "\n maxSimilarDocumentsDivergence: "
+ maxSimilarDocumentsDivergence + "\n maxSimilarTopicsDivergence: " + maxSimilarTopicsDivergence + "\n windowResolution: "
- + windowResolution + "\n processorMode: " + processorMode;
+ + windowResolution;
}
}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TopicModelFull.java b/vipra-util/src/main/java/de/vipra/util/model/TopicModelFull.java
index 50c7e4b23494a3d0e0d8cb5274b3a6925cb9cbf3..017d04f3902f1893f93fab0980c215c99bbcd5e7 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/TopicModelFull.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/TopicModelFull.java
@@ -112,7 +112,7 @@ public class TopicModelFull implements Model<ObjectId>, Comparable<TopicModelFul
return entityCount;
}
- public void setEntityCount(Long entityCount) {
+ public void setEntityCount(final Long entityCount) {
this.entityCount = entityCount;
}
@@ -152,7 +152,7 @@ public class TopicModelFull implements Model<ObjectId>, Comparable<TopicModelFul
return colorSeed;
}
- public void setColorSeed(Long colorSeed) {
+ public void setColorSeed(final Long colorSeed) {
this.colorSeed = colorSeed;
}
diff --git a/vipra-util/src/main/java/de/vipra/util/service/QueryBuilder.java b/vipra-util/src/main/java/de/vipra/util/service/QueryBuilder.java
index 3f6d2c0acf40f412b07908bf362411c235122212..3a97acba8dcd38d6d8f0e049ff6717c8f5fd6998 100644
--- a/vipra-util/src/main/java/de/vipra/util/service/QueryBuilder.java
+++ b/vipra-util/src/main/java/de/vipra/util/service/QueryBuilder.java
@@ -168,6 +168,12 @@ public class QueryBuilder {
return this;
}
+ public QueryBuilder allFields() {
+ allFields = true;
+ fields = null;
+ return this;
+ }
+
public QueryBuilder fields(final String... fields) {
return fields(true, fields);
}