diff --git a/Vagrantfile b/Vagrantfile
index 7facb0b9d5267db0e7d7777181f4364ea34195c3..28cab3ce15de4bc866fe79e2141d868d70a8b4fd 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -14,6 +14,7 @@ Vagrant.configure(2) do |config|
master.vm.network :forwarded_port, guest: 27017, host: 27017 # MongoDB
master.vm.network :forwarded_port, guest: 8080, host: 8000 # Tomcat
master.vm.network :forwarded_port, guest: 9200, host: 9200 # ElasticSearch REST API
+ master.vm.network :forwarded_port, guest: 9300, host: 9300 # ElasticSearch Native
end
end
diff --git a/ma-impl.sublime-workspace b/ma-impl.sublime-workspace
index dda352d0d4d451aae5bc7f2a703caa8b78eae3e5..29c324ca13c2734e5d90f982c41364057ab177f5 100644
--- a/ma-impl.sublime-workspace
+++ b/ma-impl.sublime-workspace
@@ -279,6 +279,14 @@
},
"buffers":
[
+ {
+ "file": "Vagrantfile",
+ "settings":
+ {
+ "buffer_size": 572,
+ "line_ending": "Unix"
+ }
+ }
],
"build_system": "",
"build_system_choices":
@@ -458,7 +466,6 @@
"expanded_folders":
[
"/home/eike/repos/master/ma-impl",
- "/home/eike/repos/master/ma-impl/vipra-ui",
"/home/eike/repos/master/ma-impl/vipra-ui/app",
"/home/eike/repos/master/ma-impl/vipra-ui/app/components",
"/home/eike/repos/master/ma-impl/vipra-ui/app/routes",
@@ -924,8 +931,39 @@
"groups":
[
{
+ "selected": 0,
"sheets":
[
+ {
+ "buffer": 0,
+ "file": "Vagrantfile",
+ "semi_transient": true,
+ "settings":
+ {
+ "buffer_size": 572,
+ "regions":
+ {
+ },
+ "selection":
+ [
+ [
+ 437,
+ 437
+ ]
+ ],
+ "settings":
+ {
+ "syntax": "Packages/Text/Plain text.tmLanguage",
+ "tab_size": 2,
+ "translate_tabs_to_spaces": true
+ },
+ "translation.x": 0.0,
+ "translation.y": 0.0,
+ "zoom_level": 1.0
+ },
+ "stack_index": 0,
+ "type": "text"
+ }
]
}
],
diff --git a/vipra-cmd/.classpath b/vipra-cmd/.classpath
index 68ba27436986c55b46eea6656f7d15f85a4b0758..b0ed5811797b0878e890049fe05bedf410f27090 100644
--- a/vipra-cmd/.classpath
+++ b/vipra-cmd/.classpath
@@ -29,7 +29,13 @@
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<attributes>
- <attribute name="owner.project.facets" value="java"/>
+ <attribute name="maven.pomderived" value="true"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry kind="src" output="target/test-classes" path="src/test/java">
+ <attributes>
+ <attribute name="optional" value="true"/>
+ <attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
diff --git a/vipra-cmd/.project b/vipra-cmd/.project
index b44a107f26b7c9daa76b66453cbb06333efaba11..064ef4670973b65f9998ad67181d77fa64f8d4d9 100644
--- a/vipra-cmd/.project
+++ b/vipra-cmd/.project
@@ -21,12 +21,12 @@
</arguments>
</buildCommand>
<buildCommand>
- <name>org.eclipse.m2e.core.maven2Builder</name>
+ <name>net.sourceforge.metrics.builder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
- <name>net.sourceforge.metrics.builder</name>
+ <name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
diff --git a/vipra-cmd/build.xml b/vipra-cmd/build.xml
index 37709cab9530c96539b00325c5f9edd67467fbf6..8ff573bc096bb2d198033af80e66654a854193f4 100644
--- a/vipra-cmd/build.xml
+++ b/vipra-cmd/build.xml
@@ -1,83 +1,84 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<project default="create_run_jar" name="Create Runnable Jar for Project vipra-cmd">
- <!--this file was created by Eclipse Runnable JAR Export Wizard-->
- <!--ANT 1.7 is required -->
- <!--define folder properties-->
- <property name="dir.buildfile" value="."/>
- <property name="dir.workspace" value="/home/eike/workspace"/>
- <property name="dir.jarfile" value="${dir.buildfile}/bin"/>
- <target name="create_run_jar">
- <jar destfile="${dir.jarfile}/vipra-cmd.jar" filesetmanifest="mergewithoutmain">
- <manifest>
- <attribute name="Main-Class" value="de.vipra.cmd.Main"/>
- <attribute name="Class-Path" value="."/>
- </manifest>
- <fileset dir="${dir.buildfile}/target/classes"/>
- <fileset dir="/home/eike/repos/master/ma-impl/vipra-util/target/classes"/>
- <fileset dir="/home/eike/repos/master/ma-impl/jgibblda/target/classes"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-api/2.4.1/log4j-api-2.4.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-core/2.4.1/log4j-core-2.4.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-slf4j-impl/2.4.1/log4j-slf4j-impl-2.4.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/slf4j/slf4j-api/1.7.12/slf4j-api-1.7.12.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/mongodb-driver/3.2.0/mongodb-driver-3.2.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/bson/3.2.0/bson-3.2.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/mongodb-driver-core/3.2.0/mongodb-driver-core-3.2.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/morphia/morphia/1.0.1/morphia-1.0.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/mongo-java-driver/3.0.2/mongo-java-driver-3.0.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/thoughtworks/proxytoys/proxytoys/1.0/proxytoys-1.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/cglib/cglib-nodep/2.2.2/cglib-nodep-2.2.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/morphia/morphia-logging-slf4j/1.0.1/morphia-logging-slf4j-1.0.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/elasticsearch/elasticsearch/2.1.0/elasticsearch-2.1.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-core/5.3.1/lucene-core-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-backward-codecs/5.3.1/lucene-backward-codecs-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-analyzers-common/5.3.1/lucene-analyzers-common-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-queries/5.3.1/lucene-queries-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-memory/5.3.1/lucene-memory-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-highlighter/5.3.1/lucene-highlighter-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-queryparser/5.3.1/lucene-queryparser-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-sandbox/5.3.1/lucene-sandbox-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-suggest/5.3.1/lucene-suggest-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-misc/5.3.1/lucene-misc-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-join/5.3.1/lucene-join-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-grouping/5.3.1/lucene-grouping-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-spatial/5.3.1/lucene-spatial-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-spatial3d/5.3.1/lucene-spatial3d-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/spatial4j/spatial4j/0.4.1/spatial4j-0.4.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/google/guava/guava/18.0/guava-18.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/carrotsearch/hppc/0.7.1/hppc-0.7.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/joda-time/joda-time/2.8.2/joda-time-2.8.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/joda/joda-convert/1.2/joda-convert-1.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/core/jackson-core/2.6.2/jackson-core-2.6.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/dataformat/jackson-dataformat-smile/2.6.2/jackson-dataformat-smile-2.6.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/dataformat/jackson-dataformat-yaml/2.6.2/jackson-dataformat-yaml-2.6.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/yaml/snakeyaml/1.15/snakeyaml-1.15.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.6.2/jackson-dataformat-cbor-2.6.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/io/netty/netty/3.10.5.Final/netty-3.10.5.Final.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/ning/compress-lzf/1.0.2/compress-lzf-1.0.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/tdunning/t-digest/3.0/t-digest-3.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/hdrhistogram/HdrHistogram/2.1.6/HdrHistogram-2.1.6.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/commons-cli/commons-cli/1.3.1/commons-cli-1.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/twitter/jsr166e/1.1.0/jsr166e-1.1.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/core/jackson-databind/2.7.0/jackson-databind-2.7.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/core/jackson-annotations/2.7.0/jackson-annotations-2.7.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/args4j/args4j/2.0.6/args4j-2.0.6.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/googlecode/json-simple/json-simple/1.1.1/json-simple-1.1.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/edu/stanford/nlp/stanford-corenlp/3.5.2/stanford-corenlp-3.5.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/io7m/xom/xom/1.2.10/xom-1.2.10.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/xml-apis/xml-apis/1.3.03/xml-apis-1.3.03.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/xerces/xercesImpl/2.8.0/xercesImpl-2.8.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/xalan/xalan/2.7.0/xalan-2.7.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/de/jollyday/jollyday/0.4.7/jollyday-0.4.7.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/javax/xml/bind/jaxb-api/2.2.7/jaxb-api-2.2.7.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/googlecode/efficient-java-matrix-library/ejml/0.23/ejml-0.23.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/javax/json/javax.json-api/1.0/javax.json-api-1.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/edu/stanford/nlp/stanford-corenlp/3.5.2/stanford-corenlp-3.5.2-models.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-core/5.4.0/lucene-core-5.4.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-analyzers-common/5.4.0/lucene-analyzers-common-5.4.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/uk/org/lidalia/sysout-over-slf4j/1.0.2/sysout-over-slf4j-1.0.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/junit/junit/4.12/junit-4.12.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar"/>
- </jar>
- </target>
+ <!--this file was created by Eclipse Runnable JAR Export Wizard-->
+ <!--ANT 1.7 is required -->
+ <!--define folder properties-->
+ <property name="dir.buildfile" value="." />
+ <property name="dir.jarfile" value="${dir.buildfile}/bin" />
+ <target name="create_run_jar">
+ <jar destfile="${dir.jarfile}/vipra-cmd.jar" filesetmanifest="mergewithoutmain" duplicate="preserve">
+ <manifest>
+ <attribute name="Main-Class" value="de.vipra.cmd.Main" />
+ <attribute name="Class-Path" value="." />
+ </manifest>
+ <fileset dir="${dir.buildfile}/target/classes" />
+ <fileset dir="/home/eike/repos/master/ma-impl/vipra-util/target/classes" />
+ <fileset dir="/home/eike/repos/master/ma-impl/jgibblda/target/classes" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/slf4j/slf4j-api/1.7.14/slf4j-api-1.7.14.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/mongodb-driver/3.2.0/mongodb-driver-3.2.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/bson/3.2.0/bson-3.2.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/mongodb-driver-core/3.2.0/mongodb-driver-core-3.2.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/morphia/morphia/1.0.1/morphia-1.0.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/mongo-java-driver/3.0.2/mongo-java-driver-3.0.2.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/thoughtworks/proxytoys/proxytoys/1.0/proxytoys-1.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/cglib/cglib-nodep/2.2.2/cglib-nodep-2.2.2.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/morphia/morphia-logging-slf4j/1.0.1/morphia-logging-slf4j-1.0.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/core/jackson-databind/2.7.0/jackson-databind-2.7.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/core/jackson-core/2.7.0/jackson-core-2.7.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/core/jackson-annotations/2.7.0/jackson-annotations-2.7.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/args4j/args4j/2.0.6/args4j-2.0.6.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-api/2.4.1/log4j-api-2.4.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-core/2.4.1/log4j-core-2.4.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-slf4j-impl/2.4.1/log4j-slf4j-impl-2.4.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/slf4j/slf4j-api/1.7.12/slf4j-api-1.7.12.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/commons-cli/commons-cli/1.3.1/commons-cli-1.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/googlecode/json-simple/json-simple/1.1.1/json-simple-1.1.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/junit/junit/4.10/junit-4.10.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/hamcrest/hamcrest-core/1.1/hamcrest-core-1.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/elasticsearch/elasticsearch/2.1.0/elasticsearch-2.1.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-core/5.3.1/lucene-core-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-backward-codecs/5.3.1/lucene-backward-codecs-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-analyzers-common/5.3.1/lucene-analyzers-common-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-queries/5.3.1/lucene-queries-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-memory/5.3.1/lucene-memory-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-highlighter/5.3.1/lucene-highlighter-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-queryparser/5.3.1/lucene-queryparser-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-sandbox/5.3.1/lucene-sandbox-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-suggest/5.3.1/lucene-suggest-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-misc/5.3.1/lucene-misc-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-join/5.3.1/lucene-join-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-grouping/5.3.1/lucene-grouping-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-spatial/5.3.1/lucene-spatial-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-spatial3d/5.3.1/lucene-spatial3d-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/spatial4j/spatial4j/0.4.1/spatial4j-0.4.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/google/guava/guava/18.0/guava-18.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/carrotsearch/hppc/0.7.1/hppc-0.7.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/joda-time/joda-time/2.8.2/joda-time-2.8.2.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/joda/joda-convert/1.2/joda-convert-1.2.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/core/jackson-core/2.6.2/jackson-core-2.6.2.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/dataformat/jackson-dataformat-smile/2.6.2/jackson-dataformat-smile-2.6.2.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/dataformat/jackson-dataformat-yaml/2.6.2/jackson-dataformat-yaml-2.6.2.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/yaml/snakeyaml/1.15/snakeyaml-1.15.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.6.2/jackson-dataformat-cbor-2.6.2.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/io/netty/netty/3.10.5.Final/netty-3.10.5.Final.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/ning/compress-lzf/1.0.2/compress-lzf-1.0.2.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/tdunning/t-digest/3.0/t-digest-3.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/hdrhistogram/HdrHistogram/2.1.6/HdrHistogram-2.1.6.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/twitter/jsr166e/1.1.0/jsr166e-1.1.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/edu/stanford/nlp/stanford-corenlp/3.5.2/stanford-corenlp-3.5.2.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/io7m/xom/xom/1.2.10/xom-1.2.10.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/xml-apis/xml-apis/1.3.03/xml-apis-1.3.03.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/xerces/xercesImpl/2.8.0/xercesImpl-2.8.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/xalan/xalan/2.7.0/xalan-2.7.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/de/jollyday/jollyday/0.4.7/jollyday-0.4.7.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/javax/xml/bind/jaxb-api/2.2.7/jaxb-api-2.2.7.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/googlecode/efficient-java-matrix-library/ejml/0.23/ejml-0.23.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/javax/json/javax.json-api/1.0/javax.json-api-1.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/edu/stanford/nlp/stanford-corenlp/3.5.2/stanford-corenlp-3.5.2-models.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-api/2.5/log4j-api-2.5.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-core/2.5/log4j-core-2.5.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-slf4j-impl/2.5/log4j-slf4j-impl-2.5.jar" />
+ </jar>
+ </target>
</project>
diff --git a/vipra-cmd/pom.xml b/vipra-cmd/pom.xml
index 5aed56ed1592d2fc7f23fb6262bfb56bdbc0963e..bcba557b22fec911cc9937c96089601df18e8934 100644
--- a/vipra-cmd/pom.xml
+++ b/vipra-cmd/pom.xml
@@ -14,8 +14,6 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
- <log4jVersion>2.4.1</log4jVersion>
- <luceneVersion>5.4.0</luceneVersion>
</properties>
<dependencies>
@@ -25,11 +23,6 @@
<artifactId>commons-cli</artifactId>
<version>1.3.1</version>
</dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <version>2.4</version>
- </dependency>
<!-- JsonSimple -->
<dependency>
@@ -58,57 +51,33 @@
<classifier>models</classifier>
</dependency>
- <!-- Lucene -->
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-core</artifactId>
- <version>${luceneVersion}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-analyzers-common</artifactId>
- <version>${luceneVersion}</version>
- </dependency>
-
<!-- Logging -->
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
- <version>${log4jVersion}</version>
+ <version>2.5</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
- <version>${log4jVersion}</version>
+ <version>2.5</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
- <version>${log4jVersion}</version>
+ <version>2.5</version>
</dependency>
<!-- MongoDB Database Adapter -->
- <dependency>
- <groupId>org.mongodb</groupId>
- <artifactId>mongodb-driver</artifactId>
- <version>3.2.0</version>
- </dependency>
<dependency>
<groupId>org.mongodb.morphia</groupId>
<artifactId>morphia</artifactId>
- <version>1.0.1</version>
+ <version>1.1.0</version>
</dependency>
<dependency>
<groupId>org.mongodb.morphia</groupId>
<artifactId>morphia-logging-slf4j</artifactId>
- <version>1.0.1</version>
- </dependency>
-
- <!-- Testing -->
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.12</version>
+ <version>1.1.0</version>
</dependency>
<!-- Workspace -->
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/CmdOptions.java b/vipra-cmd/src/main/java/de/vipra/cmd/CmdOptions.java
index a5e6824b0ea2d91575c27352c311eb66684b56c4..d3d6a79f184719d99c13368eb384c8409a338c46 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/CmdOptions.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/CmdOptions.java
@@ -17,9 +17,6 @@ public class CmdOptions extends Options {
public static final String OPT_SHELL = "x";
public static final String OPT_SHELL_LONG = "shell";
- public static final String OPT_DELETE = "d";
- public static final String OPT_DELETE_LONG = "delete";
-
public static final String OPT_CLEAR = "c";
public static final String OPT_CLEAR_LONG = "clear";
@@ -29,18 +26,28 @@ public class CmdOptions extends Options {
public static final String OPT_DEFAULTS = "n";
public static final String OPT_DEFAULTS_LONG = "defaults";
+ public static final String OPT_DEBUG = "d";
+ public static final String OPT_DEBUG_LONG = "debug";
+
+ public static final String OPT_TEST = "t";
+ public static final String OPT_TEST_LONG = "test";
+
+ public static final String OPT_SILENT = "s";
+ public static final String OPT_SILENT_LONG = "silent";
+
public CmdOptions() {
addOption(Option.builder(OPT_HELP).longOpt(OPT_HELP_LONG).desc("print this message").build());
addOption(Option.builder(OPT_SHELL).longOpt(OPT_SHELL_LONG).hasArg(true).argName("name")
.desc("run from a shell script").build());
addOption(Option.builder(OPT_IMPORT).longOpt(OPT_IMPORT_LONG).hasArgs().argName("files/dirs...")
.desc("import articles into the database").build());
- addOption(Option.builder(OPT_DELETE).longOpt(OPT_DELETE_LONG).hasArgs().argName("files/ids...")
- .desc("delete articles from the database").build());
addOption(Option.builder(OPT_CLEAR).longOpt(OPT_CLEAR_LONG).desc("clear database and filebase").build());
addOption(Option.builder(OPT_STATS).longOpt(OPT_STATS_LONG).desc("gather database and filebase information")
.build());
addOption(Option.builder(OPT_DEFAULTS).longOpt(OPT_DEFAULTS_LONG).desc("accept default decisions").build());
+ addOption(Option.builder(OPT_DEBUG).longOpt(OPT_DEBUG_LONG).desc("show debug information").build());
+ addOption(Option.builder(OPT_TEST).longOpt(OPT_TEST_LONG).desc("system tests").build());
+ addOption(Option.builder(OPT_SILENT).longOpt(OPT_SILENT_LONG).desc("mute all output").build());
}
public void printHelp(String cmd) {
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/ExecutionException.java b/vipra-cmd/src/main/java/de/vipra/cmd/ExecutionException.java
deleted file mode 100644
index 6ea2de6b50588b2b370b39e056ebcad91ff4bae0..0000000000000000000000000000000000000000
--- a/vipra-cmd/src/main/java/de/vipra/cmd/ExecutionException.java
+++ /dev/null
@@ -1,38 +0,0 @@
-package de.vipra.cmd;
-
-import java.util.List;
-
-public class ExecutionException extends Exception {
-
- private static final long serialVersionUID = 1L;
-
- private List<Exception> exceptions;
-
- public ExecutionException(String msg) {
- super(msg);
- }
-
- public ExecutionException(Exception e) {
- super(e);
- }
-
- public ExecutionException(List<Exception> e) {
- this.exceptions = e;
- }
-
- @Override
- public String getMessage() {
- if (exceptions == null) {
- return super.getMessage();
- } else if (exceptions.size() == 1) {
- return exceptions.get(0).getMessage();
- } else {
- StringBuilder sb = new StringBuilder("multiple errors:");
- for (Exception e : exceptions) {
- sb.append("\n " + e.getMessage());
- }
- return sb.toString();
- }
- }
-
-}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/Main.java b/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
index c91124e5185d32283fb33e3d2efd955ee654441e..6f2e07944fb2e59ffee54ec9ec0a93f4197c3a25 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
@@ -1,44 +1,51 @@
package de.vipra.cmd;
import static de.vipra.cmd.CmdOptions.OPT_CLEAR;
+import static de.vipra.cmd.CmdOptions.OPT_DEBUG;
import static de.vipra.cmd.CmdOptions.OPT_DEFAULTS;
-import static de.vipra.cmd.CmdOptions.OPT_DELETE;
import static de.vipra.cmd.CmdOptions.OPT_HELP;
import static de.vipra.cmd.CmdOptions.OPT_IMPORT;
import static de.vipra.cmd.CmdOptions.OPT_SHELL;
+import static de.vipra.cmd.CmdOptions.OPT_SILENT;
import static de.vipra.cmd.CmdOptions.OPT_STATS;
+import static de.vipra.cmd.CmdOptions.OPT_TEST;
import java.util.ArrayList;
import java.util.List;
import java.util.ListIterator;
+import java.util.Map.Entry;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.ParseException;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
+import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.mongodb.morphia.logging.MorphiaLoggerFactory;
import org.mongodb.morphia.logging.slf4j.SLF4JLoggerImplFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+
+import com.mongodb.MongoTimeoutException;
import de.vipra.cmd.option.ClearCommand;
import de.vipra.cmd.option.Command;
-import de.vipra.cmd.option.DeleteCommand;
import de.vipra.cmd.option.ImportCommand;
import de.vipra.cmd.option.StatsCommand;
-import de.vipra.util.ConsoleUtils;
-import de.vipra.util.ConsoleUtils.Choice;
-import de.vipra.util.StringUtils;
-import de.vipra.util.Timer;
+import de.vipra.cmd.option.TestCommand;
public class Main {
- public static final Logger log = LoggerFactory.getLogger(Main.class);
- public static final Logger out = LoggerFactory.getLogger("shellout");
+ public static final Logger log = LogManager.getLogger(Main.class);
+ public static final Logger out = LogManager.getLogger("shellout");
static {
+ // set morphia log level
MorphiaLoggerFactory.registerLogger(SLF4JLoggerImplFactory.class);
- // close stderr to mute corenlp messages
+ // set corenlp log level, close stderr to mute corenlp messages
System.err.close();
}
@@ -67,53 +74,55 @@ public class Main {
return;
}
+ // logger configuration
+ LoggerContext loggerContext = (LoggerContext) LogManager.getContext(false);
+ Configuration loggerConfigs = loggerContext.getConfiguration();
+
+ if (cline.hasOption(OPT_DEBUG))
+ loggerConfigs.getLoggerConfig(LogManager.ROOT_LOGGER_NAME).setLevel(Level.DEBUG);
+
+ if (cline.hasOption(OPT_SILENT)) {
+ for (Entry<String, LoggerConfig> loggerConfig : loggerConfigs.getLoggers().entrySet())
+ loggerConfig.getValue().setLevel(Level.OFF);
+ }
+
+ loggerContext.updateLoggers();
+
+ // check if default decisions should be auto chosen
boolean defaults = cline.hasOption(OPT_DEFAULTS);
+ // get commands
List<Command> commands = new ArrayList<>();
- if (cline.hasOption(OPT_CLEAR)) {
+ if (cline.hasOption(OPT_TEST))
+ commands.add(new TestCommand());
+
+ if (cline.hasOption(OPT_CLEAR))
commands.add(new ClearCommand(defaults));
- }
- if (cline.hasOption(OPT_IMPORT)) {
+ if (cline.hasOption(OPT_IMPORT))
commands.add(new ImportCommand(cline.getOptionValues(OPT_IMPORT)));
- }
- if (cline.hasOption(OPT_DELETE)) {
- commands.add(new DeleteCommand(cline.getOptionValues(OPT_DELETE)));
- }
-
- if (cline.hasOption(OPT_STATS)) {
+ if (cline.hasOption(OPT_STATS))
commands.add(new StatsCommand());
- }
+ // run commands
if (commands.size() > 0) {
- Timer t = new Timer();
- t.start();
- commandLoop: for (ListIterator<Command> it = commands.listIterator(); it.hasNext();) {
+ for (ListIterator<Command> it = commands.listIterator(); it.hasNext();) {
Command c = it.next();
try {
c.run();
- } catch (ExecutionException e) {
- out.error(e.getMessage(), e);
- ConsoleUtils.Choice choice;
- boolean acceptDefault = cline.hasOption(OPT_DEFAULTS);
- do {
- choice = ConsoleUtils.prompt(Choice.CONTINUE, acceptDefault, Choice.ABORT, Choice.RETRY);
- switch (choice) {
- case ABORT:
- break commandLoop;
- case RETRY:
- it.previous();
- case CONTINUE:
- continue commandLoop;
-
- }
- } while (choice == null);
+ } catch (MongoTimeoutException e) {
+ out.error("timeout while trying to connect to the database");
+ log.debug(e.getMessage(), e);
+ } catch (NoNodeAvailableException e) {
+ out.error("could not connect to elasticsearch instance");
+ log.debug(e.getMessage(), e);
+ } catch (Exception e) {
+ out.error(e.getCause().getMessage());
+ log.debug(e.getMessage(), e);
}
}
- long dur = t.stop();
- out.info("done in " + StringUtils.timeString(dur));
} else {
options.printHelp(cmd);
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/es/ESClient.java b/vipra-cmd/src/main/java/de/vipra/cmd/es/ESClient.java
new file mode 100644
index 0000000000000000000000000000000000000000..a48a17c9d219cacfe4108dab1cc42f25eab10a1e
--- /dev/null
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/es/ESClient.java
@@ -0,0 +1,19 @@
+package de.vipra.cmd.es;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+
+import de.vipra.util.Config;
+import de.vipra.util.Constants;
+
+public abstract class ESClient {
+
+ public static TransportClient getClient(Config config) throws UnknownHostException {
+ return TransportClient.builder().build().addTransportAddress(
+ new InetSocketTransportAddress(InetAddress.getByName(Constants.ES_HOST), Constants.ES_PORT));
+ }
+
+}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/ex/ClearException.java b/vipra-cmd/src/main/java/de/vipra/cmd/ex/ClearException.java
deleted file mode 100644
index 9c28a8c2f29a2a40c2d3dd4daea33be60022a91b..0000000000000000000000000000000000000000
--- a/vipra-cmd/src/main/java/de/vipra/cmd/ex/ClearException.java
+++ /dev/null
@@ -1,15 +0,0 @@
-package de.vipra.cmd.ex;
-
-public class ClearException extends Exception {
-
- private static final long serialVersionUID = 1L;
-
- public ClearException(String msg) {
- super(msg);
- }
-
- public ClearException(Exception e) {
- super(e);
- }
-
-}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/ex/ImportException.java b/vipra-cmd/src/main/java/de/vipra/cmd/ex/ImportException.java
deleted file mode 100644
index d7f6da337f6dd0752ba4b094862033e23db22647..0000000000000000000000000000000000000000
--- a/vipra-cmd/src/main/java/de/vipra/cmd/ex/ImportException.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package de.vipra.cmd.ex;
-
-public class ImportException extends Exception {
-
- private static final long serialVersionUID = 1L;
-
- private final String id;
-
- public ImportException(String msg, String id) {
- super(msg);
- this.id = id;
- }
-
- public ImportException(Exception e, String id) {
- super(e);
- this.id = id;
- }
-
- public String getId() {
- return id;
- }
-
-}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbLDAAnalyzer.java b/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbLDAAnalyzer.java
index 217f210eeac40ee2f5b216c2de523b5ff8850e98..5fe15001f12bb639817c29514329d6914ace1bd9 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbLDAAnalyzer.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbLDAAnalyzer.java
@@ -9,8 +9,8 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import de.vipra.cmd.ex.LDAAnalyzerException;
import de.vipra.util.Config;
@@ -31,7 +31,7 @@ import jgibblda.Model;
public class JGibbLDAAnalyzer extends LDAAnalyzer {
- public static final Logger log = LoggerFactory.getLogger(JGibbLDAAnalyzer.class);
+ public static final Logger log = LogManager.getLogger(JGibbLDAAnalyzer.class);
public static final String NAME = "jgibb";
private File dataDir;
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/model/ProcessedArticle.java b/vipra-cmd/src/main/java/de/vipra/cmd/model/ProcessedArticle.java
index 484edc6b37fbc4fe7c3ad25f3290e713a42daba5..e11dd9164bc233ae954f2a994afdbf94cdc34932 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/model/ProcessedArticle.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/model/ProcessedArticle.java
@@ -1,10 +1,13 @@
package de.vipra.cmd.model;
+import java.util.Date;
+
import org.json.simple.JSONObject;
import org.mongodb.morphia.annotations.Entity;
import org.mongodb.morphia.annotations.Transient;
import de.vipra.cmd.text.ProcessedText;
+import de.vipra.util.an.ElasticIndex;
@SuppressWarnings("serial")
@Entity(value = "articles", noClassnameStored = true)
@@ -21,6 +24,21 @@ public class ProcessedArticle extends de.vipra.util.model.ArticleFull {
this.processedText = processedText;
}
+ @ElasticIndex("title")
+ public String serializeTitle() {
+ return super.getTitle();
+ }
+
+ @ElasticIndex("date")
+ public Date serializeDate() {
+ return super.getDate();
+ }
+
+ @ElasticIndex("text")
+ public String serializeText() {
+ return processedText.getText();
+ }
+
public void fromJSON(JSONObject obj) {
if (obj.containsKey("title"))
setTitle(obj.get("title").toString());
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java
index a71a50a6e9dadcf87ea80c46d48712c40c7828c4..5a9e7b54c5681b614042af0c8fe5c5a195c4a24c 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java
@@ -4,16 +4,13 @@ import java.io.File;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.bson.types.ObjectId;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import de.vipra.cmd.ExecutionException;
-import de.vipra.cmd.ex.ClearException;
import de.vipra.cmd.model.ProcessedArticle;
import de.vipra.util.Config;
import de.vipra.util.ConsoleUtils;
-import de.vipra.util.ex.ConfigException;
import de.vipra.util.model.Import;
import de.vipra.util.model.TopicFull;
import de.vipra.util.model.Word;
@@ -21,8 +18,8 @@ import de.vipra.util.service.DatabaseService;
public class ClearCommand implements Command {
- public static final Logger log = LoggerFactory.getLogger(ClearCommand.class);
- public static final Logger out = LoggerFactory.getLogger("shellout");
+ public static final Logger log = LogManager.getLogger(ClearCommand.class);
+ public static final Logger out = LogManager.getLogger("shellout");
private boolean defaults;
private Config config;
@@ -35,16 +32,12 @@ public class ClearCommand implements Command {
this.defaults = defaults;
}
- private void clear() throws ClearException, ConfigException {
- try {
- config = Config.getConfig();
- dbArticles = DatabaseService.getDatabaseService(config, ProcessedArticle.class);
- dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class);
- dbWords = DatabaseService.getDatabaseService(config, Word.class);
- dbImports = DatabaseService.getDatabaseService(config, Import.class);
- } catch (Exception e) {
- throw new ClearException(e);
- }
+ private void clear() throws Exception {
+ config = Config.getConfig();
+ dbArticles = DatabaseService.getDatabaseService(config, ProcessedArticle.class);
+ dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class);
+ dbWords = DatabaseService.getDatabaseService(config, Word.class);
+ dbImports = DatabaseService.getDatabaseService(config, Import.class);
out.info("clearing database");
dbArticles.drop();
@@ -52,27 +45,23 @@ public class ClearCommand implements Command {
dbWords.drop();
dbImports.drop();
- out.info("clearing filebase");
- File dataDir = config.getDataDirectory();
- if (dataDir.exists() && dataDir.isDirectory()) {
- try {
+ try {
+ out.info("clearing filebase");
+ File dataDir = config.getDataDirectory();
+ if (dataDir.exists() && dataDir.isDirectory()) {
FileUtils.deleteDirectory(dataDir);
- } catch (IOException e) {
- out.warn("could not delete data directory: " + dataDir.getAbsolutePath());
}
+ } catch (IOException e) {
+ out.warn("could not delete data directory: " + config.getDataDirectory().getAbsolutePath());
}
}
@Override
- public void run() throws ExecutionException {
+ public void run() throws Exception {
if (!defaults)
out.info("to confirm clearing, type 'clear' and press enter");
- try {
- if (defaults || ConsoleUtils.confirm("clear")) {
- clear();
- }
- } catch (ClearException | ConfigException e) {
- throw new ExecutionException(e);
+ if (defaults || ConsoleUtils.confirm("clear")) {
+ clear();
}
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/Command.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/Command.java
index 5ad51101ead03afa8e62f16e828567a81b6ca314..d794b7590ef7f04e0829de76c0fd25cb7617e442 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/Command.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/Command.java
@@ -1,9 +1,7 @@
package de.vipra.cmd.option;
-import de.vipra.cmd.ExecutionException;
-
public interface Command {
- public void run() throws ExecutionException;
+ public void run() throws Exception;
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteCommand.java
deleted file mode 100644
index ff033a4908cbfbccf467669ee979a523a4414fb5..0000000000000000000000000000000000000000
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteCommand.java
+++ /dev/null
@@ -1,101 +0,0 @@
-package de.vipra.cmd.option;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.bson.types.ObjectId;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import de.vipra.cmd.ExecutionException;
-import de.vipra.cmd.ex.FilebaseException;
-import de.vipra.cmd.file.Filebase;
-import de.vipra.cmd.model.ProcessedArticle;
-import de.vipra.util.Config;
-import de.vipra.util.MongoUtils;
-import de.vipra.util.ex.ConfigException;
-import de.vipra.util.ex.DatabaseException;
-import de.vipra.util.service.DatabaseService;
-
-public class DeleteCommand implements Command {
-
- public static final Logger log = LoggerFactory.getLogger(DeleteCommand.class);
- public static final Logger out = LoggerFactory.getLogger("shellout");
-
- private ArrayList<String> ids = new ArrayList<>();
- private Config config;
- private DatabaseService<ProcessedArticle, ObjectId> dbArticles;
- private Filebase filebase;
-
- /**
- * Creates a new delete command. Use {@link DeleteCommand#run()} to run this
- * command.
- *
- * @param ids
- * array of ids to be used for deletion.
- */
- public DeleteCommand(String[] ids) {
- addIds(ids);
- }
-
- public void addIds(String[] strings) {
- for (String str : strings) {
- if (str.contains(File.pathSeparator)) {
- String[] parts = str.split(File.pathSeparator);
- str = parts[parts.length - 1];
- }
- ids.add(str);
- }
- }
-
- public void deleteEntry(String id) throws ExecutionException {
- ArrayList<Exception> errors = new ArrayList<>();
-
- try {
- // 1. delete mongodb entry
- dbArticles.deleteSingle(MongoUtils.objectId(id));
- } catch (DatabaseException e) {
- errors.add(e);
- }
-
- try {
- // 2. delete file
- filebase.remove(id);
- } catch (FilebaseException e) {
- errors.add(e);
- }
-
- // 3. delete elasticsearch index entry
- // TODO implement
-
- if (errors.size() > 0) {
- throw new ExecutionException(errors);
- }
- }
-
- @Override
- public void run() throws ExecutionException {
- try {
- config = Config.getConfig();
- dbArticles = DatabaseService.getDatabaseService(config, ProcessedArticle.class);
- filebase = Filebase.getFilebase(config);
- } catch (IOException | FilebaseException | ConfigException e) {
- throw new ExecutionException(e);
- }
-
- List<Exception> ex = new ArrayList<>();
- for (String id : ids) {
- try {
- deleteEntry(id);
- } catch (Exception e) {
- ex.add(e);
- }
- }
- if (ex.size() > 0) {
- throw new ExecutionException(ex);
- }
- }
-
-}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
index 0d88b358dbcefba2c6b8243f2e77180681e63035..382cc4a011fa1a7bd3771cbfa076f0d238c5d10a 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
@@ -1,26 +1,23 @@
package de.vipra.cmd.option;
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FilenameFilter;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.bson.types.ObjectId;
+import org.elasticsearch.client.Client;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
-import org.json.simple.parser.ParseException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import de.vipra.cmd.ExecutionException;
-import de.vipra.cmd.ex.ImportException;
+import de.vipra.cmd.es.ESClient;
import de.vipra.cmd.file.Filebase;
import de.vipra.cmd.file.FilebaseIndex;
import de.vipra.cmd.lda.LDAAnalyzer;
@@ -29,6 +26,7 @@ import de.vipra.cmd.text.ProcessedText;
import de.vipra.cmd.text.Processor;
import de.vipra.util.Config;
import de.vipra.util.ConvertStream;
+import de.vipra.util.ElasticSerializer;
import de.vipra.util.MongoUtils;
import de.vipra.util.StringUtils;
import de.vipra.util.Timer;
@@ -45,8 +43,8 @@ import de.vipra.util.service.DatabaseService;
public class ImportCommand implements Command {
- public static final Logger log = LoggerFactory.getLogger(ImportCommand.class);
- public static final Logger out = LoggerFactory.getLogger("shellout");
+ public static final Logger log = LogManager.getLogger(ImportCommand.class);
+ public static final Logger out = LogManager.getLogger("shellout");
private ArrayList<File> files = new ArrayList<>();
private JSONParser parser = new JSONParser();
@@ -59,6 +57,8 @@ public class ImportCommand implements Command {
private Processor preprocessor;
private WordMap wordMap;
private LDAAnalyzer analyzer;
+ private Client elasticClient;
+ private ElasticSerializer<ProcessedArticle> elasticSerializer;
/**
* Import command to import articles into the database, do topic modeling
@@ -105,49 +105,44 @@ public class ImportCommand implements Command {
*
* @param obj
* @return
- * @throws ImportException
+ * @throws Exception
*/
- private Article importArticle(JSONObject obj) throws ImportException {
+ private Article importArticle(JSONObject obj) throws Exception {
out.info("importing \"" + StringUtils.ellipsize(obj.get("title").toString(), 80) + "\"");
ProcessedArticle article = new ProcessedArticle();
article.fromJSON(obj);
- try {
- // preprocess text and generate text statistics
- ProcessedText processedText = preprocessor.preprocess(article.getText());
- ArticleStats articleStats = ArticleStats.generateFromText(processedText.getText(), wordMap);
-
- // add article to mongodb
- article.setProcessedText(processedText);
- article.setStats(articleStats);
- article = dbArticles.createSingle(article);
-
- // add words
- for(String word : processedText.getWords())
- wordMap.add(word);
-
- // add article to filebase
- filebase.add(article);
-
- // return article reference
- return new Article(article.getId());
- } catch (Exception e) {
- throw new ImportException(e, article.getId().toString());
- }
+ // preprocess text and generate text statistics
+ ProcessedText processedText = preprocessor.preprocess(article.getText());
+ ArticleStats articleStats = ArticleStats.generateFromText(processedText.getText(), wordMap);
+
+ // add article to mongodb
+ article.setProcessedText(processedText);
+ article.setStats(articleStats);
+ article = dbArticles.createSingle(article);
+
+ // add words
+ for (String word : processedText.getWords())
+ wordMap.add(word);
+
+ // add article to filebase
+ filebase.add(article);
+
+ // index article
+ Map<String, Object> source = elasticSerializer.serialize(article);
+ elasticClient.prepareIndex("articles", "article", article.getId().toString()).setSource(source).get();
+
+ // return article reference
+ return new Article(article.getId());
}
/**
* Imports a file into the database and the filebase
*
* @param file
- * @throws ParseException
- * @throws IOException
- * @throws FileNotFoundException
- * @throws ImportException
* @throws Exception
*/
- private List<Article> importFile(File file)
- throws FileNotFoundException, IOException, ParseException, ImportException {
+ private List<Article> importFile(File file) throws Exception {
Object data = parser.parse(new FileReader(file));
List<Article> articles = new ArrayList<>();
@@ -162,8 +157,7 @@ public class ImportCommand implements Command {
return articles;
}
- private List<Article> importFiles(List<File> files)
- throws FileNotFoundException, IOException, ParseException, ImportException {
+ private List<Article> importFiles(List<File> files) throws Exception {
List<Article> articles = new ArrayList<>();
for (File file : files) {
articles.addAll(importFile(file));
@@ -172,129 +166,129 @@ public class ImportCommand implements Command {
}
@Override
- public void run() throws ExecutionException {
- try {
- config = Config.getConfig();
- dbArticles = DatabaseService.getDatabaseService(config, ProcessedArticle.class);
- dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class);
- dbWords = DatabaseService.getDatabaseService(config, Word.class);
- dbImports = DatabaseService.getDatabaseService(config, Import.class);
- filebase = Filebase.getFilebase(config);
- preprocessor = Processor.getPreprocessor(config);
- wordMap = new WordMap(dbWords);
- analyzer = LDAAnalyzer.getAnalyzer(config, wordMap);
-
- out.info("using data directory: " + config.getDataDirectory().getAbsolutePath());
- out.info("using preprocessor: " + preprocessor.getName());
- out.info("using analyzer: " + analyzer.getName());
-
- Timer timer = new Timer();
- timer.start();
-
- Import importOp = new Import();
-
- /*
- * import files into database and filebase
- */
- out.info("file import");
- List<Article> importedArticles = importFiles(files);
- importOp.setArticles(importedArticles);
- timer.lap("import");
-
- /*
- * write filebase
- */
- out.info("writing file index");
- filebase.close();
- timer.lap("filebase write");
-
- /*
- * do topic modeling
- */
- out.info("topic modeling");
- analyzer.analyze();
- timer.lap("topic modeling");
-
- /*
- * save topic model
- */
- out.info("saving topic definitions");
- int batchSize = 100;
- ConvertStream<TopicFull> topicDefs = analyzer.getTopicDefinitions();
- Map<String, String> topicIndexMap = new HashMap<>();
- dbTopics.drop();
- List<TopicFull> newTopicDefs = new ArrayList<>(batchSize);
- List<Topic> newTopicRefs = new ArrayList<>();
- Iterator<TopicFull> it = topicDefs.iterator();
- while (it.hasNext()) {
- newTopicDefs.add(it.next());
- if (newTopicDefs.size() == batchSize || !it.hasNext()) {
- dbTopics.createMultiple(newTopicDefs);
- for (TopicFull newTopicDef : newTopicDefs) {
- topicIndexMap.put(Integer.toString(newTopicDef.getIndex()), newTopicDef.getId().toString());
- newTopicRefs.add(new Topic(newTopicDef.getId()));
- }
+ public void run() throws Exception {
+ config = Config.getConfig();
+ dbArticles = DatabaseService.getDatabaseService(config, ProcessedArticle.class);
+ dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class);
+ dbWords = DatabaseService.getDatabaseService(config, Word.class);
+ dbImports = DatabaseService.getDatabaseService(config, Import.class);
+ filebase = Filebase.getFilebase(config);
+ preprocessor = Processor.getPreprocessor(config);
+ wordMap = new WordMap(dbWords);
+ analyzer = LDAAnalyzer.getAnalyzer(config, wordMap);
+ elasticClient = ESClient.getClient(config);
+ elasticSerializer = new ElasticSerializer<>(ProcessedArticle.class);
+
+ out.info("using data directory: " + config.getDataDirectory().getAbsolutePath());
+ out.info("using preprocessor: " + preprocessor.getName());
+ out.info("using analyzer: " + analyzer.getName());
+
+ Timer timer = new Timer();
+ timer.start();
+
+ Import importOp = new Import();
+
+ /*
+ * import files into database and filebase
+ */
+ out.info("file import");
+ List<Article> importedArticles = importFiles(files);
+ importOp.setArticles(importedArticles);
+ timer.lap("import");
+
+ /*
+ * write filebase
+ */
+ out.info("writing file index");
+ filebase.close();
+ timer.lap("filebase write");
+
+ /*
+ * do topic modeling
+ */
+ out.info("topic modeling");
+ analyzer.analyze();
+ timer.lap("topic modeling");
+
+ /*
+ * save topic model
+ */
+ out.info("saving topic definitions");
+ int batchSize = 100;
+ ConvertStream<TopicFull> topicDefs = analyzer.getTopicDefinitions();
+ Map<String, String> topicIndexMap = new HashMap<>();
+ dbTopics.drop();
+ List<TopicFull> newTopicDefs = new ArrayList<>(batchSize);
+ List<Topic> newTopicRefs = new ArrayList<>();
+ Iterator<TopicFull> it = topicDefs.iterator();
+ while (it.hasNext()) {
+ newTopicDefs.add(it.next());
+ if (newTopicDefs.size() == batchSize || !it.hasNext()) {
+ dbTopics.createMultiple(newTopicDefs);
+ for (TopicFull newTopicDef : newTopicDefs) {
+ topicIndexMap.put(Integer.toString(newTopicDef.getIndex()), newTopicDef.getId().toString());
+ newTopicRefs.add(new Topic(newTopicDef.getId()));
}
}
- importOp.setTopics(newTopicRefs);
- timer.lap("saving topics");
-
- /*
- * save topic refs
- */
- out.info("saving document topics");
- ConvertStream<List<TopicRef>> topics = analyzer.getTopics();
- FilebaseIndex index = filebase.getIndex();
- Iterator<String> indexIter = index.iterator();
- Iterator<List<TopicRef>> topicIter = topics.iterator();
- while (indexIter.hasNext() && topicIter.hasNext()) {
- List<TopicRef> topicCount = topicIter.next();
- for (TopicRef tc : topicCount) {
- String oid = topicIndexMap.get(tc.getTopicId());
- tc.setTopicId(oid);
- if (oid == null)
- log.error("no object id for topic index " + tc.getTopicId());
- }
- String id = indexIter.next();
- ProcessedArticle a = dbArticles.getSingle(MongoUtils.objectId(id));
- if (a != null)
- a.setTopics(topicCount);
- else
- log.error("no article found in db for id " + id);
- try {
- dbArticles.updateSingle(a);
- } catch (DatabaseException e) {
- log.error("could not update article: " + a.getTitle() + " (" + a.getId() + ")");
- }
+ }
+ importOp.setTopics(newTopicRefs);
+ timer.lap("saving topics");
+
+ /*
+ * save topic refs
+ */
+ out.info("saving document topics");
+ ConvertStream<List<TopicRef>> topics = analyzer.getTopics();
+ FilebaseIndex index = filebase.getIndex();
+ Iterator<String> indexIter = index.iterator();
+ Iterator<List<TopicRef>> topicIter = topics.iterator();
+ while (indexIter.hasNext() && topicIter.hasNext()) {
+ List<TopicRef> topicCount = topicIter.next();
+ for (TopicRef tc : topicCount) {
+ String oid = topicIndexMap.get(tc.getTopicId());
+ tc.setTopicId(oid);
+ if (oid == null)
+ log.error("no object id for topic index " + tc.getTopicId());
+ }
+ String id = indexIter.next();
+ ProcessedArticle a = dbArticles.getSingle(MongoUtils.objectId(id));
+ if (a != null)
+ a.setTopics(topicCount);
+ else
+ log.error("no article found in db for id " + id);
+ try {
+ dbArticles.updateSingle(a);
+ } catch (DatabaseException e) {
+ log.error("could not update article: " + a.getTitle() + " (" + a.getId() + ")");
}
- List<Word> importedWords = wordMap.getNewWords();
- importOp.setWords(importedWords);
- timer.lap("saving topic refs");
-
- /*
- * save words
- */
- out.info("saving words");
- wordMap.create();
- timer.lap("saving words");
-
- /*
- * save import information
- */
- importOp.setDuration(timer.total());
- dbImports.createSingle(importOp);
-
- /*
- * run information
- */
- int newArticlesCount = importedArticles.size();
- int newWordsCount = importedWords.size();
- out.info("imported " + newArticlesCount + " new " + StringUtils.quantity(newArticlesCount, "article"));
- out.info("imported " + newWordsCount + " new " + StringUtils.quantity(newWordsCount, "word"));
- out.info(timer.toString());
- } catch (Exception e) {
- throw new ExecutionException(e);
}
+ List<Word> importedWords = wordMap.getNewWords();
+ importOp.setWords(importedWords);
+ timer.lap("saving topic refs");
+
+ /*
+ * save words
+ */
+ out.info("saving words");
+ wordMap.create();
+ timer.lap("saving words");
+
+ /*
+ * save import information
+ */
+ importOp.setDuration(timer.total());
+ dbImports.createSingle(importOp);
+
+ elasticClient.close();
+
+ /*
+ * run information
+ */
+ int newArticlesCount = importedArticles.size();
+ int newWordsCount = importedWords.size();
+ out.info("imported " + newArticlesCount + " new " + StringUtils.quantity(newArticlesCount, "article"));
+ out.info("imported " + newWordsCount + " new " + StringUtils.quantity(newWordsCount, "word"));
+ out.info(timer.toString());
}
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/StatsCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/StatsCommand.java
index d8ed55cf854c5b4115c56c9a4d2c54ba5fb2d839..b6e59868a9aa29ba534eb8ca28be23915ec6d275 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/StatsCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/StatsCommand.java
@@ -1,25 +1,21 @@
package de.vipra.cmd.option;
import java.io.File;
-import java.io.IOException;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.bson.types.ObjectId;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import de.vipra.cmd.ExecutionException;
-import de.vipra.cmd.ex.FilebaseException;
import de.vipra.cmd.file.Filebase;
import de.vipra.util.Config;
import de.vipra.util.StringUtils;
-import de.vipra.util.ex.ConfigException;
import de.vipra.util.model.TopicFull;
import de.vipra.util.service.DatabaseService;
public class StatsCommand implements Command {
- public static final Logger log = LoggerFactory.getLogger(StatsCommand.class);
- public static final Logger out = LoggerFactory.getLogger("shellout");
+ public static final Logger log = LogManager.getLogger(StatsCommand.class);
+ public static final Logger out = LogManager.getLogger("shellout");
private Config config;
private Filebase filebase;
@@ -34,16 +30,12 @@ public class StatsCommand implements Command {
}
@Override
- public void run() throws ExecutionException {
- try {
- config = Config.getConfig();
- filebase = Filebase.getFilebase(config);
- dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class);
-
- stats();
- } catch (IOException | ConfigException | FilebaseException e) {
- throw new ExecutionException(e);
- }
+ public void run() throws Exception {
+ config = Config.getConfig();
+ filebase = Filebase.getFilebase(config);
+ dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class);
+
+ stats();
}
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/TestCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/TestCommand.java
new file mode 100644
index 0000000000000000000000000000000000000000..9441138e7777a02c4bb648180c10c570a0ce7ac9
--- /dev/null
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/TestCommand.java
@@ -0,0 +1,39 @@
+package de.vipra.cmd.option;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.bson.types.ObjectId;
+import org.elasticsearch.client.transport.NoNodeAvailableException;
+import org.elasticsearch.client.transport.TransportClient;
+
+import de.vipra.cmd.es.ESClient;
+import de.vipra.util.Config;
+import de.vipra.util.model.Article;
+import de.vipra.util.service.DatabaseService;
+
+public class TestCommand implements Command {
+
+ public static final Logger out = LogManager.getLogger("shellout");
+
+ @Override
+ public void run() throws Exception {
+ // test if configuration readable
+ out.info("reading configuration...");
+ Config config = Config.getConfig();
+
+ // test if database is accessible
+ out.info("testing mongodb connection...");
+ DatabaseService<Article, ObjectId> dbArticles = DatabaseService.getDatabaseService(config, Article.class);
+ dbArticles.count();
+
+ // test if elasticsearch is accessible
+ out.info("testing elasticsearch connection...");
+ TransportClient esclient = ESClient.getClient(config);
+ if (esclient.connectedNodes().isEmpty()) {
+ throw new NoNodeAvailableException("no elasticsearch nodes available");
+ }
+
+ out.info("all tests passed");
+ }
+
+}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/text/CustomProcessor.java b/vipra-cmd/src/main/java/de/vipra/cmd/text/CustomProcessor.java
deleted file mode 100644
index f2a135c97fbdf1eb4fd3599b84457cb73a738323..0000000000000000000000000000000000000000
--- a/vipra-cmd/src/main/java/de/vipra/cmd/text/CustomProcessor.java
+++ /dev/null
@@ -1,36 +0,0 @@
-package de.vipra.cmd.text;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-public class CustomProcessor extends Processor {
-
- private final Set<String> stopWords;
-
- public CustomProcessor(List<String> stopWordsList) {
- super("Custom Processor");
- this.stopWords = new HashSet<>(stopWordsList);
- }
-
- private String removeStopWords(String text) {
- String[] words = text.split("\\s+");
- StringBuilder sb = new StringBuilder();
- for (String word : words) {
- if (stopWords.contains(word)) {
- continue;
- }
- sb.append(word).append(" ");
- }
- return sb.toString().trim();
- }
-
- @Override
- public ProcessedText preprocess(String input) {
- input = input.toLowerCase();
- input = removeStopWords(input);
- input = clean(input);
- return new ProcessedText(input);
- }
-
-}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/text/LuceneProcessor.java b/vipra-cmd/src/main/java/de/vipra/cmd/text/LuceneProcessor.java
deleted file mode 100644
index b8c9fbaee1cc169e4a940668bbe081fddee8a33e..0000000000000000000000000000000000000000
--- a/vipra-cmd/src/main/java/de/vipra/cmd/text/LuceneProcessor.java
+++ /dev/null
@@ -1,59 +0,0 @@
-package de.vipra.cmd.text;
-
-import java.io.IOException;
-import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.regex.Pattern;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.en.PorterStemFilter;
-import org.apache.lucene.analysis.miscellaneous.TrimFilter;
-import org.apache.lucene.analysis.pattern.PatternReplaceFilter;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.util.CharArraySet;
-
-import de.vipra.cmd.ex.PreprocessorException;
-import de.vipra.util.Constants;
-import de.vipra.util.StringUtils;
-
-public class LuceneProcessor extends Processor {
-
- private final CharArraySet stopWords;
-
- public LuceneProcessor(List<String> stopWords) {
- super("Lucene Processor");
- this.stopWords = new CharArraySet(stopWords, false);
- }
-
- @Override
- public ProcessedText preprocess(String input) throws PreprocessorException {
- Analyzer analyzer = new StandardAnalyzer(stopWords);
- TokenStream stream = analyzer.tokenStream(null, new StringReader(input));
- try {
- stream.reset();
- stream = new PorterStemFilter(stream);
- stream = new TrimFilter(stream);
- stream = new PatternReplaceFilter(stream, Pattern.compile(Constants.REGEX_EMAIL), "", true);
- stream = new PatternReplaceFilter(stream, Pattern.compile(Constants.REGEX_URL), "", true);
- stream = new PatternReplaceFilter(stream, Pattern.compile(Constants.REGEX_NUMBER), "", true);
- stream = new PatternReplaceFilter(stream, Pattern.compile(Constants.CHARS_DISALLOWED), "", true);
- stream = new PatternReplaceFilter(stream, Pattern.compile("\\s+"), " ", true);
- ArrayList<String> result = new ArrayList<>();
- while (stream.incrementToken()) {
- result.add(stream.getAttribute(CharTermAttribute.class).toString());
- }
- return new ProcessedText(StringUtils.join(result));
- } catch (IOException e) {
- throw new PreprocessorException(e);
- } finally {
- try {
- stream.close();
- } catch (IOException e) {}
- analyzer.close();
- }
- }
-
-}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/text/Processor.java b/vipra-cmd/src/main/java/de/vipra/cmd/text/Processor.java
index 0d11b2af676800a206f8eadead1f979cabcd828f..a74c32e199974b7825d5b64f7db75cec37ebb095 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/text/Processor.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/text/Processor.java
@@ -25,12 +25,8 @@ public abstract class Processor {
List<String> stopWords = Constants.STOPWORDS;
switch (Constants.Processor.fromString(config.getString(Key.PREPROCESSOR))) {
- case CUSTOM:
- return new CustomProcessor(stopWords);
case CORENLP:
return new CoreNLPProcessor(stopWords);
- case LUCENE:
- return new LuceneProcessor(stopWords);
default:
return null;
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/text/StopwordsAnnotator.java b/vipra-cmd/src/main/java/de/vipra/cmd/text/StopwordsAnnotator.java
index 838bbf033f0dcd8a134947d094c7a51f7cb64d38..d2701feee64c504c0f2d98a441e0582dd36f6de5 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/text/StopwordsAnnotator.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/text/StopwordsAnnotator.java
@@ -48,4 +48,4 @@ public class StopwordsAnnotator implements Annotator, CoreAnnotation<Boolean> {
return Boolean.class;
}
-}
+}
\ No newline at end of file
diff --git a/vipra-cmd/src/main/resources/log4j2dev.xml b/vipra-cmd/src/main/resources/log4j2dev.xml
index bb9050cff97297092d30efe39d18c9b7452c7a57..58ac9659b3dac15a9121ecd0e3fbf68eef61167b 100644
--- a/vipra-cmd/src/main/resources/log4j2dev.xml
+++ b/vipra-cmd/src/main/resources/log4j2dev.xml
@@ -11,5 +11,6 @@
</Root>
<Logger name="shellout" level="ALL"/>
<Logger name="org.mongodb" level="ERROR"/>
+ <Logger name="org.elasticsearch.transport.netty" level="ERROR"/>
</Loggers>
</Configuration>
\ No newline at end of file
diff --git a/vipra-rest/.classpath b/vipra-rest/.classpath
index f2db8ed9892d5e6702d39d0cba12cc8436a2b42e..b12640c54337a1f386277f0c3766a16d90ed9b23 100644
--- a/vipra-rest/.classpath
+++ b/vipra-rest/.classpath
@@ -19,7 +19,13 @@
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<attributes>
- <attribute name="owner.project.facets" value="java"/>
+ <attribute name="maven.pomderived" value="true"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry kind="src" output="target/test-classes" path="src/test/java">
+ <attributes>
+ <attribute name="optional" value="true"/>
+ <attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
diff --git a/vipra-rest/.project b/vipra-rest/.project
index 569e5873ae092077c30dea1efc63f96c79aa8564..7ff77ec0639e00436f95abc88c841d855d6a866c 100644
--- a/vipra-rest/.project
+++ b/vipra-rest/.project
@@ -27,12 +27,12 @@
</arguments>
</buildCommand>
<buildCommand>
- <name>org.eclipse.m2e.core.maven2Builder</name>
+ <name>net.sourceforge.metrics.builder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
- <name>net.sourceforge.metrics.builder</name>
+ <name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
diff --git a/vipra-rest/src/main/java/de/vipra/rest/provider/CORSResponseFilter.java b/vipra-rest/src/main/java/de/vipra/rest/provider/CORSResponseFilter.java
index 93c6084b63760a9fee2e64462d1948b8ba9ee6d0..8e65b96201384a46edb7bc358bfe340a579482b2 100644
--- a/vipra-rest/src/main/java/de/vipra/rest/provider/CORSResponseFilter.java
+++ b/vipra-rest/src/main/java/de/vipra/rest/provider/CORSResponseFilter.java
@@ -7,13 +7,13 @@ import javax.ws.rs.container.ContainerResponseContext;
import javax.ws.rs.container.ContainerResponseFilter;
import javax.ws.rs.ext.Provider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
@Provider
public class CORSResponseFilter implements ContainerResponseFilter {
- public static final Logger log = LoggerFactory.getLogger(CORSResponseFilter.class);
+ public static final Logger log = LogManager.getLogger(CORSResponseFilter.class);
public CORSResponseFilter() {
log.info("cors filter registered");
diff --git a/vipra-rest/src/main/java/de/vipra/rest/provider/InitializationListener.java b/vipra-rest/src/main/java/de/vipra/rest/provider/InitializationListener.java
index 6555abaf5eb276c376b1c2b26e6b0f9aa404660f..3c82ec125aaae21816e8d88042ce99d9e54bb701 100644
--- a/vipra-rest/src/main/java/de/vipra/rest/provider/InitializationListener.java
+++ b/vipra-rest/src/main/java/de/vipra/rest/provider/InitializationListener.java
@@ -4,14 +4,14 @@ import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.ehcache.CacheManager;
import org.ehcache.CacheManagerBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public class InitializationListener implements ServletContextListener {
- public static final Logger log = LoggerFactory.getLogger(InitializationListener.class);
+ public static final Logger log = LogManager.getLogger(InitializationListener.class);
@Override
public void contextDestroyed(ServletContextEvent sce) {
diff --git a/vipra-rest/src/main/java/de/vipra/rest/provider/ObjectMapperProvider.java b/vipra-rest/src/main/java/de/vipra/rest/provider/ObjectMapperProvider.java
index fcab9e67989b418e3be5c48973df8e350ab46ca8..9fcbcc76d867391c4bda9812ed681d5b8ff850e5 100644
--- a/vipra-rest/src/main/java/de/vipra/rest/provider/ObjectMapperProvider.java
+++ b/vipra-rest/src/main/java/de/vipra/rest/provider/ObjectMapperProvider.java
@@ -5,9 +5,9 @@ import java.text.SimpleDateFormat;
import javax.ws.rs.ext.ContextResolver;
import javax.ws.rs.ext.Provider;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.bson.types.ObjectId;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -27,7 +27,7 @@ import de.vipra.util.model.Word;
@Provider
public class ObjectMapperProvider implements ContextResolver<ObjectMapper> {
- public static final Logger log = LoggerFactory.getLogger(ObjectMapperProvider.class);
+ public static final Logger log = LogManager.getLogger(ObjectMapperProvider.class);
final ObjectMapper defaultObjectMapper;
public ObjectMapperProvider() {
diff --git a/vipra-rest/src/main/java/de/vipra/rest/resource/ArticleResource.java b/vipra-rest/src/main/java/de/vipra/rest/resource/ArticleResource.java
index 3c9e531a333ee1b28089b98cac75183a5e06f9f7..e4f8a2b8744292e85985ce575156f4a79d294217 100644
--- a/vipra-rest/src/main/java/de/vipra/rest/resource/ArticleResource.java
+++ b/vipra-rest/src/main/java/de/vipra/rest/resource/ArticleResource.java
@@ -55,8 +55,8 @@ public class ArticleResource {
CacheManager manager = (CacheManager) servletContext.getAttribute("cachemanager");
Cache<String, ArticleFull> articleCache = manager.getCache("articlecache", String.class, ArticleFull.class);
if (articleCache == null)
- articleCache = manager.createCache("articlecache",
- CacheConfigurationBuilder.newCacheConfigurationBuilder().buildConfig(String.class, ArticleFull.class));
+ articleCache = manager.createCache("articlecache", CacheConfigurationBuilder.newCacheConfigurationBuilder()
+ .buildConfig(String.class, ArticleFull.class));
this.cache = articleCache;
}
@@ -74,12 +74,12 @@ public class ArticleResource {
try {
List<ArticleFull> articles = service.getMultiple(skip, limit, sortBy, StringUtils.getFields(fields));
-
- if((skip != null && skip > 0) || (limit != null && limit > 0))
+
+ if ((skip != null && skip > 0) || (limit != null && limit > 0))
res.addMeta("total", service.count());
else
res.addMeta("total", articles.size());
-
+
return res.ok(articles);
} catch (Exception e) {
res.addError(new APIError(Response.Status.BAD_REQUEST, "Error", e.getMessage()));
diff --git a/vipra-rest/src/main/java/de/vipra/rest/resource/TopicResource.java b/vipra-rest/src/main/java/de/vipra/rest/resource/TopicResource.java
index 691dc746606bc19ca3ad87d016eed66449e35766..2e65bf3939d8a44496da4afe9fb1c79a92a41e76 100644
--- a/vipra-rest/src/main/java/de/vipra/rest/resource/TopicResource.java
+++ b/vipra-rest/src/main/java/de/vipra/rest/resource/TopicResource.java
@@ -68,12 +68,12 @@ public class TopicResource {
try {
List<TopicFull> topics = service.getMultiple(skip, limit, sortBy, StringUtils.getFields(fields));
-
- if((skip != null && skip > 0) || (limit != null && limit > 0))
+
+ if ((skip != null && skip > 0) || (limit != null && limit > 0))
res.addMeta("total", service.count());
else
res.addMeta("total", topics.size());
-
+
return res.ok(topics);
} catch (Exception e) {
res.addError(new APIError(Response.Status.BAD_REQUEST, "Error", e.getMessage()));
diff --git a/vipra-util/.classpath b/vipra-util/.classpath
index cf6dc8577b7a1ef815c8060b30722361a43c9e05..332896b54e5e51f9182bf4cc67b19abb799ed950 100644
--- a/vipra-util/.classpath
+++ b/vipra-util/.classpath
@@ -20,7 +20,7 @@
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<attributes>
- <attribute name="owner.project.facets" value="java"/>
+ <attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
diff --git a/vipra-util/.project b/vipra-util/.project
index 8f910748986bb91f3b50301de8b035f643c557ad..ff1b8e15fd6bc9f8fbeb06a83e8bc19368fdeac0 100644
--- a/vipra-util/.project
+++ b/vipra-util/.project
@@ -21,12 +21,12 @@
</arguments>
</buildCommand>
<buildCommand>
- <name>org.eclipse.m2e.core.maven2Builder</name>
+ <name>net.sourceforge.metrics.builder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
- <name>net.sourceforge.metrics.builder</name>
+ <name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
diff --git a/vipra-util/pom.xml b/vipra-util/pom.xml
index 7809631072e6a1215271c217742ab54b1faacbeb..d7682c6490149e5d84b0f4e16e9df186f78b21ed 100644
--- a/vipra-util/pom.xml
+++ b/vipra-util/pom.xml
@@ -10,8 +10,6 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
- <log4jVersion>2.4.1</log4jVersion>
- <jacksonVersion>2.7.0</jacksonVersion>
</properties>
<dependencies>
@@ -22,57 +20,35 @@
<version>2.4</version>
</dependency>
- <!-- Logging -->
+ <!-- SLF4j logging -->
<dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- <version>${log4jVersion}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- <version>${log4jVersion}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
- <version>${log4jVersion}</version>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.14</version>
</dependency>
<!-- MongoDB Database Adapter -->
- <dependency>
- <groupId>org.mongodb</groupId>
- <artifactId>mongodb-driver</artifactId>
- <version>3.2.0</version>
- </dependency>
<dependency>
<groupId>org.mongodb.morphia</groupId>
<artifactId>morphia</artifactId>
- <version>1.0.1</version>
+ <version>1.1.0</version>
</dependency>
<dependency>
<groupId>org.mongodb.morphia</groupId>
<artifactId>morphia-logging-slf4j</artifactId>
- <version>1.0.1</version>
- </dependency>
-
- <!-- ElasticSearch Adapter -->
- <dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- <version>2.1.0</version>
+ <version>1.1.0</version>
</dependency>
<!-- Jackson -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
- <version>${jacksonVersion}</version>
+ <version>2.7.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
- <version>${jacksonVersion}</version>
+ <version>2.7.0</version>
</dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/vipra-util/src/main/java/de/vipra/util/Config.java b/vipra-util/src/main/java/de/vipra/util/Config.java
index 134f5668e3db1092452d476a85899faf7f9b5c79..8bd41a0cd58c2b57d5fe3767bd458a9e8c7ad56b 100644
--- a/vipra-util/src/main/java/de/vipra/util/Config.java
+++ b/vipra-util/src/main/java/de/vipra/util/Config.java
@@ -9,13 +9,15 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import de.vipra.util.ex.ConfigException;
+import de.vipra.util.model.Model;
+import de.vipra.util.service.DatabaseService;
public class Config {
public static enum Key {
- DBHOST("db.host", Constants.DEFAULT_HOST),
- DBPORT("db.port", Constants.DEFAULT_PORT),
- DBNAME("db.name", Constants.DEFAULT_DB),
+ DBHOST("db.host", Constants.DB_HOST),
+ DBPORT("db.port", Constants.DB_PORT),
+ DBNAME("db.name", Constants.DB_NAME),
DATADIR("fs.datadir", null),
PREPROCESSOR("an.preprocessor", Constants.Processor.DEFAULT.name),
ANALYZER("an.analyzer", Constants.Analyzer.DEFAULT.name),
@@ -50,7 +52,7 @@ public class Config {
// load config from generic config dir
File configDir = getGenericConfigDir();
if (configDir != null && configDir.exists() && configDir.isDirectory()) {
- File file = new File(configDir, "config.properties");
+ File file = new File(configDir, Constants.CONFIG_FILE);
if (file.exists() && file.isFile()) {
in = FileUtils.openInputStream(file);
}
@@ -60,7 +62,6 @@ public class Config {
if (in == null) {
in = FileUtils.getResource(Constants.CONFIG_FILE);
}
-
load(in);
}
@@ -103,6 +104,15 @@ public class Config {
return dataDir;
}
+ public Mongo getMongo() throws ConfigException {
+ return Mongo.getInstance(this);
+ }
+
+ public <Type extends Model<IdType>, IdType> DatabaseService<Type, IdType> getDatabaseService(Class<Type> clazz)
+ throws ConfigException {
+ return DatabaseService.getDatabaseService(this, clazz);
+ }
+
public static File getGenericDataDir() {
File base = PathUtils.appDataDir();
return new File(base, Constants.FB_DIR);
diff --git a/vipra-util/src/main/java/de/vipra/util/Constants.java b/vipra-util/src/main/java/de/vipra/util/Constants.java
index 870b3a8545e32a5c776104ec2d72808f45be0c22..b4440ea1426fd2a57d236b9a5d582d1143e4d60c 100644
--- a/vipra-util/src/main/java/de/vipra/util/Constants.java
+++ b/vipra-util/src/main/java/de/vipra/util/Constants.java
@@ -26,9 +26,15 @@ public class Constants {
* DATABASE
*/
- public static final String DEFAULT_HOST = "localhost";
- public static final int DEFAULT_PORT = 27017;
- public static final String DEFAULT_DB = "test";
+ public static final String DB_HOST = "localhost";
+ public static final int DB_PORT = 27017;
+ public static final String DB_NAME = "test";
+
+ /*
+ * ELASTICSEARCH
+ */
+ public static final String ES_HOST = "127.0.0.1";
+ public static final int ES_PORT = 9300;
/*
* TOPIC MODELING
@@ -177,9 +183,7 @@ public class Constants {
* The text processors available, including the default text processor
*/
public static enum Processor {
- CUSTOM("custom"),
CORENLP("corenlp"),
- LUCENE("lucene"),
DEFAULT(CORENLP);
public final String name;
diff --git a/vipra-util/src/main/java/de/vipra/util/ElasticSerializer.java b/vipra-util/src/main/java/de/vipra/util/ElasticSerializer.java
new file mode 100644
index 0000000000000000000000000000000000000000..01eec7f98f3b78f120997b26c1a07e8c295c8c2c
--- /dev/null
+++ b/vipra-util/src/main/java/de/vipra/util/ElasticSerializer.java
@@ -0,0 +1,72 @@
+package de.vipra.util;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import de.vipra.util.an.ElasticIndex;
+
+public class ElasticSerializer<T> {
+
+ private Set<Entry<String, Field>> fields;
+ private Set<Entry<String, Method>> methods;
+
+ public ElasticSerializer(Class<T> clazz) {
+ Map<String, Field> foundFields = new HashMap<>();
+ Map<String, Method> foundMethods = new HashMap<>();
+
+ for (Field field : clazz.getDeclaredFields()) {
+ if (!Modifier.isStatic(field.getModifiers())) {
+ if (!field.isAccessible())
+ field.setAccessible(true);
+
+ ElasticIndex ei = field.getDeclaredAnnotation(ElasticIndex.class);
+ if (ei == null)
+ continue;
+
+ foundFields.put(ei.value(), field);
+ }
+ }
+
+ for (Method method : clazz.getDeclaredMethods()) {
+ if (!Modifier.isStatic(method.getModifiers()) && method.getParameterCount() == 0) {
+ if (!method.isAccessible())
+ method.setAccessible(true);
+
+ ElasticIndex ei = method.getDeclaredAnnotation(ElasticIndex.class);
+ if (ei == null)
+ continue;
+
+ foundMethods.put(ei.value(), method);
+ }
+ }
+
+ this.fields = foundFields.entrySet();
+ this.methods = foundMethods.entrySet();
+ }
+
+ public Map<String, Object> serialize(T t) {
+ Map<String, Object> values = new HashMap<>();
+ for (Entry<String, Field> e : fields) {
+ try {
+ values.put(e.getKey(), e.getValue().get(t));
+ } catch (IllegalArgumentException | IllegalAccessException e1) {
+ throw new RuntimeException("could not serialize field value " + e.getKey(), e1);
+ }
+ }
+ for (Entry<String, Method> m : methods) {
+ try {
+ values.put(m.getKey(), m.getValue().invoke(t));
+ } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e1) {
+ throw new RuntimeException("could not serialize method value " + m.getKey(), e1);
+ }
+ }
+ return values;
+ }
+
+}
diff --git a/vipra-util/src/main/java/de/vipra/util/Mongo.java b/vipra-util/src/main/java/de/vipra/util/Mongo.java
index c56f6a2998d067460525ea0b6b37bb057cbbc615..b38c61640e457073856980636bdbda95efc516b6 100644
--- a/vipra-util/src/main/java/de/vipra/util/Mongo.java
+++ b/vipra-util/src/main/java/de/vipra/util/Mongo.java
@@ -8,7 +8,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.mongodb.MongoClient;
-import com.mongodb.client.MongoDatabase;
+import com.mongodb.MongoClientOptions;
import de.vipra.util.Config.Key;
import de.vipra.util.ex.ConfigException;
@@ -24,7 +24,6 @@ public class Mongo {
private static Mongo instance;
private final MongoClient client;
- private final MongoDatabase database;
private final Morphia morphia;
private final Datastore datastore;
@@ -38,8 +37,9 @@ public class Mongo {
throw new ConfigException("host/port/dbname missing in configuration");
}
- client = new MongoClient(host, port);
- database = client.getDatabase(databaseName);
+ MongoClientOptions options = MongoClientOptions.builder().connectTimeout(10000).build();
+
+ client = new MongoClient(host + ":" + port, options);
morphia = new Morphia();
morphia.mapPackage("de.vipra.util.model");
@@ -50,10 +50,6 @@ public class Mongo {
return client;
}
- public MongoDatabase getDatabase() {
- return database;
- }
-
public Morphia getMorphia() {
return morphia;
}
diff --git a/vipra-util/src/main/java/de/vipra/util/an/ElasticIndex.java b/vipra-util/src/main/java/de/vipra/util/an/ElasticIndex.java
new file mode 100644
index 0000000000000000000000000000000000000000..b1d88bc6a1aa795a1e4bc7aae6df17b61c49d357
--- /dev/null
+++ b/vipra-util/src/main/java/de/vipra/util/an/ElasticIndex.java
@@ -0,0 +1,14 @@
+package de.vipra.util.an;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ ElementType.FIELD, ElementType.METHOD })
+public @interface ElasticIndex {
+
+ public String value() default "";
+
+}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/Topic.java b/vipra-util/src/main/java/de/vipra/util/model/Topic.java
index 894ba962562005cdb318ba3c7a2c00984cf9b736..8678b4684f16370756335eb5d46181235ffb3486 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/Topic.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/Topic.java
@@ -14,7 +14,7 @@ import de.vipra.util.an.JsonType;
@SuppressWarnings("serial")
@JsonType("topic")
@Entity(value = "topics", noClassnameStored = true)
-@Indexes({@Index("name"), @Index("-created")})
+@Indexes({ @Index("name"), @Index("-created") })
public class Topic implements Model<ObjectId>, Serializable {
@Id
diff --git a/vipra-util/src/main/java/de/vipra/util/service/DatabaseService.java b/vipra-util/src/main/java/de/vipra/util/service/DatabaseService.java
index ab2b058b18ff1cd2a55f63648793471b0184685f..d68e1bcc1c56d129e406bbf4c8a866b0652bcb8d 100644
--- a/vipra-util/src/main/java/de/vipra/util/service/DatabaseService.java
+++ b/vipra-util/src/main/java/de/vipra/util/service/DatabaseService.java
@@ -18,14 +18,14 @@ import de.vipra.util.ex.ConfigException;
import de.vipra.util.ex.DatabaseException;
import de.vipra.util.model.Model;
-public class DatabaseService<T extends Model<?>, U> implements Service<T, U, DatabaseException> {
+public class DatabaseService<Type extends Model<IdType>, IdType> implements Service<Type, IdType, DatabaseException> {
private final Datastore datastore;
- private final Class<T> clazz;
+ private final Class<Type> clazz;
private final String[] ignoredFieldsSingle;
private final String[] ignoredFieldsMulti;
- public DatabaseService(Mongo mongo, Class<T> clazz) {
+ public DatabaseService(Mongo mongo, Class<Type> clazz) {
this.datastore = mongo.getDatastore();
this.clazz = clazz;
@@ -46,8 +46,8 @@ public class DatabaseService<T extends Model<?>, U> implements Service<T, U, Dat
}
@Override
- public T getSingle(U id, String... fields) {
- Query<T> q = datastore.createQuery(clazz).field("_id").equal(id);
+ public Type getSingle(IdType id, String... fields) {
+ Query<Type> q = datastore.createQuery(clazz).field("_id").equal(id);
if (fields != null && fields.length > 0)
q.retrievedFields(true, setMinus(fields, ignoredFieldsSingle));
else if (ignoredFieldsSingle.length > 0)
@@ -56,13 +56,13 @@ public class DatabaseService<T extends Model<?>, U> implements Service<T, U, Dat
}
@Override
- public List<T> getMultiple(Integer skip, Integer limit, String sortBy, String... fields) {
+ public List<Type> getMultiple(Integer skip, Integer limit, String sortBy, String... fields) {
return getMultiple(skip, limit, sortBy, true, fields);
}
@Override
- public List<T> getMultiple(Integer skip, Integer limit, String sortBy, boolean defaultIgnore, String... fields) {
- Query<T> q = datastore.createQuery(clazz);
+ public List<Type> getMultiple(Integer skip, Integer limit, String sortBy, boolean defaultIgnore, String... fields) {
+ Query<Type> q = datastore.createQuery(clazz);
if (skip != null && skip > 0)
q.offset(skip);
if (limit != null && limit > 0)
@@ -73,35 +73,35 @@ public class DatabaseService<T extends Model<?>, U> implements Service<T, U, Dat
q.retrievedFields(true, setMinus(fields, ignoredFieldsMulti));
else if (defaultIgnore && ignoredFieldsMulti.length > 0)
q.retrievedFields(false, ignoredFieldsMulti);
- List<T> list = q.asList();
+ List<Type> list = q.asList();
return list;
}
@Override
- public List<T> getAll(String... fields) {
+ public List<Type> getAll(String... fields) {
return getMultiple(null, null, null, fields);
}
@Override
- public T createSingle(T t) throws DatabaseException {
+ public Type createSingle(Type t) throws DatabaseException {
datastore.save(t);
return t;
}
@Override
- public List<T> createMultiple(Iterable<T> t) throws DatabaseException {
- List<T> list = ListUtils.toList(t);
+ public List<Type> createMultiple(Iterable<Type> t) throws DatabaseException {
+ List<Type> list = ListUtils.toList(t);
datastore.save(list);
return list;
}
@Override
- public long deleteSingle(U id) throws DatabaseException {
+ public long deleteSingle(IdType id) throws DatabaseException {
return datastore.delete(id).getN();
}
@Override
- public void updateSingle(T t) throws DatabaseException {
+ public void updateSingle(Type t) throws DatabaseException {
datastore.save(t);
}
@@ -115,10 +115,10 @@ public class DatabaseService<T extends Model<?>, U> implements Service<T, U, Dat
return datastore.getCount(clazz);
}
- public static <T extends Model<?>, U> DatabaseService<T, U> getDatabaseService(Config config, Class<T> clazz)
- throws ConfigException {
- Mongo mongo = Mongo.getInstance(config);
- return new DatabaseService<T, U>(mongo, clazz);
+ public static <Type extends Model<IdType>, IdType> DatabaseService<Type, IdType> getDatabaseService(Config config,
+ Class<Type> clazz) throws ConfigException {
+ Mongo mongo = config.getMongo();
+ return new DatabaseService<Type, IdType>(mongo, clazz);
}
private String[] setMinus(String[] a, String[] b) {
diff --git a/vipra-util/src/main/java/de/vipra/util/service/Service.java b/vipra-util/src/main/java/de/vipra/util/service/Service.java
index 95f6a6d0f500796ebcf913d06b176fe622a713d4..3282b1728c499bd70178ff7326d073fe8ecf1ed6 100644
--- a/vipra-util/src/main/java/de/vipra/util/service/Service.java
+++ b/vipra-util/src/main/java/de/vipra/util/service/Service.java
@@ -4,7 +4,7 @@ import java.util.List;
import de.vipra.util.model.Model;
-public interface Service<Type extends Model<?>, IdType, E extends Exception> {
+public interface Service<Type extends Model<IdType>, IdType, E extends Exception> {
Type getSingle(IdType id, String... fields) throws E;
diff --git a/vipra-util/vipra-util.jardesc b/vipra-util/vipra-util.jardesc
deleted file mode 100644
index f3d665873afd03d481b117484e29aafad42a4473..0000000000000000000000000000000000000000
--- a/vipra-util/vipra-util.jardesc
+++ /dev/null
@@ -1,24 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<jardesc>
- <jar path="vipra-util/bin/vipra-util.jar"/>
- <options buildIfNeeded="true" compress="true" descriptionLocation="/vipra-util/vipra-util.jardesc" exportErrors="true" exportWarnings="true" includeDirectoryEntries="false" overwrite="true" saveDescription="true" storeRefactorings="false" useSourceFolders="false"/>
- <storedRefactorings deprecationInfo="true" structuralOnly="false"/>
- <selectedProjects/>
- <manifest generateManifest="true" manifestLocation="" manifestVersion="1.0" reuseManifest="false" saveManifest="false" usesManifest="true">
- <sealing sealJar="false">
- <packagesToSeal/>
- <packagesToUnSeal/>
- </sealing>
- </manifest>
- <selectedElements exportClassFiles="true" exportJavaFiles="false" exportOutputFolder="false">
- <folder path="/vipra-util/src"/>
- <file path="/vipra-util/.classpath"/>
- <file path="/vipra-util/.gitignore"/>
- <folder path="/vipra-util/target"/>
- <file path="/vipra-util/.project"/>
- <folder path="/vipra-util/bin"/>
- <javaElement handleIdentifier="=vipra-util/src\/main\/resources"/>
- <javaElement handleIdentifier="=vipra-util/src\/main\/java"/>
- <file path="/vipra-util/pom.xml"/>
- </selectedElements>
-</jardesc>
diff --git a/vipra-ws/.settings/org.eclipse.wst.common.component b/vipra-ws/.settings/org.eclipse.wst.common.component
index 850a09742c3dd384045201d1aacf175da9677dff..4bef75339481d8416e5c64cd6a4a042c9070b784 100644
--- a/vipra-ws/.settings/org.eclipse.wst.common.component
+++ b/vipra-ws/.settings/org.eclipse.wst.common.component
@@ -3,7 +3,7 @@
<wb-resource deploy-path="/" source-path="/target/m2e-wtp/web-resources"/>
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/java"/>
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/resources"/>
- <wb-resource deploy-path="/" source-path="/src/main/webapp"/>
+ <wb-resource deploy-path="/" source-path="/src/main/webapp" tag="defaultRootSource"/>
<property name="context-root" value="vipra-ws"/>
<property name="java-output-path" value="/vipra-ws/build/classes"/>
</wb-module>