From d23a05288222cd88980f2ec09531d505d6dd4648 Mon Sep 17 00:00:00 2001
From: Eike Cochu <eike@cochu.com>
Date: Thu, 28 Jan 2016 13:50:14 +0100
Subject: [PATCH] too many changes
removed lucene and custom processors
added debug switch to enable extensive productive logging
added silent switch to mute all logging
replaced cmd project slf4j loggers with log4j loggers
simplified cmd exception structure
added catching mongo and elasticsearch connection exceptions with explicit error messages
removed delete cmd
removed lucene dependency, clashes with elasticsearch lucene 5.0 dependency in executable jar
deleted unnecessary jardesc
removed log4j dependency from utils project
renamed database constants
added elasticsearch constants
added database and elasticsearch config convenience functions
added missing database generic types
added test command to test connections to database and elasticsearch
---
Vagrantfile | 1 +
ma-impl.sublime-workspace | 40 ++-
vipra-cmd/.classpath | 8 +-
vipra-cmd/.project | 4 +-
vipra-cmd/build.xml | 161 ++++-----
vipra-cmd/pom.xml | 41 +--
.../main/java/de/vipra/cmd/CmdOptions.java | 17 +-
.../java/de/vipra/cmd/ExecutionException.java | 38 ---
.../src/main/java/de/vipra/cmd/Main.java | 93 +++---
.../main/java/de/vipra/cmd/es/ESClient.java | 19 ++
.../java/de/vipra/cmd/ex/ClearException.java | 15 -
.../java/de/vipra/cmd/ex/ImportException.java | 23 --
.../de/vipra/cmd/lda/JGibbLDAAnalyzer.java | 6 +-
.../de/vipra/cmd/model/ProcessedArticle.java | 18 +
.../de/vipra/cmd/option/ClearCommand.java | 49 ++-
.../java/de/vipra/cmd/option/Command.java | 4 +-
.../de/vipra/cmd/option/DeleteCommand.java | 101 ------
.../de/vipra/cmd/option/ImportCommand.java | 314 +++++++++---------
.../de/vipra/cmd/option/StatsCommand.java | 28 +-
.../java/de/vipra/cmd/option/TestCommand.java | 39 +++
.../de/vipra/cmd/text/CustomProcessor.java | 36 --
.../de/vipra/cmd/text/LuceneProcessor.java | 59 ----
.../java/de/vipra/cmd/text/Processor.java | 4 -
.../de/vipra/cmd/text/StopwordsAnnotator.java | 2 +-
vipra-cmd/src/main/resources/log4j2dev.xml | 1 +
vipra-rest/.classpath | 8 +-
vipra-rest/.project | 4 +-
.../rest/provider/CORSResponseFilter.java | 6 +-
.../rest/provider/InitializationListener.java | 6 +-
.../rest/provider/ObjectMapperProvider.java | 6 +-
.../vipra/rest/resource/ArticleResource.java | 10 +-
.../de/vipra/rest/resource/TopicResource.java | 6 +-
vipra-util/.classpath | 2 +-
vipra-util/.project | 4 +-
vipra-util/pom.xml | 40 +--
.../src/main/java/de/vipra/util/Config.java | 20 +-
.../main/java/de/vipra/util/Constants.java | 14 +-
.../java/de/vipra/util/ElasticSerializer.java | 72 ++++
.../src/main/java/de/vipra/util/Mongo.java | 12 +-
.../java/de/vipra/util/an/ElasticIndex.java | 14 +
.../main/java/de/vipra/util/model/Topic.java | 2 +-
.../vipra/util/service/DatabaseService.java | 38 +--
.../java/de/vipra/util/service/Service.java | 2 +-
vipra-util/vipra-util.jardesc | 24 --
.../org.eclipse.wst.common.component | 2 +-
45 files changed, 636 insertions(+), 777 deletions(-)
delete mode 100644 vipra-cmd/src/main/java/de/vipra/cmd/ExecutionException.java
create mode 100644 vipra-cmd/src/main/java/de/vipra/cmd/es/ESClient.java
delete mode 100644 vipra-cmd/src/main/java/de/vipra/cmd/ex/ClearException.java
delete mode 100644 vipra-cmd/src/main/java/de/vipra/cmd/ex/ImportException.java
delete mode 100644 vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteCommand.java
create mode 100644 vipra-cmd/src/main/java/de/vipra/cmd/option/TestCommand.java
delete mode 100644 vipra-cmd/src/main/java/de/vipra/cmd/text/CustomProcessor.java
delete mode 100644 vipra-cmd/src/main/java/de/vipra/cmd/text/LuceneProcessor.java
create mode 100644 vipra-util/src/main/java/de/vipra/util/ElasticSerializer.java
create mode 100644 vipra-util/src/main/java/de/vipra/util/an/ElasticIndex.java
delete mode 100644 vipra-util/vipra-util.jardesc
diff --git a/Vagrantfile b/Vagrantfile
index 7facb0b9..28cab3ce 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -14,6 +14,7 @@ Vagrant.configure(2) do |config|
master.vm.network :forwarded_port, guest: 27017, host: 27017 # MongoDB
master.vm.network :forwarded_port, guest: 8080, host: 8000 # Tomcat
master.vm.network :forwarded_port, guest: 9200, host: 9200 # ElasticSearch REST API
+ master.vm.network :forwarded_port, guest: 9300, host: 9300 # ElasticSearch Native
end
end
diff --git a/ma-impl.sublime-workspace b/ma-impl.sublime-workspace
index dda352d0..29c324ca 100644
--- a/ma-impl.sublime-workspace
+++ b/ma-impl.sublime-workspace
@@ -279,6 +279,14 @@
},
"buffers":
[
+ {
+ "file": "Vagrantfile",
+ "settings":
+ {
+ "buffer_size": 572,
+ "line_ending": "Unix"
+ }
+ }
],
"build_system": "",
"build_system_choices":
@@ -458,7 +466,6 @@
"expanded_folders":
[
"/home/eike/repos/master/ma-impl",
- "/home/eike/repos/master/ma-impl/vipra-ui",
"/home/eike/repos/master/ma-impl/vipra-ui/app",
"/home/eike/repos/master/ma-impl/vipra-ui/app/components",
"/home/eike/repos/master/ma-impl/vipra-ui/app/routes",
@@ -924,8 +931,39 @@
"groups":
[
{
+ "selected": 0,
"sheets":
[
+ {
+ "buffer": 0,
+ "file": "Vagrantfile",
+ "semi_transient": true,
+ "settings":
+ {
+ "buffer_size": 572,
+ "regions":
+ {
+ },
+ "selection":
+ [
+ [
+ 437,
+ 437
+ ]
+ ],
+ "settings":
+ {
+ "syntax": "Packages/Text/Plain text.tmLanguage",
+ "tab_size": 2,
+ "translate_tabs_to_spaces": true
+ },
+ "translation.x": 0.0,
+ "translation.y": 0.0,
+ "zoom_level": 1.0
+ },
+ "stack_index": 0,
+ "type": "text"
+ }
]
}
],
diff --git a/vipra-cmd/.classpath b/vipra-cmd/.classpath
index 68ba2743..b0ed5811 100644
--- a/vipra-cmd/.classpath
+++ b/vipra-cmd/.classpath
@@ -29,7 +29,13 @@
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<attributes>
- <attribute name="owner.project.facets" value="java"/>
+ <attribute name="maven.pomderived" value="true"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry kind="src" output="target/test-classes" path="src/test/java">
+ <attributes>
+ <attribute name="optional" value="true"/>
+ <attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
diff --git a/vipra-cmd/.project b/vipra-cmd/.project
index b44a107f..064ef467 100644
--- a/vipra-cmd/.project
+++ b/vipra-cmd/.project
@@ -21,12 +21,12 @@
</arguments>
</buildCommand>
<buildCommand>
- <name>org.eclipse.m2e.core.maven2Builder</name>
+ <name>net.sourceforge.metrics.builder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
- <name>net.sourceforge.metrics.builder</name>
+ <name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
diff --git a/vipra-cmd/build.xml b/vipra-cmd/build.xml
index 37709cab..8ff573bc 100644
--- a/vipra-cmd/build.xml
+++ b/vipra-cmd/build.xml
@@ -1,83 +1,84 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<project default="create_run_jar" name="Create Runnable Jar for Project vipra-cmd">
- <!--this file was created by Eclipse Runnable JAR Export Wizard-->
- <!--ANT 1.7 is required -->
- <!--define folder properties-->
- <property name="dir.buildfile" value="."/>
- <property name="dir.workspace" value="/home/eike/workspace"/>
- <property name="dir.jarfile" value="${dir.buildfile}/bin"/>
- <target name="create_run_jar">
- <jar destfile="${dir.jarfile}/vipra-cmd.jar" filesetmanifest="mergewithoutmain">
- <manifest>
- <attribute name="Main-Class" value="de.vipra.cmd.Main"/>
- <attribute name="Class-Path" value="."/>
- </manifest>
- <fileset dir="${dir.buildfile}/target/classes"/>
- <fileset dir="/home/eike/repos/master/ma-impl/vipra-util/target/classes"/>
- <fileset dir="/home/eike/repos/master/ma-impl/jgibblda/target/classes"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-api/2.4.1/log4j-api-2.4.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-core/2.4.1/log4j-core-2.4.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-slf4j-impl/2.4.1/log4j-slf4j-impl-2.4.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/slf4j/slf4j-api/1.7.12/slf4j-api-1.7.12.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/mongodb-driver/3.2.0/mongodb-driver-3.2.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/bson/3.2.0/bson-3.2.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/mongodb-driver-core/3.2.0/mongodb-driver-core-3.2.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/morphia/morphia/1.0.1/morphia-1.0.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/mongo-java-driver/3.0.2/mongo-java-driver-3.0.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/thoughtworks/proxytoys/proxytoys/1.0/proxytoys-1.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/cglib/cglib-nodep/2.2.2/cglib-nodep-2.2.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/morphia/morphia-logging-slf4j/1.0.1/morphia-logging-slf4j-1.0.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/elasticsearch/elasticsearch/2.1.0/elasticsearch-2.1.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-core/5.3.1/lucene-core-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-backward-codecs/5.3.1/lucene-backward-codecs-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-analyzers-common/5.3.1/lucene-analyzers-common-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-queries/5.3.1/lucene-queries-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-memory/5.3.1/lucene-memory-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-highlighter/5.3.1/lucene-highlighter-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-queryparser/5.3.1/lucene-queryparser-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-sandbox/5.3.1/lucene-sandbox-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-suggest/5.3.1/lucene-suggest-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-misc/5.3.1/lucene-misc-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-join/5.3.1/lucene-join-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-grouping/5.3.1/lucene-grouping-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-spatial/5.3.1/lucene-spatial-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-spatial3d/5.3.1/lucene-spatial3d-5.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/spatial4j/spatial4j/0.4.1/spatial4j-0.4.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/google/guava/guava/18.0/guava-18.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/carrotsearch/hppc/0.7.1/hppc-0.7.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/joda-time/joda-time/2.8.2/joda-time-2.8.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/joda/joda-convert/1.2/joda-convert-1.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/core/jackson-core/2.6.2/jackson-core-2.6.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/dataformat/jackson-dataformat-smile/2.6.2/jackson-dataformat-smile-2.6.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/dataformat/jackson-dataformat-yaml/2.6.2/jackson-dataformat-yaml-2.6.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/yaml/snakeyaml/1.15/snakeyaml-1.15.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.6.2/jackson-dataformat-cbor-2.6.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/io/netty/netty/3.10.5.Final/netty-3.10.5.Final.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/ning/compress-lzf/1.0.2/compress-lzf-1.0.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/tdunning/t-digest/3.0/t-digest-3.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/hdrhistogram/HdrHistogram/2.1.6/HdrHistogram-2.1.6.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/commons-cli/commons-cli/1.3.1/commons-cli-1.3.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/twitter/jsr166e/1.1.0/jsr166e-1.1.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/core/jackson-databind/2.7.0/jackson-databind-2.7.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/core/jackson-annotations/2.7.0/jackson-annotations-2.7.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/args4j/args4j/2.0.6/args4j-2.0.6.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/googlecode/json-simple/json-simple/1.1.1/json-simple-1.1.1.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/edu/stanford/nlp/stanford-corenlp/3.5.2/stanford-corenlp-3.5.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/io7m/xom/xom/1.2.10/xom-1.2.10.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/xml-apis/xml-apis/1.3.03/xml-apis-1.3.03.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/xerces/xercesImpl/2.8.0/xercesImpl-2.8.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/xalan/xalan/2.7.0/xalan-2.7.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/de/jollyday/jollyday/0.4.7/jollyday-0.4.7.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/javax/xml/bind/jaxb-api/2.2.7/jaxb-api-2.2.7.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/googlecode/efficient-java-matrix-library/ejml/0.23/ejml-0.23.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/javax/json/javax.json-api/1.0/javax.json-api-1.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/edu/stanford/nlp/stanford-corenlp/3.5.2/stanford-corenlp-3.5.2-models.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-core/5.4.0/lucene-core-5.4.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-analyzers-common/5.4.0/lucene-analyzers-common-5.4.0.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/uk/org/lidalia/sysout-over-slf4j/1.0.2/sysout-over-slf4j-1.0.2.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/junit/junit/4.12/junit-4.12.jar"/>
- <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar"/>
- </jar>
- </target>
+ <!--this file was created by Eclipse Runnable JAR Export Wizard-->
+ <!--ANT 1.7 is required -->
+ <!--define folder properties-->
+ <property name="dir.buildfile" value="." />
+ <property name="dir.jarfile" value="${dir.buildfile}/bin" />
+ <target name="create_run_jar">
+ <jar destfile="${dir.jarfile}/vipra-cmd.jar" filesetmanifest="mergewithoutmain" duplicate="preserve">
+ <manifest>
+ <attribute name="Main-Class" value="de.vipra.cmd.Main" />
+ <attribute name="Class-Path" value="." />
+ </manifest>
+ <fileset dir="${dir.buildfile}/target/classes" />
+ <fileset dir="/home/eike/repos/master/ma-impl/vipra-util/target/classes" />
+ <fileset dir="/home/eike/repos/master/ma-impl/jgibblda/target/classes" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/slf4j/slf4j-api/1.7.14/slf4j-api-1.7.14.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/mongodb-driver/3.2.0/mongodb-driver-3.2.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/bson/3.2.0/bson-3.2.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/mongodb-driver-core/3.2.0/mongodb-driver-core-3.2.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/morphia/morphia/1.0.1/morphia-1.0.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/mongo-java-driver/3.0.2/mongo-java-driver-3.0.2.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/thoughtworks/proxytoys/proxytoys/1.0/proxytoys-1.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/cglib/cglib-nodep/2.2.2/cglib-nodep-2.2.2.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/morphia/morphia-logging-slf4j/1.0.1/morphia-logging-slf4j-1.0.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/core/jackson-databind/2.7.0/jackson-databind-2.7.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/core/jackson-core/2.7.0/jackson-core-2.7.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/core/jackson-annotations/2.7.0/jackson-annotations-2.7.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/args4j/args4j/2.0.6/args4j-2.0.6.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-api/2.4.1/log4j-api-2.4.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-core/2.4.1/log4j-core-2.4.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-slf4j-impl/2.4.1/log4j-slf4j-impl-2.4.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/slf4j/slf4j-api/1.7.12/slf4j-api-1.7.12.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/commons-cli/commons-cli/1.3.1/commons-cli-1.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/googlecode/json-simple/json-simple/1.1.1/json-simple-1.1.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/junit/junit/4.10/junit-4.10.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/hamcrest/hamcrest-core/1.1/hamcrest-core-1.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/elasticsearch/elasticsearch/2.1.0/elasticsearch-2.1.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-core/5.3.1/lucene-core-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-backward-codecs/5.3.1/lucene-backward-codecs-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-analyzers-common/5.3.1/lucene-analyzers-common-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-queries/5.3.1/lucene-queries-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-memory/5.3.1/lucene-memory-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-highlighter/5.3.1/lucene-highlighter-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-queryparser/5.3.1/lucene-queryparser-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-sandbox/5.3.1/lucene-sandbox-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-suggest/5.3.1/lucene-suggest-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-misc/5.3.1/lucene-misc-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-join/5.3.1/lucene-join-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-grouping/5.3.1/lucene-grouping-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-spatial/5.3.1/lucene-spatial-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-spatial3d/5.3.1/lucene-spatial3d-5.3.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/spatial4j/spatial4j/0.4.1/spatial4j-0.4.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/google/guava/guava/18.0/guava-18.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/carrotsearch/hppc/0.7.1/hppc-0.7.1.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/joda-time/joda-time/2.8.2/joda-time-2.8.2.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/joda/joda-convert/1.2/joda-convert-1.2.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/core/jackson-core/2.6.2/jackson-core-2.6.2.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/dataformat/jackson-dataformat-smile/2.6.2/jackson-dataformat-smile-2.6.2.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/dataformat/jackson-dataformat-yaml/2.6.2/jackson-dataformat-yaml-2.6.2.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/yaml/snakeyaml/1.15/snakeyaml-1.15.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.6.2/jackson-dataformat-cbor-2.6.2.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/io/netty/netty/3.10.5.Final/netty-3.10.5.Final.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/ning/compress-lzf/1.0.2/compress-lzf-1.0.2.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/tdunning/t-digest/3.0/t-digest-3.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/hdrhistogram/HdrHistogram/2.1.6/HdrHistogram-2.1.6.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/twitter/jsr166e/1.1.0/jsr166e-1.1.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/edu/stanford/nlp/stanford-corenlp/3.5.2/stanford-corenlp-3.5.2.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/io7m/xom/xom/1.2.10/xom-1.2.10.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/xml-apis/xml-apis/1.3.03/xml-apis-1.3.03.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/xerces/xercesImpl/2.8.0/xercesImpl-2.8.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/xalan/xalan/2.7.0/xalan-2.7.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/de/jollyday/jollyday/0.4.7/jollyday-0.4.7.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/javax/xml/bind/jaxb-api/2.2.7/jaxb-api-2.2.7.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/googlecode/efficient-java-matrix-library/ejml/0.23/ejml-0.23.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/javax/json/javax.json-api/1.0/javax.json-api-1.0.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/edu/stanford/nlp/stanford-corenlp/3.5.2/stanford-corenlp-3.5.2-models.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-api/2.5/log4j-api-2.5.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-core/2.5/log4j-core-2.5.jar" />
+ <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-slf4j-impl/2.5/log4j-slf4j-impl-2.5.jar" />
+ </jar>
+ </target>
</project>
diff --git a/vipra-cmd/pom.xml b/vipra-cmd/pom.xml
index 5aed56ed..bcba557b 100644
--- a/vipra-cmd/pom.xml
+++ b/vipra-cmd/pom.xml
@@ -14,8 +14,6 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
- <log4jVersion>2.4.1</log4jVersion>
- <luceneVersion>5.4.0</luceneVersion>
</properties>
<dependencies>
@@ -25,11 +23,6 @@
<artifactId>commons-cli</artifactId>
<version>1.3.1</version>
</dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <version>2.4</version>
- </dependency>
<!-- JsonSimple -->
<dependency>
@@ -58,57 +51,33 @@
<classifier>models</classifier>
</dependency>
- <!-- Lucene -->
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-core</artifactId>
- <version>${luceneVersion}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-analyzers-common</artifactId>
- <version>${luceneVersion}</version>
- </dependency>
-
<!-- Logging -->
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
- <version>${log4jVersion}</version>
+ <version>2.5</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
- <version>${log4jVersion}</version>
+ <version>2.5</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
- <version>${log4jVersion}</version>
+ <version>2.5</version>
</dependency>
<!-- MongoDB Database Adapter -->
- <dependency>
- <groupId>org.mongodb</groupId>
- <artifactId>mongodb-driver</artifactId>
- <version>3.2.0</version>
- </dependency>
<dependency>
<groupId>org.mongodb.morphia</groupId>
<artifactId>morphia</artifactId>
- <version>1.0.1</version>
+ <version>1.1.0</version>
</dependency>
<dependency>
<groupId>org.mongodb.morphia</groupId>
<artifactId>morphia-logging-slf4j</artifactId>
- <version>1.0.1</version>
- </dependency>
-
- <!-- Testing -->
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.12</version>
+ <version>1.1.0</version>
</dependency>
<!-- Workspace -->
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/CmdOptions.java b/vipra-cmd/src/main/java/de/vipra/cmd/CmdOptions.java
index a5e6824b..d3d6a79f 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/CmdOptions.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/CmdOptions.java
@@ -17,9 +17,6 @@ public class CmdOptions extends Options {
public static final String OPT_SHELL = "x";
public static final String OPT_SHELL_LONG = "shell";
- public static final String OPT_DELETE = "d";
- public static final String OPT_DELETE_LONG = "delete";
-
public static final String OPT_CLEAR = "c";
public static final String OPT_CLEAR_LONG = "clear";
@@ -29,18 +26,28 @@ public class CmdOptions extends Options {
public static final String OPT_DEFAULTS = "n";
public static final String OPT_DEFAULTS_LONG = "defaults";
+ public static final String OPT_DEBUG = "d";
+ public static final String OPT_DEBUG_LONG = "debug";
+
+ public static final String OPT_TEST = "t";
+ public static final String OPT_TEST_LONG = "test";
+
+ public static final String OPT_SILENT = "s";
+ public static final String OPT_SILENT_LONG = "silent";
+
public CmdOptions() {
addOption(Option.builder(OPT_HELP).longOpt(OPT_HELP_LONG).desc("print this message").build());
addOption(Option.builder(OPT_SHELL).longOpt(OPT_SHELL_LONG).hasArg(true).argName("name")
.desc("run from a shell script").build());
addOption(Option.builder(OPT_IMPORT).longOpt(OPT_IMPORT_LONG).hasArgs().argName("files/dirs...")
.desc("import articles into the database").build());
- addOption(Option.builder(OPT_DELETE).longOpt(OPT_DELETE_LONG).hasArgs().argName("files/ids...")
- .desc("delete articles from the database").build());
addOption(Option.builder(OPT_CLEAR).longOpt(OPT_CLEAR_LONG).desc("clear database and filebase").build());
addOption(Option.builder(OPT_STATS).longOpt(OPT_STATS_LONG).desc("gather database and filebase information")
.build());
addOption(Option.builder(OPT_DEFAULTS).longOpt(OPT_DEFAULTS_LONG).desc("accept default decisions").build());
+ addOption(Option.builder(OPT_DEBUG).longOpt(OPT_DEBUG_LONG).desc("show debug information").build());
+ addOption(Option.builder(OPT_TEST).longOpt(OPT_TEST_LONG).desc("system tests").build());
+ addOption(Option.builder(OPT_SILENT).longOpt(OPT_SILENT_LONG).desc("mute all output").build());
}
public void printHelp(String cmd) {
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/ExecutionException.java b/vipra-cmd/src/main/java/de/vipra/cmd/ExecutionException.java
deleted file mode 100644
index 6ea2de6b..00000000
--- a/vipra-cmd/src/main/java/de/vipra/cmd/ExecutionException.java
+++ /dev/null
@@ -1,38 +0,0 @@
-package de.vipra.cmd;
-
-import java.util.List;
-
-public class ExecutionException extends Exception {
-
- private static final long serialVersionUID = 1L;
-
- private List<Exception> exceptions;
-
- public ExecutionException(String msg) {
- super(msg);
- }
-
- public ExecutionException(Exception e) {
- super(e);
- }
-
- public ExecutionException(List<Exception> e) {
- this.exceptions = e;
- }
-
- @Override
- public String getMessage() {
- if (exceptions == null) {
- return super.getMessage();
- } else if (exceptions.size() == 1) {
- return exceptions.get(0).getMessage();
- } else {
- StringBuilder sb = new StringBuilder("multiple errors:");
- for (Exception e : exceptions) {
- sb.append("\n " + e.getMessage());
- }
- return sb.toString();
- }
- }
-
-}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/Main.java b/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
index c91124e5..6f2e0794 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/Main.java
@@ -1,44 +1,51 @@
package de.vipra.cmd;
import static de.vipra.cmd.CmdOptions.OPT_CLEAR;
+import static de.vipra.cmd.CmdOptions.OPT_DEBUG;
import static de.vipra.cmd.CmdOptions.OPT_DEFAULTS;
-import static de.vipra.cmd.CmdOptions.OPT_DELETE;
import static de.vipra.cmd.CmdOptions.OPT_HELP;
import static de.vipra.cmd.CmdOptions.OPT_IMPORT;
import static de.vipra.cmd.CmdOptions.OPT_SHELL;
+import static de.vipra.cmd.CmdOptions.OPT_SILENT;
import static de.vipra.cmd.CmdOptions.OPT_STATS;
+import static de.vipra.cmd.CmdOptions.OPT_TEST;
import java.util.ArrayList;
import java.util.List;
import java.util.ListIterator;
+import java.util.Map.Entry;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.ParseException;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
+import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.mongodb.morphia.logging.MorphiaLoggerFactory;
import org.mongodb.morphia.logging.slf4j.SLF4JLoggerImplFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+
+import com.mongodb.MongoTimeoutException;
import de.vipra.cmd.option.ClearCommand;
import de.vipra.cmd.option.Command;
-import de.vipra.cmd.option.DeleteCommand;
import de.vipra.cmd.option.ImportCommand;
import de.vipra.cmd.option.StatsCommand;
-import de.vipra.util.ConsoleUtils;
-import de.vipra.util.ConsoleUtils.Choice;
-import de.vipra.util.StringUtils;
-import de.vipra.util.Timer;
+import de.vipra.cmd.option.TestCommand;
public class Main {
- public static final Logger log = LoggerFactory.getLogger(Main.class);
- public static final Logger out = LoggerFactory.getLogger("shellout");
+ public static final Logger log = LogManager.getLogger(Main.class);
+ public static final Logger out = LogManager.getLogger("shellout");
static {
+ // set morphia log level
MorphiaLoggerFactory.registerLogger(SLF4JLoggerImplFactory.class);
- // close stderr to mute corenlp messages
+ // set corenlp log level, close stderr to mute corenlp messages
System.err.close();
}
@@ -67,53 +74,55 @@ public class Main {
return;
}
+ // logger configuration
+ LoggerContext loggerContext = (LoggerContext) LogManager.getContext(false);
+ Configuration loggerConfigs = loggerContext.getConfiguration();
+
+ if (cline.hasOption(OPT_DEBUG))
+ loggerConfigs.getLoggerConfig(LogManager.ROOT_LOGGER_NAME).setLevel(Level.DEBUG);
+
+ if (cline.hasOption(OPT_SILENT)) {
+ for (Entry<String, LoggerConfig> loggerConfig : loggerConfigs.getLoggers().entrySet())
+ loggerConfig.getValue().setLevel(Level.OFF);
+ }
+
+ loggerContext.updateLoggers();
+
+ // check if default decisions should be auto chosen
boolean defaults = cline.hasOption(OPT_DEFAULTS);
+ // get commands
List<Command> commands = new ArrayList<>();
- if (cline.hasOption(OPT_CLEAR)) {
+ if (cline.hasOption(OPT_TEST))
+ commands.add(new TestCommand());
+
+ if (cline.hasOption(OPT_CLEAR))
commands.add(new ClearCommand(defaults));
- }
- if (cline.hasOption(OPT_IMPORT)) {
+ if (cline.hasOption(OPT_IMPORT))
commands.add(new ImportCommand(cline.getOptionValues(OPT_IMPORT)));
- }
- if (cline.hasOption(OPT_DELETE)) {
- commands.add(new DeleteCommand(cline.getOptionValues(OPT_DELETE)));
- }
-
- if (cline.hasOption(OPT_STATS)) {
+ if (cline.hasOption(OPT_STATS))
commands.add(new StatsCommand());
- }
+ // run commands
if (commands.size() > 0) {
- Timer t = new Timer();
- t.start();
- commandLoop: for (ListIterator<Command> it = commands.listIterator(); it.hasNext();) {
+ for (ListIterator<Command> it = commands.listIterator(); it.hasNext();) {
Command c = it.next();
try {
c.run();
- } catch (ExecutionException e) {
- out.error(e.getMessage(), e);
- ConsoleUtils.Choice choice;
- boolean acceptDefault = cline.hasOption(OPT_DEFAULTS);
- do {
- choice = ConsoleUtils.prompt(Choice.CONTINUE, acceptDefault, Choice.ABORT, Choice.RETRY);
- switch (choice) {
- case ABORT:
- break commandLoop;
- case RETRY:
- it.previous();
- case CONTINUE:
- continue commandLoop;
-
- }
- } while (choice == null);
+ } catch (MongoTimeoutException e) {
+ out.error("timeout while trying to connect to the database");
+ log.debug(e.getMessage(), e);
+ } catch (NoNodeAvailableException e) {
+ out.error("could not connect to elasticsearch instance");
+ log.debug(e.getMessage(), e);
+ } catch (Exception e) {
+ out.error(e.getCause().getMessage());
+ log.debug(e.getMessage(), e);
}
}
- long dur = t.stop();
- out.info("done in " + StringUtils.timeString(dur));
} else {
options.printHelp(cmd);
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/es/ESClient.java b/vipra-cmd/src/main/java/de/vipra/cmd/es/ESClient.java
new file mode 100644
index 00000000..a48a17c9
--- /dev/null
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/es/ESClient.java
@@ -0,0 +1,19 @@
+package de.vipra.cmd.es;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+
+import de.vipra.util.Config;
+import de.vipra.util.Constants;
+
+public abstract class ESClient {
+
+ public static TransportClient getClient(Config config) throws UnknownHostException {
+ return TransportClient.builder().build().addTransportAddress(
+ new InetSocketTransportAddress(InetAddress.getByName(Constants.ES_HOST), Constants.ES_PORT));
+ }
+
+}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/ex/ClearException.java b/vipra-cmd/src/main/java/de/vipra/cmd/ex/ClearException.java
deleted file mode 100644
index 9c28a8c2..00000000
--- a/vipra-cmd/src/main/java/de/vipra/cmd/ex/ClearException.java
+++ /dev/null
@@ -1,15 +0,0 @@
-package de.vipra.cmd.ex;
-
-public class ClearException extends Exception {
-
- private static final long serialVersionUID = 1L;
-
- public ClearException(String msg) {
- super(msg);
- }
-
- public ClearException(Exception e) {
- super(e);
- }
-
-}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/ex/ImportException.java b/vipra-cmd/src/main/java/de/vipra/cmd/ex/ImportException.java
deleted file mode 100644
index d7f6da33..00000000
--- a/vipra-cmd/src/main/java/de/vipra/cmd/ex/ImportException.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package de.vipra.cmd.ex;
-
-public class ImportException extends Exception {
-
- private static final long serialVersionUID = 1L;
-
- private final String id;
-
- public ImportException(String msg, String id) {
- super(msg);
- this.id = id;
- }
-
- public ImportException(Exception e, String id) {
- super(e);
- this.id = id;
- }
-
- public String getId() {
- return id;
- }
-
-}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbLDAAnalyzer.java b/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbLDAAnalyzer.java
index 217f210e..5fe15001 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbLDAAnalyzer.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbLDAAnalyzer.java
@@ -9,8 +9,8 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import de.vipra.cmd.ex.LDAAnalyzerException;
import de.vipra.util.Config;
@@ -31,7 +31,7 @@ import jgibblda.Model;
public class JGibbLDAAnalyzer extends LDAAnalyzer {
- public static final Logger log = LoggerFactory.getLogger(JGibbLDAAnalyzer.class);
+ public static final Logger log = LogManager.getLogger(JGibbLDAAnalyzer.class);
public static final String NAME = "jgibb";
private File dataDir;
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/model/ProcessedArticle.java b/vipra-cmd/src/main/java/de/vipra/cmd/model/ProcessedArticle.java
index 484edc6b..e11dd916 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/model/ProcessedArticle.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/model/ProcessedArticle.java
@@ -1,10 +1,13 @@
package de.vipra.cmd.model;
+import java.util.Date;
+
import org.json.simple.JSONObject;
import org.mongodb.morphia.annotations.Entity;
import org.mongodb.morphia.annotations.Transient;
import de.vipra.cmd.text.ProcessedText;
+import de.vipra.util.an.ElasticIndex;
@SuppressWarnings("serial")
@Entity(value = "articles", noClassnameStored = true)
@@ -21,6 +24,21 @@ public class ProcessedArticle extends de.vipra.util.model.ArticleFull {
this.processedText = processedText;
}
+ @ElasticIndex("title")
+ public String serializeTitle() {
+ return super.getTitle();
+ }
+
+ @ElasticIndex("date")
+ public Date serializeDate() {
+ return super.getDate();
+ }
+
+ @ElasticIndex("text")
+ public String serializeText() {
+ return processedText.getText();
+ }
+
public void fromJSON(JSONObject obj) {
if (obj.containsKey("title"))
setTitle(obj.get("title").toString());
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java
index a71a50a6..5a9e7b54 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java
@@ -4,16 +4,13 @@ import java.io.File;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.bson.types.ObjectId;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import de.vipra.cmd.ExecutionException;
-import de.vipra.cmd.ex.ClearException;
import de.vipra.cmd.model.ProcessedArticle;
import de.vipra.util.Config;
import de.vipra.util.ConsoleUtils;
-import de.vipra.util.ex.ConfigException;
import de.vipra.util.model.Import;
import de.vipra.util.model.TopicFull;
import de.vipra.util.model.Word;
@@ -21,8 +18,8 @@ import de.vipra.util.service.DatabaseService;
public class ClearCommand implements Command {
- public static final Logger log = LoggerFactory.getLogger(ClearCommand.class);
- public static final Logger out = LoggerFactory.getLogger("shellout");
+ public static final Logger log = LogManager.getLogger(ClearCommand.class);
+ public static final Logger out = LogManager.getLogger("shellout");
private boolean defaults;
private Config config;
@@ -35,16 +32,12 @@ public class ClearCommand implements Command {
this.defaults = defaults;
}
- private void clear() throws ClearException, ConfigException {
- try {
- config = Config.getConfig();
- dbArticles = DatabaseService.getDatabaseService(config, ProcessedArticle.class);
- dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class);
- dbWords = DatabaseService.getDatabaseService(config, Word.class);
- dbImports = DatabaseService.getDatabaseService(config, Import.class);
- } catch (Exception e) {
- throw new ClearException(e);
- }
+ private void clear() throws Exception {
+ config = Config.getConfig();
+ dbArticles = DatabaseService.getDatabaseService(config, ProcessedArticle.class);
+ dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class);
+ dbWords = DatabaseService.getDatabaseService(config, Word.class);
+ dbImports = DatabaseService.getDatabaseService(config, Import.class);
out.info("clearing database");
dbArticles.drop();
@@ -52,27 +45,23 @@ public class ClearCommand implements Command {
dbWords.drop();
dbImports.drop();
- out.info("clearing filebase");
- File dataDir = config.getDataDirectory();
- if (dataDir.exists() && dataDir.isDirectory()) {
- try {
+ try {
+ out.info("clearing filebase");
+ File dataDir = config.getDataDirectory();
+ if (dataDir.exists() && dataDir.isDirectory()) {
FileUtils.deleteDirectory(dataDir);
- } catch (IOException e) {
- out.warn("could not delete data directory: " + dataDir.getAbsolutePath());
}
+ } catch (IOException e) {
+ out.warn("could not delete data directory: " + config.getDataDirectory().getAbsolutePath());
}
}
@Override
- public void run() throws ExecutionException {
+ public void run() throws Exception {
if (!defaults)
out.info("to confirm clearing, type 'clear' and press enter");
- try {
- if (defaults || ConsoleUtils.confirm("clear")) {
- clear();
- }
- } catch (ClearException | ConfigException e) {
- throw new ExecutionException(e);
+ if (defaults || ConsoleUtils.confirm("clear")) {
+ clear();
}
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/Command.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/Command.java
index 5ad51101..d794b759 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/Command.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/Command.java
@@ -1,9 +1,7 @@
package de.vipra.cmd.option;
-import de.vipra.cmd.ExecutionException;
-
public interface Command {
- public void run() throws ExecutionException;
+ public void run() throws Exception;
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteCommand.java
deleted file mode 100644
index ff033a49..00000000
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteCommand.java
+++ /dev/null
@@ -1,101 +0,0 @@
-package de.vipra.cmd.option;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.bson.types.ObjectId;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import de.vipra.cmd.ExecutionException;
-import de.vipra.cmd.ex.FilebaseException;
-import de.vipra.cmd.file.Filebase;
-import de.vipra.cmd.model.ProcessedArticle;
-import de.vipra.util.Config;
-import de.vipra.util.MongoUtils;
-import de.vipra.util.ex.ConfigException;
-import de.vipra.util.ex.DatabaseException;
-import de.vipra.util.service.DatabaseService;
-
-public class DeleteCommand implements Command {
-
- public static final Logger log = LoggerFactory.getLogger(DeleteCommand.class);
- public static final Logger out = LoggerFactory.getLogger("shellout");
-
- private ArrayList<String> ids = new ArrayList<>();
- private Config config;
- private DatabaseService<ProcessedArticle, ObjectId> dbArticles;
- private Filebase filebase;
-
- /**
- * Creates a new delete command. Use {@link DeleteCommand#run()} to run this
- * command.
- *
- * @param ids
- * array of ids to be used for deletion.
- */
- public DeleteCommand(String[] ids) {
- addIds(ids);
- }
-
- public void addIds(String[] strings) {
- for (String str : strings) {
- if (str.contains(File.pathSeparator)) {
- String[] parts = str.split(File.pathSeparator);
- str = parts[parts.length - 1];
- }
- ids.add(str);
- }
- }
-
- public void deleteEntry(String id) throws ExecutionException {
- ArrayList<Exception> errors = new ArrayList<>();
-
- try {
- // 1. delete mongodb entry
- dbArticles.deleteSingle(MongoUtils.objectId(id));
- } catch (DatabaseException e) {
- errors.add(e);
- }
-
- try {
- // 2. delete file
- filebase.remove(id);
- } catch (FilebaseException e) {
- errors.add(e);
- }
-
- // 3. delete elasticsearch index entry
- // TODO implement
-
- if (errors.size() > 0) {
- throw new ExecutionException(errors);
- }
- }
-
- @Override
- public void run() throws ExecutionException {
- try {
- config = Config.getConfig();
- dbArticles = DatabaseService.getDatabaseService(config, ProcessedArticle.class);
- filebase = Filebase.getFilebase(config);
- } catch (IOException | FilebaseException | ConfigException e) {
- throw new ExecutionException(e);
- }
-
- List<Exception> ex = new ArrayList<>();
- for (String id : ids) {
- try {
- deleteEntry(id);
- } catch (Exception e) {
- ex.add(e);
- }
- }
- if (ex.size() > 0) {
- throw new ExecutionException(ex);
- }
- }
-
-}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
index 0d88b358..382cc4a0 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
@@ -1,26 +1,23 @@
package de.vipra.cmd.option;
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FilenameFilter;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.bson.types.ObjectId;
+import org.elasticsearch.client.Client;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
-import org.json.simple.parser.ParseException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import de.vipra.cmd.ExecutionException;
-import de.vipra.cmd.ex.ImportException;
+import de.vipra.cmd.es.ESClient;
import de.vipra.cmd.file.Filebase;
import de.vipra.cmd.file.FilebaseIndex;
import de.vipra.cmd.lda.LDAAnalyzer;
@@ -29,6 +26,7 @@ import de.vipra.cmd.text.ProcessedText;
import de.vipra.cmd.text.Processor;
import de.vipra.util.Config;
import de.vipra.util.ConvertStream;
+import de.vipra.util.ElasticSerializer;
import de.vipra.util.MongoUtils;
import de.vipra.util.StringUtils;
import de.vipra.util.Timer;
@@ -45,8 +43,8 @@ import de.vipra.util.service.DatabaseService;
public class ImportCommand implements Command {
- public static final Logger log = LoggerFactory.getLogger(ImportCommand.class);
- public static final Logger out = LoggerFactory.getLogger("shellout");
+ public static final Logger log = LogManager.getLogger(ImportCommand.class);
+ public static final Logger out = LogManager.getLogger("shellout");
private ArrayList<File> files = new ArrayList<>();
private JSONParser parser = new JSONParser();
@@ -59,6 +57,8 @@ public class ImportCommand implements Command {
private Processor preprocessor;
private WordMap wordMap;
private LDAAnalyzer analyzer;
+ private Client elasticClient;
+ private ElasticSerializer<ProcessedArticle> elasticSerializer;
/**
* Import command to import articles into the database, do topic modeling
@@ -105,49 +105,44 @@ public class ImportCommand implements Command {
*
* @param obj
* @return
- * @throws ImportException
+ * @throws Exception
*/
- private Article importArticle(JSONObject obj) throws ImportException {
+ private Article importArticle(JSONObject obj) throws Exception {
out.info("importing \"" + StringUtils.ellipsize(obj.get("title").toString(), 80) + "\"");
ProcessedArticle article = new ProcessedArticle();
article.fromJSON(obj);
- try {
- // preprocess text and generate text statistics
- ProcessedText processedText = preprocessor.preprocess(article.getText());
- ArticleStats articleStats = ArticleStats.generateFromText(processedText.getText(), wordMap);
-
- // add article to mongodb
- article.setProcessedText(processedText);
- article.setStats(articleStats);
- article = dbArticles.createSingle(article);
-
- // add words
- for(String word : processedText.getWords())
- wordMap.add(word);
-
- // add article to filebase
- filebase.add(article);
-
- // return article reference
- return new Article(article.getId());
- } catch (Exception e) {
- throw new ImportException(e, article.getId().toString());
- }
+ // preprocess text and generate text statistics
+ ProcessedText processedText = preprocessor.preprocess(article.getText());
+ ArticleStats articleStats = ArticleStats.generateFromText(processedText.getText(), wordMap);
+
+ // add article to mongodb
+ article.setProcessedText(processedText);
+ article.setStats(articleStats);
+ article = dbArticles.createSingle(article);
+
+ // add words
+ for (String word : processedText.getWords())
+ wordMap.add(word);
+
+ // add article to filebase
+ filebase.add(article);
+
+ // index article
+ Map<String, Object> source = elasticSerializer.serialize(article);
+ elasticClient.prepareIndex("articles", "article", article.getId().toString()).setSource(source).get();
+
+ // return article reference
+ return new Article(article.getId());
}
/**
* Imports a file into the database and the filebase
*
* @param file
- * @throws ParseException
- * @throws IOException
- * @throws FileNotFoundException
- * @throws ImportException
* @throws Exception
*/
- private List<Article> importFile(File file)
- throws FileNotFoundException, IOException, ParseException, ImportException {
+ private List<Article> importFile(File file) throws Exception {
Object data = parser.parse(new FileReader(file));
List<Article> articles = new ArrayList<>();
@@ -162,8 +157,7 @@ public class ImportCommand implements Command {
return articles;
}
- private List<Article> importFiles(List<File> files)
- throws FileNotFoundException, IOException, ParseException, ImportException {
+ private List<Article> importFiles(List<File> files) throws Exception {
List<Article> articles = new ArrayList<>();
for (File file : files) {
articles.addAll(importFile(file));
@@ -172,129 +166,129 @@ public class ImportCommand implements Command {
}
@Override
- public void run() throws ExecutionException {
- try {
- config = Config.getConfig();
- dbArticles = DatabaseService.getDatabaseService(config, ProcessedArticle.class);
- dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class);
- dbWords = DatabaseService.getDatabaseService(config, Word.class);
- dbImports = DatabaseService.getDatabaseService(config, Import.class);
- filebase = Filebase.getFilebase(config);
- preprocessor = Processor.getPreprocessor(config);
- wordMap = new WordMap(dbWords);
- analyzer = LDAAnalyzer.getAnalyzer(config, wordMap);
-
- out.info("using data directory: " + config.getDataDirectory().getAbsolutePath());
- out.info("using preprocessor: " + preprocessor.getName());
- out.info("using analyzer: " + analyzer.getName());
-
- Timer timer = new Timer();
- timer.start();
-
- Import importOp = new Import();
-
- /*
- * import files into database and filebase
- */
- out.info("file import");
- List<Article> importedArticles = importFiles(files);
- importOp.setArticles(importedArticles);
- timer.lap("import");
-
- /*
- * write filebase
- */
- out.info("writing file index");
- filebase.close();
- timer.lap("filebase write");
-
- /*
- * do topic modeling
- */
- out.info("topic modeling");
- analyzer.analyze();
- timer.lap("topic modeling");
-
- /*
- * save topic model
- */
- out.info("saving topic definitions");
- int batchSize = 100;
- ConvertStream<TopicFull> topicDefs = analyzer.getTopicDefinitions();
- Map<String, String> topicIndexMap = new HashMap<>();
- dbTopics.drop();
- List<TopicFull> newTopicDefs = new ArrayList<>(batchSize);
- List<Topic> newTopicRefs = new ArrayList<>();
- Iterator<TopicFull> it = topicDefs.iterator();
- while (it.hasNext()) {
- newTopicDefs.add(it.next());
- if (newTopicDefs.size() == batchSize || !it.hasNext()) {
- dbTopics.createMultiple(newTopicDefs);
- for (TopicFull newTopicDef : newTopicDefs) {
- topicIndexMap.put(Integer.toString(newTopicDef.getIndex()), newTopicDef.getId().toString());
- newTopicRefs.add(new Topic(newTopicDef.getId()));
- }
+ public void run() throws Exception {
+ config = Config.getConfig();
+ dbArticles = DatabaseService.getDatabaseService(config, ProcessedArticle.class);
+ dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class);
+ dbWords = DatabaseService.getDatabaseService(config, Word.class);
+ dbImports = DatabaseService.getDatabaseService(config, Import.class);
+ filebase = Filebase.getFilebase(config);
+ preprocessor = Processor.getPreprocessor(config);
+ wordMap = new WordMap(dbWords);
+ analyzer = LDAAnalyzer.getAnalyzer(config, wordMap);
+ elasticClient = ESClient.getClient(config);
+ elasticSerializer = new ElasticSerializer<>(ProcessedArticle.class);
+
+ out.info("using data directory: " + config.getDataDirectory().getAbsolutePath());
+ out.info("using preprocessor: " + preprocessor.getName());
+ out.info("using analyzer: " + analyzer.getName());
+
+ Timer timer = new Timer();
+ timer.start();
+
+ Import importOp = new Import();
+
+ /*
+ * import files into database and filebase
+ */
+ out.info("file import");
+ List<Article> importedArticles = importFiles(files);
+ importOp.setArticles(importedArticles);
+ timer.lap("import");
+
+ /*
+ * write filebase
+ */
+ out.info("writing file index");
+ filebase.close();
+ timer.lap("filebase write");
+
+ /*
+ * do topic modeling
+ */
+ out.info("topic modeling");
+ analyzer.analyze();
+ timer.lap("topic modeling");
+
+ /*
+ * save topic model
+ */
+ out.info("saving topic definitions");
+ int batchSize = 100;
+ ConvertStream<TopicFull> topicDefs = analyzer.getTopicDefinitions();
+ Map<String, String> topicIndexMap = new HashMap<>();
+ dbTopics.drop();
+ List<TopicFull> newTopicDefs = new ArrayList<>(batchSize);
+ List<Topic> newTopicRefs = new ArrayList<>();
+ Iterator<TopicFull> it = topicDefs.iterator();
+ while (it.hasNext()) {
+ newTopicDefs.add(it.next());
+ if (newTopicDefs.size() == batchSize || !it.hasNext()) {
+ dbTopics.createMultiple(newTopicDefs);
+ for (TopicFull newTopicDef : newTopicDefs) {
+ topicIndexMap.put(Integer.toString(newTopicDef.getIndex()), newTopicDef.getId().toString());
+ newTopicRefs.add(new Topic(newTopicDef.getId()));
}
}
- importOp.setTopics(newTopicRefs);
- timer.lap("saving topics");
-
- /*
- * save topic refs
- */
- out.info("saving document topics");
- ConvertStream<List<TopicRef>> topics = analyzer.getTopics();
- FilebaseIndex index = filebase.getIndex();
- Iterator<String> indexIter = index.iterator();
- Iterator<List<TopicRef>> topicIter = topics.iterator();
- while (indexIter.hasNext() && topicIter.hasNext()) {
- List<TopicRef> topicCount = topicIter.next();
- for (TopicRef tc : topicCount) {
- String oid = topicIndexMap.get(tc.getTopicId());
- tc.setTopicId(oid);
- if (oid == null)
- log.error("no object id for topic index " + tc.getTopicId());
- }
- String id = indexIter.next();
- ProcessedArticle a = dbArticles.getSingle(MongoUtils.objectId(id));
- if (a != null)
- a.setTopics(topicCount);
- else
- log.error("no article found in db for id " + id);
- try {
- dbArticles.updateSingle(a);
- } catch (DatabaseException e) {
- log.error("could not update article: " + a.getTitle() + " (" + a.getId() + ")");
- }
+ }
+ importOp.setTopics(newTopicRefs);
+ timer.lap("saving topics");
+
+ /*
+ * save topic refs
+ */
+ out.info("saving document topics");
+ ConvertStream<List<TopicRef>> topics = analyzer.getTopics();
+ FilebaseIndex index = filebase.getIndex();
+ Iterator<String> indexIter = index.iterator();
+ Iterator<List<TopicRef>> topicIter = topics.iterator();
+ while (indexIter.hasNext() && topicIter.hasNext()) {
+ List<TopicRef> topicCount = topicIter.next();
+ for (TopicRef tc : topicCount) {
+ String oid = topicIndexMap.get(tc.getTopicId());
+ tc.setTopicId(oid);
+ if (oid == null)
+ log.error("no object id for topic index " + tc.getTopicId());
+ }
+ String id = indexIter.next();
+ ProcessedArticle a = dbArticles.getSingle(MongoUtils.objectId(id));
+ if (a != null)
+ a.setTopics(topicCount);
+ else
+ log.error("no article found in db for id " + id);
+ try {
+ dbArticles.updateSingle(a);
+ } catch (DatabaseException e) {
+ log.error("could not update article: " + a.getTitle() + " (" + a.getId() + ")");
}
- List<Word> importedWords = wordMap.getNewWords();
- importOp.setWords(importedWords);
- timer.lap("saving topic refs");
-
- /*
- * save words
- */
- out.info("saving words");
- wordMap.create();
- timer.lap("saving words");
-
- /*
- * save import information
- */
- importOp.setDuration(timer.total());
- dbImports.createSingle(importOp);
-
- /*
- * run information
- */
- int newArticlesCount = importedArticles.size();
- int newWordsCount = importedWords.size();
- out.info("imported " + newArticlesCount + " new " + StringUtils.quantity(newArticlesCount, "article"));
- out.info("imported " + newWordsCount + " new " + StringUtils.quantity(newWordsCount, "word"));
- out.info(timer.toString());
- } catch (Exception e) {
- throw new ExecutionException(e);
}
+ List<Word> importedWords = wordMap.getNewWords();
+ importOp.setWords(importedWords);
+ timer.lap("saving topic refs");
+
+ /*
+ * save words
+ */
+ out.info("saving words");
+ wordMap.create();
+ timer.lap("saving words");
+
+ /*
+ * save import information
+ */
+ importOp.setDuration(timer.total());
+ dbImports.createSingle(importOp);
+
+ elasticClient.close();
+
+ /*
+ * run information
+ */
+ int newArticlesCount = importedArticles.size();
+ int newWordsCount = importedWords.size();
+ out.info("imported " + newArticlesCount + " new " + StringUtils.quantity(newArticlesCount, "article"));
+ out.info("imported " + newWordsCount + " new " + StringUtils.quantity(newWordsCount, "word"));
+ out.info(timer.toString());
}
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/StatsCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/StatsCommand.java
index d8ed55cf..b6e59868 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/option/StatsCommand.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/StatsCommand.java
@@ -1,25 +1,21 @@
package de.vipra.cmd.option;
import java.io.File;
-import java.io.IOException;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.bson.types.ObjectId;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import de.vipra.cmd.ExecutionException;
-import de.vipra.cmd.ex.FilebaseException;
import de.vipra.cmd.file.Filebase;
import de.vipra.util.Config;
import de.vipra.util.StringUtils;
-import de.vipra.util.ex.ConfigException;
import de.vipra.util.model.TopicFull;
import de.vipra.util.service.DatabaseService;
public class StatsCommand implements Command {
- public static final Logger log = LoggerFactory.getLogger(StatsCommand.class);
- public static final Logger out = LoggerFactory.getLogger("shellout");
+ public static final Logger log = LogManager.getLogger(StatsCommand.class);
+ public static final Logger out = LogManager.getLogger("shellout");
private Config config;
private Filebase filebase;
@@ -34,16 +30,12 @@ public class StatsCommand implements Command {
}
@Override
- public void run() throws ExecutionException {
- try {
- config = Config.getConfig();
- filebase = Filebase.getFilebase(config);
- dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class);
-
- stats();
- } catch (IOException | ConfigException | FilebaseException e) {
- throw new ExecutionException(e);
- }
+ public void run() throws Exception {
+ config = Config.getConfig();
+ filebase = Filebase.getFilebase(config);
+ dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class);
+
+ stats();
}
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/TestCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/TestCommand.java
new file mode 100644
index 00000000..9441138e
--- /dev/null
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/TestCommand.java
@@ -0,0 +1,39 @@
+package de.vipra.cmd.option;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.bson.types.ObjectId;
+import org.elasticsearch.client.transport.NoNodeAvailableException;
+import org.elasticsearch.client.transport.TransportClient;
+
+import de.vipra.cmd.es.ESClient;
+import de.vipra.util.Config;
+import de.vipra.util.model.Article;
+import de.vipra.util.service.DatabaseService;
+
+public class TestCommand implements Command {
+
+ public static final Logger out = LogManager.getLogger("shellout");
+
+ @Override
+ public void run() throws Exception {
+ // test if configuration readable
+ out.info("reading configuration...");
+ Config config = Config.getConfig();
+
+ // test if database is accessible
+ out.info("testing mongodb connection...");
+ DatabaseService<Article, ObjectId> dbArticles = DatabaseService.getDatabaseService(config, Article.class);
+ dbArticles.count();
+
+ // test if elasticsearch is accessible
+ out.info("testing elasticsearch connection...");
+ TransportClient esclient = ESClient.getClient(config);
+ if (esclient.connectedNodes().isEmpty()) {
+ throw new NoNodeAvailableException("no elasticsearch nodes available");
+ }
+
+ out.info("all tests passed");
+ }
+
+}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/text/CustomProcessor.java b/vipra-cmd/src/main/java/de/vipra/cmd/text/CustomProcessor.java
deleted file mode 100644
index f2a135c9..00000000
--- a/vipra-cmd/src/main/java/de/vipra/cmd/text/CustomProcessor.java
+++ /dev/null
@@ -1,36 +0,0 @@
-package de.vipra.cmd.text;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-public class CustomProcessor extends Processor {
-
- private final Set<String> stopWords;
-
- public CustomProcessor(List<String> stopWordsList) {
- super("Custom Processor");
- this.stopWords = new HashSet<>(stopWordsList);
- }
-
- private String removeStopWords(String text) {
- String[] words = text.split("\\s+");
- StringBuilder sb = new StringBuilder();
- for (String word : words) {
- if (stopWords.contains(word)) {
- continue;
- }
- sb.append(word).append(" ");
- }
- return sb.toString().trim();
- }
-
- @Override
- public ProcessedText preprocess(String input) {
- input = input.toLowerCase();
- input = removeStopWords(input);
- input = clean(input);
- return new ProcessedText(input);
- }
-
-}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/text/LuceneProcessor.java b/vipra-cmd/src/main/java/de/vipra/cmd/text/LuceneProcessor.java
deleted file mode 100644
index b8c9fbae..00000000
--- a/vipra-cmd/src/main/java/de/vipra/cmd/text/LuceneProcessor.java
+++ /dev/null
@@ -1,59 +0,0 @@
-package de.vipra.cmd.text;
-
-import java.io.IOException;
-import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.regex.Pattern;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.en.PorterStemFilter;
-import org.apache.lucene.analysis.miscellaneous.TrimFilter;
-import org.apache.lucene.analysis.pattern.PatternReplaceFilter;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.util.CharArraySet;
-
-import de.vipra.cmd.ex.PreprocessorException;
-import de.vipra.util.Constants;
-import de.vipra.util.StringUtils;
-
-public class LuceneProcessor extends Processor {
-
- private final CharArraySet stopWords;
-
- public LuceneProcessor(List<String> stopWords) {
- super("Lucene Processor");
- this.stopWords = new CharArraySet(stopWords, false);
- }
-
- @Override
- public ProcessedText preprocess(String input) throws PreprocessorException {
- Analyzer analyzer = new StandardAnalyzer(stopWords);
- TokenStream stream = analyzer.tokenStream(null, new StringReader(input));
- try {
- stream.reset();
- stream = new PorterStemFilter(stream);
- stream = new TrimFilter(stream);
- stream = new PatternReplaceFilter(stream, Pattern.compile(Constants.REGEX_EMAIL), "", true);
- stream = new PatternReplaceFilter(stream, Pattern.compile(Constants.REGEX_URL), "", true);
- stream = new PatternReplaceFilter(stream, Pattern.compile(Constants.REGEX_NUMBER), "", true);
- stream = new PatternReplaceFilter(stream, Pattern.compile(Constants.CHARS_DISALLOWED), "", true);
- stream = new PatternReplaceFilter(stream, Pattern.compile("\\s+"), " ", true);
- ArrayList<String> result = new ArrayList<>();
- while (stream.incrementToken()) {
- result.add(stream.getAttribute(CharTermAttribute.class).toString());
- }
- return new ProcessedText(StringUtils.join(result));
- } catch (IOException e) {
- throw new PreprocessorException(e);
- } finally {
- try {
- stream.close();
- } catch (IOException e) {}
- analyzer.close();
- }
- }
-
-}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/text/Processor.java b/vipra-cmd/src/main/java/de/vipra/cmd/text/Processor.java
index 0d11b2af..a74c32e1 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/text/Processor.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/text/Processor.java
@@ -25,12 +25,8 @@ public abstract class Processor {
List<String> stopWords = Constants.STOPWORDS;
switch (Constants.Processor.fromString(config.getString(Key.PREPROCESSOR))) {
- case CUSTOM:
- return new CustomProcessor(stopWords);
case CORENLP:
return new CoreNLPProcessor(stopWords);
- case LUCENE:
- return new LuceneProcessor(stopWords);
default:
return null;
}
diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/text/StopwordsAnnotator.java b/vipra-cmd/src/main/java/de/vipra/cmd/text/StopwordsAnnotator.java
index 838bbf03..d2701fee 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/text/StopwordsAnnotator.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/text/StopwordsAnnotator.java
@@ -48,4 +48,4 @@ public class StopwordsAnnotator implements Annotator, CoreAnnotation<Boolean> {
return Boolean.class;
}
-}
+}
\ No newline at end of file
diff --git a/vipra-cmd/src/main/resources/log4j2dev.xml b/vipra-cmd/src/main/resources/log4j2dev.xml
index bb9050cf..58ac9659 100644
--- a/vipra-cmd/src/main/resources/log4j2dev.xml
+++ b/vipra-cmd/src/main/resources/log4j2dev.xml
@@ -11,5 +11,6 @@
</Root>
<Logger name="shellout" level="ALL"/>
<Logger name="org.mongodb" level="ERROR"/>
+ <Logger name="org.elasticsearch.transport.netty" level="ERROR"/>
</Loggers>
</Configuration>
\ No newline at end of file
diff --git a/vipra-rest/.classpath b/vipra-rest/.classpath
index f2db8ed9..b12640c5 100644
--- a/vipra-rest/.classpath
+++ b/vipra-rest/.classpath
@@ -19,7 +19,13 @@
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<attributes>
- <attribute name="owner.project.facets" value="java"/>
+ <attribute name="maven.pomderived" value="true"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry kind="src" output="target/test-classes" path="src/test/java">
+ <attributes>
+ <attribute name="optional" value="true"/>
+ <attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
diff --git a/vipra-rest/.project b/vipra-rest/.project
index 569e5873..7ff77ec0 100644
--- a/vipra-rest/.project
+++ b/vipra-rest/.project
@@ -27,12 +27,12 @@
</arguments>
</buildCommand>
<buildCommand>
- <name>org.eclipse.m2e.core.maven2Builder</name>
+ <name>net.sourceforge.metrics.builder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
- <name>net.sourceforge.metrics.builder</name>
+ <name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
diff --git a/vipra-rest/src/main/java/de/vipra/rest/provider/CORSResponseFilter.java b/vipra-rest/src/main/java/de/vipra/rest/provider/CORSResponseFilter.java
index 93c6084b..8e65b962 100644
--- a/vipra-rest/src/main/java/de/vipra/rest/provider/CORSResponseFilter.java
+++ b/vipra-rest/src/main/java/de/vipra/rest/provider/CORSResponseFilter.java
@@ -7,13 +7,13 @@ import javax.ws.rs.container.ContainerResponseContext;
import javax.ws.rs.container.ContainerResponseFilter;
import javax.ws.rs.ext.Provider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
@Provider
public class CORSResponseFilter implements ContainerResponseFilter {
- public static final Logger log = LoggerFactory.getLogger(CORSResponseFilter.class);
+ public static final Logger log = LogManager.getLogger(CORSResponseFilter.class);
public CORSResponseFilter() {
log.info("cors filter registered");
diff --git a/vipra-rest/src/main/java/de/vipra/rest/provider/InitializationListener.java b/vipra-rest/src/main/java/de/vipra/rest/provider/InitializationListener.java
index 6555abaf..3c82ec12 100644
--- a/vipra-rest/src/main/java/de/vipra/rest/provider/InitializationListener.java
+++ b/vipra-rest/src/main/java/de/vipra/rest/provider/InitializationListener.java
@@ -4,14 +4,14 @@ import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.ehcache.CacheManager;
import org.ehcache.CacheManagerBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public class InitializationListener implements ServletContextListener {
- public static final Logger log = LoggerFactory.getLogger(InitializationListener.class);
+ public static final Logger log = LogManager.getLogger(InitializationListener.class);
@Override
public void contextDestroyed(ServletContextEvent sce) {
diff --git a/vipra-rest/src/main/java/de/vipra/rest/provider/ObjectMapperProvider.java b/vipra-rest/src/main/java/de/vipra/rest/provider/ObjectMapperProvider.java
index fcab9e67..9fcbcc76 100644
--- a/vipra-rest/src/main/java/de/vipra/rest/provider/ObjectMapperProvider.java
+++ b/vipra-rest/src/main/java/de/vipra/rest/provider/ObjectMapperProvider.java
@@ -5,9 +5,9 @@ import java.text.SimpleDateFormat;
import javax.ws.rs.ext.ContextResolver;
import javax.ws.rs.ext.Provider;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.bson.types.ObjectId;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -27,7 +27,7 @@ import de.vipra.util.model.Word;
@Provider
public class ObjectMapperProvider implements ContextResolver<ObjectMapper> {
- public static final Logger log = LoggerFactory.getLogger(ObjectMapperProvider.class);
+ public static final Logger log = LogManager.getLogger(ObjectMapperProvider.class);
final ObjectMapper defaultObjectMapper;
public ObjectMapperProvider() {
diff --git a/vipra-rest/src/main/java/de/vipra/rest/resource/ArticleResource.java b/vipra-rest/src/main/java/de/vipra/rest/resource/ArticleResource.java
index 3c9e531a..e4f8a2b8 100644
--- a/vipra-rest/src/main/java/de/vipra/rest/resource/ArticleResource.java
+++ b/vipra-rest/src/main/java/de/vipra/rest/resource/ArticleResource.java
@@ -55,8 +55,8 @@ public class ArticleResource {
CacheManager manager = (CacheManager) servletContext.getAttribute("cachemanager");
Cache<String, ArticleFull> articleCache = manager.getCache("articlecache", String.class, ArticleFull.class);
if (articleCache == null)
- articleCache = manager.createCache("articlecache",
- CacheConfigurationBuilder.newCacheConfigurationBuilder().buildConfig(String.class, ArticleFull.class));
+ articleCache = manager.createCache("articlecache", CacheConfigurationBuilder.newCacheConfigurationBuilder()
+ .buildConfig(String.class, ArticleFull.class));
this.cache = articleCache;
}
@@ -74,12 +74,12 @@ public class ArticleResource {
try {
List<ArticleFull> articles = service.getMultiple(skip, limit, sortBy, StringUtils.getFields(fields));
-
- if((skip != null && skip > 0) || (limit != null && limit > 0))
+
+ if ((skip != null && skip > 0) || (limit != null && limit > 0))
res.addMeta("total", service.count());
else
res.addMeta("total", articles.size());
-
+
return res.ok(articles);
} catch (Exception e) {
res.addError(new APIError(Response.Status.BAD_REQUEST, "Error", e.getMessage()));
diff --git a/vipra-rest/src/main/java/de/vipra/rest/resource/TopicResource.java b/vipra-rest/src/main/java/de/vipra/rest/resource/TopicResource.java
index 691dc746..2e65bf39 100644
--- a/vipra-rest/src/main/java/de/vipra/rest/resource/TopicResource.java
+++ b/vipra-rest/src/main/java/de/vipra/rest/resource/TopicResource.java
@@ -68,12 +68,12 @@ public class TopicResource {
try {
List<TopicFull> topics = service.getMultiple(skip, limit, sortBy, StringUtils.getFields(fields));
-
- if((skip != null && skip > 0) || (limit != null && limit > 0))
+
+ if ((skip != null && skip > 0) || (limit != null && limit > 0))
res.addMeta("total", service.count());
else
res.addMeta("total", topics.size());
-
+
return res.ok(topics);
} catch (Exception e) {
res.addError(new APIError(Response.Status.BAD_REQUEST, "Error", e.getMessage()));
diff --git a/vipra-util/.classpath b/vipra-util/.classpath
index cf6dc857..332896b5 100644
--- a/vipra-util/.classpath
+++ b/vipra-util/.classpath
@@ -20,7 +20,7 @@
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<attributes>
- <attribute name="owner.project.facets" value="java"/>
+ <attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
diff --git a/vipra-util/.project b/vipra-util/.project
index 8f910748..ff1b8e15 100644
--- a/vipra-util/.project
+++ b/vipra-util/.project
@@ -21,12 +21,12 @@
</arguments>
</buildCommand>
<buildCommand>
- <name>org.eclipse.m2e.core.maven2Builder</name>
+ <name>net.sourceforge.metrics.builder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
- <name>net.sourceforge.metrics.builder</name>
+ <name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
diff --git a/vipra-util/pom.xml b/vipra-util/pom.xml
index 78096310..d7682c64 100644
--- a/vipra-util/pom.xml
+++ b/vipra-util/pom.xml
@@ -10,8 +10,6 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
- <log4jVersion>2.4.1</log4jVersion>
- <jacksonVersion>2.7.0</jacksonVersion>
</properties>
<dependencies>
@@ -22,57 +20,35 @@
<version>2.4</version>
</dependency>
- <!-- Logging -->
+ <!-- SLF4j logging -->
<dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- <version>${log4jVersion}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- <version>${log4jVersion}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
- <version>${log4jVersion}</version>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.14</version>
</dependency>
<!-- MongoDB Database Adapter -->
- <dependency>
- <groupId>org.mongodb</groupId>
- <artifactId>mongodb-driver</artifactId>
- <version>3.2.0</version>
- </dependency>
<dependency>
<groupId>org.mongodb.morphia</groupId>
<artifactId>morphia</artifactId>
- <version>1.0.1</version>
+ <version>1.1.0</version>
</dependency>
<dependency>
<groupId>org.mongodb.morphia</groupId>
<artifactId>morphia-logging-slf4j</artifactId>
- <version>1.0.1</version>
- </dependency>
-
- <!-- ElasticSearch Adapter -->
- <dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- <version>2.1.0</version>
+ <version>1.1.0</version>
</dependency>
<!-- Jackson -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
- <version>${jacksonVersion}</version>
+ <version>2.7.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
- <version>${jacksonVersion}</version>
+ <version>2.7.0</version>
</dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/vipra-util/src/main/java/de/vipra/util/Config.java b/vipra-util/src/main/java/de/vipra/util/Config.java
index 134f5668..8bd41a0c 100644
--- a/vipra-util/src/main/java/de/vipra/util/Config.java
+++ b/vipra-util/src/main/java/de/vipra/util/Config.java
@@ -9,13 +9,15 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import de.vipra.util.ex.ConfigException;
+import de.vipra.util.model.Model;
+import de.vipra.util.service.DatabaseService;
public class Config {
public static enum Key {
- DBHOST("db.host", Constants.DEFAULT_HOST),
- DBPORT("db.port", Constants.DEFAULT_PORT),
- DBNAME("db.name", Constants.DEFAULT_DB),
+ DBHOST("db.host", Constants.DB_HOST),
+ DBPORT("db.port", Constants.DB_PORT),
+ DBNAME("db.name", Constants.DB_NAME),
DATADIR("fs.datadir", null),
PREPROCESSOR("an.preprocessor", Constants.Processor.DEFAULT.name),
ANALYZER("an.analyzer", Constants.Analyzer.DEFAULT.name),
@@ -50,7 +52,7 @@ public class Config {
// load config from generic config dir
File configDir = getGenericConfigDir();
if (configDir != null && configDir.exists() && configDir.isDirectory()) {
- File file = new File(configDir, "config.properties");
+ File file = new File(configDir, Constants.CONFIG_FILE);
if (file.exists() && file.isFile()) {
in = FileUtils.openInputStream(file);
}
@@ -60,7 +62,6 @@ public class Config {
if (in == null) {
in = FileUtils.getResource(Constants.CONFIG_FILE);
}
-
load(in);
}
@@ -103,6 +104,15 @@ public class Config {
return dataDir;
}
+ public Mongo getMongo() throws ConfigException {
+ return Mongo.getInstance(this);
+ }
+
+ public <Type extends Model<IdType>, IdType> DatabaseService<Type, IdType> getDatabaseService(Class<Type> clazz)
+ throws ConfigException {
+ return DatabaseService.getDatabaseService(this, clazz);
+ }
+
public static File getGenericDataDir() {
File base = PathUtils.appDataDir();
return new File(base, Constants.FB_DIR);
diff --git a/vipra-util/src/main/java/de/vipra/util/Constants.java b/vipra-util/src/main/java/de/vipra/util/Constants.java
index 870b3a85..b4440ea1 100644
--- a/vipra-util/src/main/java/de/vipra/util/Constants.java
+++ b/vipra-util/src/main/java/de/vipra/util/Constants.java
@@ -26,9 +26,15 @@ public class Constants {
* DATABASE
*/
- public static final String DEFAULT_HOST = "localhost";
- public static final int DEFAULT_PORT = 27017;
- public static final String DEFAULT_DB = "test";
+ public static final String DB_HOST = "localhost";
+ public static final int DB_PORT = 27017;
+ public static final String DB_NAME = "test";
+
+ /*
+ * ELASTICSEARCH
+ */
+ public static final String ES_HOST = "127.0.0.1";
+ public static final int ES_PORT = 9300;
/*
* TOPIC MODELING
@@ -177,9 +183,7 @@ public class Constants {
* The text processors available, including the default text processor
*/
public static enum Processor {
- CUSTOM("custom"),
CORENLP("corenlp"),
- LUCENE("lucene"),
DEFAULT(CORENLP);
public final String name;
diff --git a/vipra-util/src/main/java/de/vipra/util/ElasticSerializer.java b/vipra-util/src/main/java/de/vipra/util/ElasticSerializer.java
new file mode 100644
index 00000000..01eec7f9
--- /dev/null
+++ b/vipra-util/src/main/java/de/vipra/util/ElasticSerializer.java
@@ -0,0 +1,72 @@
+package de.vipra.util;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import de.vipra.util.an.ElasticIndex;
+
+public class ElasticSerializer<T> {
+
+ private Set<Entry<String, Field>> fields;
+ private Set<Entry<String, Method>> methods;
+
+ public ElasticSerializer(Class<T> clazz) {
+ Map<String, Field> foundFields = new HashMap<>();
+ Map<String, Method> foundMethods = new HashMap<>();
+
+ for (Field field : clazz.getDeclaredFields()) {
+ if (!Modifier.isStatic(field.getModifiers())) {
+ if (!field.isAccessible())
+ field.setAccessible(true);
+
+ ElasticIndex ei = field.getDeclaredAnnotation(ElasticIndex.class);
+ if (ei == null)
+ continue;
+
+ foundFields.put(ei.value(), field);
+ }
+ }
+
+ for (Method method : clazz.getDeclaredMethods()) {
+ if (!Modifier.isStatic(method.getModifiers()) && method.getParameterCount() == 0) {
+ if (!method.isAccessible())
+ method.setAccessible(true);
+
+ ElasticIndex ei = method.getDeclaredAnnotation(ElasticIndex.class);
+ if (ei == null)
+ continue;
+
+ foundMethods.put(ei.value(), method);
+ }
+ }
+
+ this.fields = foundFields.entrySet();
+ this.methods = foundMethods.entrySet();
+ }
+
+ public Map<String, Object> serialize(T t) {
+ Map<String, Object> values = new HashMap<>();
+ for (Entry<String, Field> e : fields) {
+ try {
+ values.put(e.getKey(), e.getValue().get(t));
+ } catch (IllegalArgumentException | IllegalAccessException e1) {
+ throw new RuntimeException("could not serialize field value " + e.getKey(), e1);
+ }
+ }
+ for (Entry<String, Method> m : methods) {
+ try {
+ values.put(m.getKey(), m.getValue().invoke(t));
+ } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e1) {
+ throw new RuntimeException("could not serialize method value " + m.getKey(), e1);
+ }
+ }
+ return values;
+ }
+
+}
diff --git a/vipra-util/src/main/java/de/vipra/util/Mongo.java b/vipra-util/src/main/java/de/vipra/util/Mongo.java
index c56f6a29..b38c6164 100644
--- a/vipra-util/src/main/java/de/vipra/util/Mongo.java
+++ b/vipra-util/src/main/java/de/vipra/util/Mongo.java
@@ -8,7 +8,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.mongodb.MongoClient;
-import com.mongodb.client.MongoDatabase;
+import com.mongodb.MongoClientOptions;
import de.vipra.util.Config.Key;
import de.vipra.util.ex.ConfigException;
@@ -24,7 +24,6 @@ public class Mongo {
private static Mongo instance;
private final MongoClient client;
- private final MongoDatabase database;
private final Morphia morphia;
private final Datastore datastore;
@@ -38,8 +37,9 @@ public class Mongo {
throw new ConfigException("host/port/dbname missing in configuration");
}
- client = new MongoClient(host, port);
- database = client.getDatabase(databaseName);
+ MongoClientOptions options = MongoClientOptions.builder().connectTimeout(10000).build();
+
+ client = new MongoClient(host + ":" + port, options);
morphia = new Morphia();
morphia.mapPackage("de.vipra.util.model");
@@ -50,10 +50,6 @@ public class Mongo {
return client;
}
- public MongoDatabase getDatabase() {
- return database;
- }
-
public Morphia getMorphia() {
return morphia;
}
diff --git a/vipra-util/src/main/java/de/vipra/util/an/ElasticIndex.java b/vipra-util/src/main/java/de/vipra/util/an/ElasticIndex.java
new file mode 100644
index 00000000..b1d88bc6
--- /dev/null
+++ b/vipra-util/src/main/java/de/vipra/util/an/ElasticIndex.java
@@ -0,0 +1,14 @@
+package de.vipra.util.an;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ ElementType.FIELD, ElementType.METHOD })
+public @interface ElasticIndex {
+
+ public String value() default "";
+
+}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/Topic.java b/vipra-util/src/main/java/de/vipra/util/model/Topic.java
index 894ba962..8678b468 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/Topic.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/Topic.java
@@ -14,7 +14,7 @@ import de.vipra.util.an.JsonType;
@SuppressWarnings("serial")
@JsonType("topic")
@Entity(value = "topics", noClassnameStored = true)
-@Indexes({@Index("name"), @Index("-created")})
+@Indexes({ @Index("name"), @Index("-created") })
public class Topic implements Model<ObjectId>, Serializable {
@Id
diff --git a/vipra-util/src/main/java/de/vipra/util/service/DatabaseService.java b/vipra-util/src/main/java/de/vipra/util/service/DatabaseService.java
index ab2b058b..d68e1bcc 100644
--- a/vipra-util/src/main/java/de/vipra/util/service/DatabaseService.java
+++ b/vipra-util/src/main/java/de/vipra/util/service/DatabaseService.java
@@ -18,14 +18,14 @@ import de.vipra.util.ex.ConfigException;
import de.vipra.util.ex.DatabaseException;
import de.vipra.util.model.Model;
-public class DatabaseService<T extends Model<?>, U> implements Service<T, U, DatabaseException> {
+public class DatabaseService<Type extends Model<IdType>, IdType> implements Service<Type, IdType, DatabaseException> {
private final Datastore datastore;
- private final Class<T> clazz;
+ private final Class<Type> clazz;
private final String[] ignoredFieldsSingle;
private final String[] ignoredFieldsMulti;
- public DatabaseService(Mongo mongo, Class<T> clazz) {
+ public DatabaseService(Mongo mongo, Class<Type> clazz) {
this.datastore = mongo.getDatastore();
this.clazz = clazz;
@@ -46,8 +46,8 @@ public class DatabaseService<T extends Model<?>, U> implements Service<T, U, Dat
}
@Override
- public T getSingle(U id, String... fields) {
- Query<T> q = datastore.createQuery(clazz).field("_id").equal(id);
+ public Type getSingle(IdType id, String... fields) {
+ Query<Type> q = datastore.createQuery(clazz).field("_id").equal(id);
if (fields != null && fields.length > 0)
q.retrievedFields(true, setMinus(fields, ignoredFieldsSingle));
else if (ignoredFieldsSingle.length > 0)
@@ -56,13 +56,13 @@ public class DatabaseService<T extends Model<?>, U> implements Service<T, U, Dat
}
@Override
- public List<T> getMultiple(Integer skip, Integer limit, String sortBy, String... fields) {
+ public List<Type> getMultiple(Integer skip, Integer limit, String sortBy, String... fields) {
return getMultiple(skip, limit, sortBy, true, fields);
}
@Override
- public List<T> getMultiple(Integer skip, Integer limit, String sortBy, boolean defaultIgnore, String... fields) {
- Query<T> q = datastore.createQuery(clazz);
+ public List<Type> getMultiple(Integer skip, Integer limit, String sortBy, boolean defaultIgnore, String... fields) {
+ Query<Type> q = datastore.createQuery(clazz);
if (skip != null && skip > 0)
q.offset(skip);
if (limit != null && limit > 0)
@@ -73,35 +73,35 @@ public class DatabaseService<T extends Model<?>, U> implements Service<T, U, Dat
q.retrievedFields(true, setMinus(fields, ignoredFieldsMulti));
else if (defaultIgnore && ignoredFieldsMulti.length > 0)
q.retrievedFields(false, ignoredFieldsMulti);
- List<T> list = q.asList();
+ List<Type> list = q.asList();
return list;
}
@Override
- public List<T> getAll(String... fields) {
+ public List<Type> getAll(String... fields) {
return getMultiple(null, null, null, fields);
}
@Override
- public T createSingle(T t) throws DatabaseException {
+ public Type createSingle(Type t) throws DatabaseException {
datastore.save(t);
return t;
}
@Override
- public List<T> createMultiple(Iterable<T> t) throws DatabaseException {
- List<T> list = ListUtils.toList(t);
+ public List<Type> createMultiple(Iterable<Type> t) throws DatabaseException {
+ List<Type> list = ListUtils.toList(t);
datastore.save(list);
return list;
}
@Override
- public long deleteSingle(U id) throws DatabaseException {
+ public long deleteSingle(IdType id) throws DatabaseException {
return datastore.delete(id).getN();
}
@Override
- public void updateSingle(T t) throws DatabaseException {
+ public void updateSingle(Type t) throws DatabaseException {
datastore.save(t);
}
@@ -115,10 +115,10 @@ public class DatabaseService<T extends Model<?>, U> implements Service<T, U, Dat
return datastore.getCount(clazz);
}
- public static <T extends Model<?>, U> DatabaseService<T, U> getDatabaseService(Config config, Class<T> clazz)
- throws ConfigException {
- Mongo mongo = Mongo.getInstance(config);
- return new DatabaseService<T, U>(mongo, clazz);
+ public static <Type extends Model<IdType>, IdType> DatabaseService<Type, IdType> getDatabaseService(Config config,
+ Class<Type> clazz) throws ConfigException {
+ Mongo mongo = config.getMongo();
+ return new DatabaseService<Type, IdType>(mongo, clazz);
}
private String[] setMinus(String[] a, String[] b) {
diff --git a/vipra-util/src/main/java/de/vipra/util/service/Service.java b/vipra-util/src/main/java/de/vipra/util/service/Service.java
index 95f6a6d0..3282b172 100644
--- a/vipra-util/src/main/java/de/vipra/util/service/Service.java
+++ b/vipra-util/src/main/java/de/vipra/util/service/Service.java
@@ -4,7 +4,7 @@ import java.util.List;
import de.vipra.util.model.Model;
-public interface Service<Type extends Model<?>, IdType, E extends Exception> {
+public interface Service<Type extends Model<IdType>, IdType, E extends Exception> {
Type getSingle(IdType id, String... fields) throws E;
diff --git a/vipra-util/vipra-util.jardesc b/vipra-util/vipra-util.jardesc
deleted file mode 100644
index f3d66587..00000000
--- a/vipra-util/vipra-util.jardesc
+++ /dev/null
@@ -1,24 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<jardesc>
- <jar path="vipra-util/bin/vipra-util.jar"/>
- <options buildIfNeeded="true" compress="true" descriptionLocation="/vipra-util/vipra-util.jardesc" exportErrors="true" exportWarnings="true" includeDirectoryEntries="false" overwrite="true" saveDescription="true" storeRefactorings="false" useSourceFolders="false"/>
- <storedRefactorings deprecationInfo="true" structuralOnly="false"/>
- <selectedProjects/>
- <manifest generateManifest="true" manifestLocation="" manifestVersion="1.0" reuseManifest="false" saveManifest="false" usesManifest="true">
- <sealing sealJar="false">
- <packagesToSeal/>
- <packagesToUnSeal/>
- </sealing>
- </manifest>
- <selectedElements exportClassFiles="true" exportJavaFiles="false" exportOutputFolder="false">
- <folder path="/vipra-util/src"/>
- <file path="/vipra-util/.classpath"/>
- <file path="/vipra-util/.gitignore"/>
- <folder path="/vipra-util/target"/>
- <file path="/vipra-util/.project"/>
- <folder path="/vipra-util/bin"/>
- <javaElement handleIdentifier="=vipra-util/src\/main\/resources"/>
- <javaElement handleIdentifier="=vipra-util/src\/main\/java"/>
- <file path="/vipra-util/pom.xml"/>
- </selectedElements>
-</jardesc>
diff --git a/vipra-ws/.settings/org.eclipse.wst.common.component b/vipra-ws/.settings/org.eclipse.wst.common.component
index 850a0974..4bef7533 100644
--- a/vipra-ws/.settings/org.eclipse.wst.common.component
+++ b/vipra-ws/.settings/org.eclipse.wst.common.component
@@ -3,7 +3,7 @@
<wb-resource deploy-path="/" source-path="/target/m2e-wtp/web-resources"/>
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/java"/>
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/resources"/>
- <wb-resource deploy-path="/" source-path="/src/main/webapp"/>
+ <wb-resource deploy-path="/" source-path="/src/main/webapp" tag="defaultRootSource"/>
<property name="context-root" value="vipra-ws"/>
<property name="java-output-path" value="/vipra-ws/build/classes"/>
</wb-module>
--
GitLab