Skip to content
Snippets Groups Projects
Commit d23a0528 authored by Eike Cochu's avatar Eike Cochu
Browse files

too many changes 

removed lucene and custom processors
added debug switch to enable extensive productive logging
added silent switch to mute all logging
replaced cmd project slf4j loggers with log4j loggers
simplified cmd exception structure
added catching mongo and elasticsearch connection exceptions with explicit error messages
removed delete cmd
removed lucene dependency, clashes with elasticsearch lucene 5.0 dependency in executable jar
deleted unnecessary jardesc
removed log4j dependency from utils project
renamed database constants
added elasticsearch constants
added database and elasticsearch config convenience functions
added missing database generic types
added test command to test connections to database and elasticsearch
parent d9cfa80c
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
Showing
with 461 additions and 558 deletions
Vagrantfile
+ 1
− 0
View file @ d23a0528
...@@ -14,6 +14,7 @@ Vagrant.configure(2) do |config| ...@@ -14,6 +14,7 @@ Vagrant.configure(2) do |config|
master.vm.network :forwarded_port, guest: 27017, host: 27017 # MongoDB master.vm.network :forwarded_port, guest: 27017, host: 27017 # MongoDB
master.vm.network :forwarded_port, guest: 8080, host: 8000 # Tomcat master.vm.network :forwarded_port, guest: 8080, host: 8000 # Tomcat
master.vm.network :forwarded_port, guest: 9200, host: 9200 # ElasticSearch REST API master.vm.network :forwarded_port, guest: 9200, host: 9200 # ElasticSearch REST API
master.vm.network :forwarded_port, guest: 9300, host: 9300 # ElasticSearch Native
end end
end end
ma-impl.sublime-workspace
+ 39
− 1
View file @ d23a0528
...@@ -279,6 +279,14 @@ ...@@ -279,6 +279,14 @@
}, },
"buffers": "buffers":
[ [
{
"file": "Vagrantfile",
"settings":
{
"buffer_size": 572,
"line_ending": "Unix"
}
}
], ],
"build_system": "", "build_system": "",
"build_system_choices": "build_system_choices":
...@@ -458,7 +466,6 @@ ...@@ -458,7 +466,6 @@
"expanded_folders": "expanded_folders":
[ [
"/home/eike/repos/master/ma-impl", "/home/eike/repos/master/ma-impl",
"/home/eike/repos/master/ma-impl/vipra-ui",
"/home/eike/repos/master/ma-impl/vipra-ui/app", "/home/eike/repos/master/ma-impl/vipra-ui/app",
"/home/eike/repos/master/ma-impl/vipra-ui/app/components", "/home/eike/repos/master/ma-impl/vipra-ui/app/components",
"/home/eike/repos/master/ma-impl/vipra-ui/app/routes", "/home/eike/repos/master/ma-impl/vipra-ui/app/routes",
...@@ -924,8 +931,39 @@ ...@@ -924,8 +931,39 @@
"groups": "groups":
[ [
{ {
"selected": 0,
"sheets": "sheets":
[ [
{
"buffer": 0,
"file": "Vagrantfile",
"semi_transient": true,
"settings":
{
"buffer_size": 572,
"regions":
{
},
"selection":
[
[
437,
437
]
],
"settings":
{
"syntax": "Packages/Text/Plain text.tmLanguage",
"tab_size": 2,
"translate_tabs_to_spaces": true
},
"translation.x": 0.0,
"translation.y": 0.0,
"zoom_level": 1.0
},
"stack_index": 0,
"type": "text"
}
] ]
} }
], ],
......
vipra-cmd/.classpath
+ 7
− 1
View file @ d23a0528
...@@ -29,7 +29,13 @@ ...@@ -29,7 +29,13 @@
</classpathentry> </classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8"> <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<attributes> <attributes>
<attribute name="owner.project.facets" value="java"/> <attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes> </attributes>
</classpathentry> </classpathentry>
<classpathentry kind="output" path="target/classes"/> <classpathentry kind="output" path="target/classes"/>
......
vipra-cmd/.project
+ 2
− 2
View file @ d23a0528
...@@ -21,12 +21,12 @@ ...@@ -21,12 +21,12 @@
</arguments> </arguments>
</buildCommand> </buildCommand>
<buildCommand> <buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name> <name>net.sourceforge.metrics.builder</name>
<arguments> <arguments>
</arguments> </arguments>
</buildCommand> </buildCommand>
<buildCommand> <buildCommand>
<name>net.sourceforge.metrics.builder</name> <name>org.eclipse.m2e.core.maven2Builder</name>
<arguments> <arguments>
</arguments> </arguments>
</buildCommand> </buildCommand>
......
vipra-cmd/build.xml
+ 81
− 80
View file @ d23a0528
...@@ -4,10 +4,9 @@ ...@@ -4,10 +4,9 @@
<!--ANT 1.7 is required --> <!--ANT 1.7 is required -->
<!--define folder properties--> <!--define folder properties-->
<property name="dir.buildfile" value="." /> <property name="dir.buildfile" value="." />
<property name="dir.workspace" value="/home/eike/workspace"/>
<property name="dir.jarfile" value="${dir.buildfile}/bin" /> <property name="dir.jarfile" value="${dir.buildfile}/bin" />
<target name="create_run_jar"> <target name="create_run_jar">
<jar destfile="${dir.jarfile}/vipra-cmd.jar" filesetmanifest="mergewithoutmain"> <jar destfile="${dir.jarfile}/vipra-cmd.jar" filesetmanifest="mergewithoutmain" duplicate="preserve">
<manifest> <manifest>
<attribute name="Main-Class" value="de.vipra.cmd.Main" /> <attribute name="Main-Class" value="de.vipra.cmd.Main" />
<attribute name="Class-Path" value="." /> <attribute name="Class-Path" value="." />
...@@ -16,10 +15,7 @@ ...@@ -16,10 +15,7 @@
<fileset dir="/home/eike/repos/master/ma-impl/vipra-util/target/classes" /> <fileset dir="/home/eike/repos/master/ma-impl/vipra-util/target/classes" />
<fileset dir="/home/eike/repos/master/ma-impl/jgibblda/target/classes" /> <fileset dir="/home/eike/repos/master/ma-impl/jgibblda/target/classes" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar" /> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-api/2.4.1/log4j-api-2.4.1.jar"/> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/slf4j/slf4j-api/1.7.14/slf4j-api-1.7.14.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-core/2.4.1/log4j-core-2.4.1.jar"/>
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-slf4j-impl/2.4.1/log4j-slf4j-impl-2.4.1.jar"/>
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/slf4j/slf4j-api/1.7.12/slf4j-api-1.7.12.jar"/>
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/mongodb-driver/3.2.0/mongodb-driver-3.2.0.jar" /> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/mongodb-driver/3.2.0/mongodb-driver-3.2.0.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/bson/3.2.0/bson-3.2.0.jar" /> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/bson/3.2.0/bson-3.2.0.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/mongodb-driver-core/3.2.0/mongodb-driver-core-3.2.0.jar" /> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/mongodb-driver-core/3.2.0/mongodb-driver-core-3.2.0.jar" />
...@@ -28,6 +24,18 @@ ...@@ -28,6 +24,18 @@
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/thoughtworks/proxytoys/proxytoys/1.0/proxytoys-1.0.jar" /> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/thoughtworks/proxytoys/proxytoys/1.0/proxytoys-1.0.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/cglib/cglib-nodep/2.2.2/cglib-nodep-2.2.2.jar" /> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/cglib/cglib-nodep/2.2.2/cglib-nodep-2.2.2.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/morphia/morphia-logging-slf4j/1.0.1/morphia-logging-slf4j-1.0.1.jar" /> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/morphia/morphia-logging-slf4j/1.0.1/morphia-logging-slf4j-1.0.1.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/core/jackson-databind/2.7.0/jackson-databind-2.7.0.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/core/jackson-core/2.7.0/jackson-core-2.7.0.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/core/jackson-annotations/2.7.0/jackson-annotations-2.7.0.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/args4j/args4j/2.0.6/args4j-2.0.6.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-api/2.4.1/log4j-api-2.4.1.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-core/2.4.1/log4j-core-2.4.1.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-slf4j-impl/2.4.1/log4j-slf4j-impl-2.4.1.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/slf4j/slf4j-api/1.7.12/slf4j-api-1.7.12.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/commons-cli/commons-cli/1.3.1/commons-cli-1.3.1.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/googlecode/json-simple/json-simple/1.1.1/json-simple-1.1.1.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/junit/junit/4.10/junit-4.10.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/hamcrest/hamcrest-core/1.1/hamcrest-core-1.1.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/elasticsearch/elasticsearch/2.1.0/elasticsearch-2.1.0.jar" /> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/elasticsearch/elasticsearch/2.1.0/elasticsearch-2.1.0.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-core/5.3.1/lucene-core-5.3.1.jar" /> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-core/5.3.1/lucene-core-5.3.1.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-backward-codecs/5.3.1/lucene-backward-codecs-5.3.1.jar" /> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-backward-codecs/5.3.1/lucene-backward-codecs-5.3.1.jar" />
...@@ -57,12 +65,7 @@ ...@@ -57,12 +65,7 @@
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/ning/compress-lzf/1.0.2/compress-lzf-1.0.2.jar" /> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/ning/compress-lzf/1.0.2/compress-lzf-1.0.2.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/tdunning/t-digest/3.0/t-digest-3.0.jar" /> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/tdunning/t-digest/3.0/t-digest-3.0.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/hdrhistogram/HdrHistogram/2.1.6/HdrHistogram-2.1.6.jar" /> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/hdrhistogram/HdrHistogram/2.1.6/HdrHistogram-2.1.6.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/commons-cli/commons-cli/1.3.1/commons-cli-1.3.1.jar"/>
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/twitter/jsr166e/1.1.0/jsr166e-1.1.0.jar" /> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/twitter/jsr166e/1.1.0/jsr166e-1.1.0.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/core/jackson-databind/2.7.0/jackson-databind-2.7.0.jar"/>
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/fasterxml/jackson/core/jackson-annotations/2.7.0/jackson-annotations-2.7.0.jar"/>
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/args4j/args4j/2.0.6/args4j-2.0.6.jar"/>
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/googlecode/json-simple/json-simple/1.1.1/json-simple-1.1.1.jar"/>
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/edu/stanford/nlp/stanford-corenlp/3.5.2/stanford-corenlp-3.5.2.jar" /> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/edu/stanford/nlp/stanford-corenlp/3.5.2/stanford-corenlp-3.5.2.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/io7m/xom/xom/1.2.10/xom-1.2.10.jar" /> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/io7m/xom/xom/1.2.10/xom-1.2.10.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/xml-apis/xml-apis/1.3.03/xml-apis-1.3.03.jar" /> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/xml-apis/xml-apis/1.3.03/xml-apis-1.3.03.jar" />
...@@ -73,11 +76,9 @@ ...@@ -73,11 +76,9 @@
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/googlecode/efficient-java-matrix-library/ejml/0.23/ejml-0.23.jar" /> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/googlecode/efficient-java-matrix-library/ejml/0.23/ejml-0.23.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/javax/json/javax.json-api/1.0/javax.json-api-1.0.jar" /> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/javax/json/javax.json-api/1.0/javax.json-api-1.0.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/edu/stanford/nlp/stanford-corenlp/3.5.2/stanford-corenlp-3.5.2-models.jar" /> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/edu/stanford/nlp/stanford-corenlp/3.5.2/stanford-corenlp-3.5.2-models.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-core/5.4.0/lucene-core-5.4.0.jar"/> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-api/2.5/log4j-api-2.5.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/lucene/lucene-analyzers-common/5.4.0/lucene-analyzers-common-5.4.0.jar"/> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-core/2.5/log4j-core-2.5.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/uk/org/lidalia/sysout-over-slf4j/1.0.2/sysout-over-slf4j-1.0.2.jar"/> <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-slf4j-impl/2.5/log4j-slf4j-impl-2.5.jar" />
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/junit/junit/4.12/junit-4.12.jar"/>
<zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar"/>
</jar> </jar>
</target> </target>
</project> </project>
vipra-cmd/pom.xml
+ 5
− 36
View file @ d23a0528
...@@ -14,8 +14,6 @@ ...@@ -14,8 +14,6 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.target>1.8</maven.compiler.target> <maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source> <maven.compiler.source>1.8</maven.compiler.source>
<log4jVersion>2.4.1</log4jVersion>
<luceneVersion>5.4.0</luceneVersion>
</properties> </properties>
<dependencies> <dependencies>
...@@ -25,11 +23,6 @@ ...@@ -25,11 +23,6 @@
<artifactId>commons-cli</artifactId> <artifactId>commons-cli</artifactId>
<version>1.3.1</version> <version>1.3.1</version>
</dependency> </dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.4</version>
</dependency>
<!-- JsonSimple --> <!-- JsonSimple -->
<dependency> <dependency>
...@@ -58,57 +51,33 @@ ...@@ -58,57 +51,33 @@
<classifier>models</classifier> <classifier>models</classifier>
</dependency> </dependency>
<!-- Lucene -->
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>${luceneVersion}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
<version>${luceneVersion}</version>
</dependency>
<!-- Logging --> <!-- Logging -->
<dependency> <dependency>
<groupId>org.apache.logging.log4j</groupId> <groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId> <artifactId>log4j-api</artifactId>
<version>${log4jVersion}</version> <version>2.5</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.logging.log4j</groupId> <groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId> <artifactId>log4j-core</artifactId>
<version>${log4jVersion}</version> <version>2.5</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.logging.log4j</groupId> <groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId> <artifactId>log4j-slf4j-impl</artifactId>
<version>${log4jVersion}</version> <version>2.5</version>
</dependency> </dependency>
<!-- MongoDB Database Adapter --> <!-- MongoDB Database Adapter -->
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongodb-driver</artifactId>
<version>3.2.0</version>
</dependency>
<dependency> <dependency>
<groupId>org.mongodb.morphia</groupId> <groupId>org.mongodb.morphia</groupId>
<artifactId>morphia</artifactId> <artifactId>morphia</artifactId>
<version>1.0.1</version> <version>1.1.0</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.mongodb.morphia</groupId> <groupId>org.mongodb.morphia</groupId>
<artifactId>morphia-logging-slf4j</artifactId> <artifactId>morphia-logging-slf4j</artifactId>
<version>1.0.1</version> <version>1.1.0</version>
</dependency>
<!-- Testing -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
</dependency> </dependency>
<!-- Workspace --> <!-- Workspace -->
......
vipra-cmd/src/main/java/de/vipra/cmd/CmdOptions.java
+ 12
− 5
View file @ d23a0528
...@@ -17,9 +17,6 @@ public class CmdOptions extends Options { ...@@ -17,9 +17,6 @@ public class CmdOptions extends Options {
public static final String OPT_SHELL = "x"; public static final String OPT_SHELL = "x";
public static final String OPT_SHELL_LONG = "shell"; public static final String OPT_SHELL_LONG = "shell";
public static final String OPT_DELETE = "d";
public static final String OPT_DELETE_LONG = "delete";
public static final String OPT_CLEAR = "c"; public static final String OPT_CLEAR = "c";
public static final String OPT_CLEAR_LONG = "clear"; public static final String OPT_CLEAR_LONG = "clear";
...@@ -29,18 +26,28 @@ public class CmdOptions extends Options { ...@@ -29,18 +26,28 @@ public class CmdOptions extends Options {
public static final String OPT_DEFAULTS = "n"; public static final String OPT_DEFAULTS = "n";
public static final String OPT_DEFAULTS_LONG = "defaults"; public static final String OPT_DEFAULTS_LONG = "defaults";
public static final String OPT_DEBUG = "d";
public static final String OPT_DEBUG_LONG = "debug";
public static final String OPT_TEST = "t";
public static final String OPT_TEST_LONG = "test";
public static final String OPT_SILENT = "s";
public static final String OPT_SILENT_LONG = "silent";
public CmdOptions() { public CmdOptions() {
addOption(Option.builder(OPT_HELP).longOpt(OPT_HELP_LONG).desc("print this message").build()); addOption(Option.builder(OPT_HELP).longOpt(OPT_HELP_LONG).desc("print this message").build());
addOption(Option.builder(OPT_SHELL).longOpt(OPT_SHELL_LONG).hasArg(true).argName("name") addOption(Option.builder(OPT_SHELL).longOpt(OPT_SHELL_LONG).hasArg(true).argName("name")
.desc("run from a shell script").build()); .desc("run from a shell script").build());
addOption(Option.builder(OPT_IMPORT).longOpt(OPT_IMPORT_LONG).hasArgs().argName("files/dirs...") addOption(Option.builder(OPT_IMPORT).longOpt(OPT_IMPORT_LONG).hasArgs().argName("files/dirs...")
.desc("import articles into the database").build()); .desc("import articles into the database").build());
addOption(Option.builder(OPT_DELETE).longOpt(OPT_DELETE_LONG).hasArgs().argName("files/ids...")
.desc("delete articles from the database").build());
addOption(Option.builder(OPT_CLEAR).longOpt(OPT_CLEAR_LONG).desc("clear database and filebase").build()); addOption(Option.builder(OPT_CLEAR).longOpt(OPT_CLEAR_LONG).desc("clear database and filebase").build());
addOption(Option.builder(OPT_STATS).longOpt(OPT_STATS_LONG).desc("gather database and filebase information") addOption(Option.builder(OPT_STATS).longOpt(OPT_STATS_LONG).desc("gather database and filebase information")
.build()); .build());
addOption(Option.builder(OPT_DEFAULTS).longOpt(OPT_DEFAULTS_LONG).desc("accept default decisions").build()); addOption(Option.builder(OPT_DEFAULTS).longOpt(OPT_DEFAULTS_LONG).desc("accept default decisions").build());
addOption(Option.builder(OPT_DEBUG).longOpt(OPT_DEBUG_LONG).desc("show debug information").build());
addOption(Option.builder(OPT_TEST).longOpt(OPT_TEST_LONG).desc("system tests").build());
addOption(Option.builder(OPT_SILENT).longOpt(OPT_SILENT_LONG).desc("mute all output").build());
} }
public void printHelp(String cmd) { public void printHelp(String cmd) {
......
vipra-cmd/src/main/java/de/vipra/cmd/ExecutionException.java deleted 100644 → 0
+ 0
− 38
View file @ d9cfa80c
package de.vipra.cmd;
import java.util.List;
public class ExecutionException extends Exception {
private static final long serialVersionUID = 1L;
private List<Exception> exceptions;
public ExecutionException(String msg) {
super(msg);
}
public ExecutionException(Exception e) {
super(e);
}
public ExecutionException(List<Exception> e) {
this.exceptions = e;
}
@Override
public String getMessage() {
if (exceptions == null) {
return super.getMessage();
} else if (exceptions.size() == 1) {
return exceptions.get(0).getMessage();
} else {
StringBuilder sb = new StringBuilder("multiple errors:");
for (Exception e : exceptions) {
sb.append("\n " + e.getMessage());
}
return sb.toString();
}
}
}
vipra-cmd/src/main/java/de/vipra/cmd/Main.java
+ 51
− 42
View file @ d23a0528
package de.vipra.cmd; package de.vipra.cmd;
import static de.vipra.cmd.CmdOptions.OPT_CLEAR; import static de.vipra.cmd.CmdOptions.OPT_CLEAR;
import static de.vipra.cmd.CmdOptions.OPT_DEBUG;
import static de.vipra.cmd.CmdOptions.OPT_DEFAULTS; import static de.vipra.cmd.CmdOptions.OPT_DEFAULTS;
import static de.vipra.cmd.CmdOptions.OPT_DELETE;
import static de.vipra.cmd.CmdOptions.OPT_HELP; import static de.vipra.cmd.CmdOptions.OPT_HELP;
import static de.vipra.cmd.CmdOptions.OPT_IMPORT; import static de.vipra.cmd.CmdOptions.OPT_IMPORT;
import static de.vipra.cmd.CmdOptions.OPT_SHELL; import static de.vipra.cmd.CmdOptions.OPT_SHELL;
import static de.vipra.cmd.CmdOptions.OPT_SILENT;
import static de.vipra.cmd.CmdOptions.OPT_STATS; import static de.vipra.cmd.CmdOptions.OPT_STATS;
import static de.vipra.cmd.CmdOptions.OPT_TEST;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.ListIterator; import java.util.ListIterator;
import java.util.Map.Entry;
import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.ParseException; import org.apache.commons.cli.ParseException;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.LoggerContext;
import org.apache.logging.log4j.core.config.Configuration;
import org.apache.logging.log4j.core.config.LoggerConfig;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.mongodb.morphia.logging.MorphiaLoggerFactory; import org.mongodb.morphia.logging.MorphiaLoggerFactory;
import org.mongodb.morphia.logging.slf4j.SLF4JLoggerImplFactory; import org.mongodb.morphia.logging.slf4j.SLF4JLoggerImplFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import com.mongodb.MongoTimeoutException;
import de.vipra.cmd.option.ClearCommand; import de.vipra.cmd.option.ClearCommand;
import de.vipra.cmd.option.Command; import de.vipra.cmd.option.Command;
import de.vipra.cmd.option.DeleteCommand;
import de.vipra.cmd.option.ImportCommand; import de.vipra.cmd.option.ImportCommand;
import de.vipra.cmd.option.StatsCommand; import de.vipra.cmd.option.StatsCommand;
import de.vipra.util.ConsoleUtils; import de.vipra.cmd.option.TestCommand;
import de.vipra.util.ConsoleUtils.Choice;
import de.vipra.util.StringUtils;
import de.vipra.util.Timer;
public class Main { public class Main {
public static final Logger log = LoggerFactory.getLogger(Main.class); public static final Logger log = LogManager.getLogger(Main.class);
public static final Logger out = LoggerFactory.getLogger("shellout"); public static final Logger out = LogManager.getLogger("shellout");
static { static {
// set morphia log level
MorphiaLoggerFactory.registerLogger(SLF4JLoggerImplFactory.class); MorphiaLoggerFactory.registerLogger(SLF4JLoggerImplFactory.class);
// close stderr to mute corenlp messages // set corenlp log level, close stderr to mute corenlp messages
System.err.close(); System.err.close();
} }
...@@ -67,53 +74,55 @@ public class Main { ...@@ -67,53 +74,55 @@ public class Main {
return; return;
} }
// logger configuration
LoggerContext loggerContext = (LoggerContext) LogManager.getContext(false);
Configuration loggerConfigs = loggerContext.getConfiguration();
if (cline.hasOption(OPT_DEBUG))
loggerConfigs.getLoggerConfig(LogManager.ROOT_LOGGER_NAME).setLevel(Level.DEBUG);
if (cline.hasOption(OPT_SILENT)) {
for (Entry<String, LoggerConfig> loggerConfig : loggerConfigs.getLoggers().entrySet())
loggerConfig.getValue().setLevel(Level.OFF);
}
loggerContext.updateLoggers();
// check if default decisions should be auto chosen
boolean defaults = cline.hasOption(OPT_DEFAULTS); boolean defaults = cline.hasOption(OPT_DEFAULTS);
// get commands
List<Command> commands = new ArrayList<>(); List<Command> commands = new ArrayList<>();
if (cline.hasOption(OPT_CLEAR)) { if (cline.hasOption(OPT_TEST))
commands.add(new TestCommand());
if (cline.hasOption(OPT_CLEAR))
commands.add(new ClearCommand(defaults)); commands.add(new ClearCommand(defaults));
}
if (cline.hasOption(OPT_IMPORT)) { if (cline.hasOption(OPT_IMPORT))
commands.add(new ImportCommand(cline.getOptionValues(OPT_IMPORT))); commands.add(new ImportCommand(cline.getOptionValues(OPT_IMPORT)));
}
if (cline.hasOption(OPT_DELETE)) {
commands.add(new DeleteCommand(cline.getOptionValues(OPT_DELETE)));
}
if (cline.hasOption(OPT_STATS)) { if (cline.hasOption(OPT_STATS))
commands.add(new StatsCommand()); commands.add(new StatsCommand());
}
// run commands
if (commands.size() > 0) { if (commands.size() > 0) {
Timer t = new Timer(); for (ListIterator<Command> it = commands.listIterator(); it.hasNext();) {
t.start();
commandLoop: for (ListIterator<Command> it = commands.listIterator(); it.hasNext();) {
Command c = it.next(); Command c = it.next();
try { try {
c.run(); c.run();
} catch (ExecutionException e) { } catch (MongoTimeoutException e) {
out.error(e.getMessage(), e); out.error("timeout while trying to connect to the database");
ConsoleUtils.Choice choice; log.debug(e.getMessage(), e);
boolean acceptDefault = cline.hasOption(OPT_DEFAULTS); } catch (NoNodeAvailableException e) {
do { out.error("could not connect to elasticsearch instance");
choice = ConsoleUtils.prompt(Choice.CONTINUE, acceptDefault, Choice.ABORT, Choice.RETRY); log.debug(e.getMessage(), e);
switch (choice) { } catch (Exception e) {
case ABORT: out.error(e.getCause().getMessage());
break commandLoop; log.debug(e.getMessage(), e);
case RETRY:
it.previous();
case CONTINUE:
continue commandLoop;
}
} while (choice == null);
} }
} }
long dur = t.stop();
out.info("done in " + StringUtils.timeString(dur));
} else { } else {
options.printHelp(cmd); options.printHelp(cmd);
} }
......
vipra-cmd/src/main/java/de/vipra/cmd/es/ESClient.java 0 → 100644
+ 19
− 0
View file @ d23a0528
package de.vipra.cmd.es;
import java.net.InetAddress;
import java.net.UnknownHostException;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import de.vipra.util.Config;
import de.vipra.util.Constants;
public abstract class ESClient {
public static TransportClient getClient(Config config) throws UnknownHostException {
return TransportClient.builder().build().addTransportAddress(
new InetSocketTransportAddress(InetAddress.getByName(Constants.ES_HOST), Constants.ES_PORT));
}
}
vipra-cmd/src/main/java/de/vipra/cmd/ex/ClearException.java deleted 100644 → 0
+ 0
− 15
View file @ d9cfa80c
package de.vipra.cmd.ex;
public class ClearException extends Exception {
private static final long serialVersionUID = 1L;
public ClearException(String msg) {
super(msg);
}
public ClearException(Exception e) {
super(e);
}
}
vipra-cmd/src/main/java/de/vipra/cmd/ex/ImportException.java deleted 100644 → 0
+ 0
− 23
View file @ d9cfa80c
package de.vipra.cmd.ex;
public class ImportException extends Exception {
private static final long serialVersionUID = 1L;
private final String id;
public ImportException(String msg, String id) {
super(msg);
this.id = id;
}
public ImportException(Exception e, String id) {
super(e);
this.id = id;
}
public String getId() {
return id;
}
}
vipra-cmd/src/main/java/de/vipra/cmd/lda/JGibbLDAAnalyzer.java
+ 3
− 3
View file @ d23a0528
...@@ -9,8 +9,8 @@ import java.util.List; ...@@ -9,8 +9,8 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import org.slf4j.Logger; import org.apache.logging.log4j.LogManager;
import org.slf4j.LoggerFactory; import org.apache.logging.log4j.Logger;
import de.vipra.cmd.ex.LDAAnalyzerException; import de.vipra.cmd.ex.LDAAnalyzerException;
import de.vipra.util.Config; import de.vipra.util.Config;
...@@ -31,7 +31,7 @@ import jgibblda.Model; ...@@ -31,7 +31,7 @@ import jgibblda.Model;
public class JGibbLDAAnalyzer extends LDAAnalyzer { public class JGibbLDAAnalyzer extends LDAAnalyzer {
public static final Logger log = LoggerFactory.getLogger(JGibbLDAAnalyzer.class); public static final Logger log = LogManager.getLogger(JGibbLDAAnalyzer.class);
public static final String NAME = "jgibb"; public static final String NAME = "jgibb";
private File dataDir; private File dataDir;
......
vipra-cmd/src/main/java/de/vipra/cmd/model/ProcessedArticle.java
+ 18
− 0
View file @ d23a0528
package de.vipra.cmd.model; package de.vipra.cmd.model;
import java.util.Date;
import org.json.simple.JSONObject; import org.json.simple.JSONObject;
import org.mongodb.morphia.annotations.Entity; import org.mongodb.morphia.annotations.Entity;
import org.mongodb.morphia.annotations.Transient; import org.mongodb.morphia.annotations.Transient;
import de.vipra.cmd.text.ProcessedText; import de.vipra.cmd.text.ProcessedText;
import de.vipra.util.an.ElasticIndex;
@SuppressWarnings("serial") @SuppressWarnings("serial")
@Entity(value = "articles", noClassnameStored = true) @Entity(value = "articles", noClassnameStored = true)
...@@ -21,6 +24,21 @@ public class ProcessedArticle extends de.vipra.util.model.ArticleFull { ...@@ -21,6 +24,21 @@ public class ProcessedArticle extends de.vipra.util.model.ArticleFull {
this.processedText = processedText; this.processedText = processedText;
} }
@ElasticIndex("title")
public String serializeTitle() {
return super.getTitle();
}
@ElasticIndex("date")
public Date serializeDate() {
return super.getDate();
}
@ElasticIndex("text")
public String serializeText() {
return processedText.getText();
}
public void fromJSON(JSONObject obj) { public void fromJSON(JSONObject obj) {
if (obj.containsKey("title")) if (obj.containsKey("title"))
setTitle(obj.get("title").toString()); setTitle(obj.get("title").toString());
......
vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java
+ 19
− 30
View file @ d23a0528
...@@ -4,16 +4,13 @@ import java.io.File; ...@@ -4,16 +4,13 @@ import java.io.File;
import java.io.IOException; import java.io.IOException;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bson.types.ObjectId; import org.bson.types.ObjectId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import de.vipra.cmd.ExecutionException;
import de.vipra.cmd.ex.ClearException;
import de.vipra.cmd.model.ProcessedArticle; import de.vipra.cmd.model.ProcessedArticle;
import de.vipra.util.Config; import de.vipra.util.Config;
import de.vipra.util.ConsoleUtils; import de.vipra.util.ConsoleUtils;
import de.vipra.util.ex.ConfigException;
import de.vipra.util.model.Import; import de.vipra.util.model.Import;
import de.vipra.util.model.TopicFull; import de.vipra.util.model.TopicFull;
import de.vipra.util.model.Word; import de.vipra.util.model.Word;
...@@ -21,8 +18,8 @@ import de.vipra.util.service.DatabaseService; ...@@ -21,8 +18,8 @@ import de.vipra.util.service.DatabaseService;
public class ClearCommand implements Command { public class ClearCommand implements Command {
public static final Logger log = LoggerFactory.getLogger(ClearCommand.class); public static final Logger log = LogManager.getLogger(ClearCommand.class);
public static final Logger out = LoggerFactory.getLogger("shellout"); public static final Logger out = LogManager.getLogger("shellout");
private boolean defaults; private boolean defaults;
private Config config; private Config config;
...@@ -35,16 +32,12 @@ public class ClearCommand implements Command { ...@@ -35,16 +32,12 @@ public class ClearCommand implements Command {
this.defaults = defaults; this.defaults = defaults;
} }
private void clear() throws ClearException, ConfigException { private void clear() throws Exception {
try {
config = Config.getConfig(); config = Config.getConfig();
dbArticles = DatabaseService.getDatabaseService(config, ProcessedArticle.class); dbArticles = DatabaseService.getDatabaseService(config, ProcessedArticle.class);
dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class); dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class);
dbWords = DatabaseService.getDatabaseService(config, Word.class); dbWords = DatabaseService.getDatabaseService(config, Word.class);
dbImports = DatabaseService.getDatabaseService(config, Import.class); dbImports = DatabaseService.getDatabaseService(config, Import.class);
} catch (Exception e) {
throw new ClearException(e);
}
out.info("clearing database"); out.info("clearing database");
dbArticles.drop(); dbArticles.drop();
...@@ -52,28 +45,24 @@ public class ClearCommand implements Command { ...@@ -52,28 +45,24 @@ public class ClearCommand implements Command {
dbWords.drop(); dbWords.drop();
dbImports.drop(); dbImports.drop();
try {
out.info("clearing filebase"); out.info("clearing filebase");
File dataDir = config.getDataDirectory(); File dataDir = config.getDataDirectory();
if (dataDir.exists() && dataDir.isDirectory()) { if (dataDir.exists() && dataDir.isDirectory()) {
try {
FileUtils.deleteDirectory(dataDir); FileUtils.deleteDirectory(dataDir);
} catch (IOException e) {
out.warn("could not delete data directory: " + dataDir.getAbsolutePath());
} }
} catch (IOException e) {
out.warn("could not delete data directory: " + config.getDataDirectory().getAbsolutePath());
} }
} }
@Override @Override
public void run() throws ExecutionException { public void run() throws Exception {
if (!defaults) if (!defaults)
out.info("to confirm clearing, type 'clear' and press enter"); out.info("to confirm clearing, type 'clear' and press enter");
try {
if (defaults || ConsoleUtils.confirm("clear")) { if (defaults || ConsoleUtils.confirm("clear")) {
clear(); clear();
} }
} catch (ClearException | ConfigException e) {
throw new ExecutionException(e);
}
} }
} }
vipra-cmd/src/main/java/de/vipra/cmd/option/Command.java
+ 1
− 3
View file @ d23a0528
package de.vipra.cmd.option; package de.vipra.cmd.option;
import de.vipra.cmd.ExecutionException;
public interface Command { public interface Command {
public void run() throws ExecutionException; public void run() throws Exception;
} }
vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteCommand.java deleted 100644 → 0
+ 0
− 101
View file @ d9cfa80c
package de.vipra.cmd.option;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.bson.types.ObjectId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import de.vipra.cmd.ExecutionException;
import de.vipra.cmd.ex.FilebaseException;
import de.vipra.cmd.file.Filebase;
import de.vipra.cmd.model.ProcessedArticle;
import de.vipra.util.Config;
import de.vipra.util.MongoUtils;
import de.vipra.util.ex.ConfigException;
import de.vipra.util.ex.DatabaseException;
import de.vipra.util.service.DatabaseService;
public class DeleteCommand implements Command {
public static final Logger log = LoggerFactory.getLogger(DeleteCommand.class);
public static final Logger out = LoggerFactory.getLogger("shellout");
private ArrayList<String> ids = new ArrayList<>();
private Config config;
private DatabaseService<ProcessedArticle, ObjectId> dbArticles;
private Filebase filebase;
/**
* Creates a new delete command. Use {@link DeleteCommand#run()} to run this
* command.
*
* @param ids
* array of ids to be used for deletion.
*/
public DeleteCommand(String[] ids) {
addIds(ids);
}
public void addIds(String[] strings) {
for (String str : strings) {
if (str.contains(File.pathSeparator)) {
String[] parts = str.split(File.pathSeparator);
str = parts[parts.length - 1];
}
ids.add(str);
}
}
public void deleteEntry(String id) throws ExecutionException {
ArrayList<Exception> errors = new ArrayList<>();
try {
// 1. delete mongodb entry
dbArticles.deleteSingle(MongoUtils.objectId(id));
} catch (DatabaseException e) {
errors.add(e);
}
try {
// 2. delete file
filebase.remove(id);
} catch (FilebaseException e) {
errors.add(e);
}
// 3. delete elasticsearch index entry
// TODO implement
if (errors.size() > 0) {
throw new ExecutionException(errors);
}
}
@Override
public void run() throws ExecutionException {
try {
config = Config.getConfig();
dbArticles = DatabaseService.getDatabaseService(config, ProcessedArticle.class);
filebase = Filebase.getFilebase(config);
} catch (IOException | FilebaseException | ConfigException e) {
throw new ExecutionException(e);
}
List<Exception> ex = new ArrayList<>();
for (String id : ids) {
try {
deleteEntry(id);
} catch (Exception e) {
ex.add(e);
}
}
if (ex.size() > 0) {
throw new ExecutionException(ex);
}
}
}
vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java
+ 154
− 160
View file @ d23a0528
package de.vipra.cmd.option; package de.vipra.cmd.option;
import java.io.File; import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader; import java.io.FileReader;
import java.io.FilenameFilter; import java.io.FilenameFilter;
import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bson.types.ObjectId; import org.bson.types.ObjectId;
import org.elasticsearch.client.Client;
import org.json.simple.JSONArray; import org.json.simple.JSONArray;
import org.json.simple.JSONObject; import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser; import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import de.vipra.cmd.ExecutionException; import de.vipra.cmd.es.ESClient;
import de.vipra.cmd.ex.ImportException;
import de.vipra.cmd.file.Filebase; import de.vipra.cmd.file.Filebase;
import de.vipra.cmd.file.FilebaseIndex; import de.vipra.cmd.file.FilebaseIndex;
import de.vipra.cmd.lda.LDAAnalyzer; import de.vipra.cmd.lda.LDAAnalyzer;
...@@ -29,6 +26,7 @@ import de.vipra.cmd.text.ProcessedText; ...@@ -29,6 +26,7 @@ import de.vipra.cmd.text.ProcessedText;
import de.vipra.cmd.text.Processor; import de.vipra.cmd.text.Processor;
import de.vipra.util.Config; import de.vipra.util.Config;
import de.vipra.util.ConvertStream; import de.vipra.util.ConvertStream;
import de.vipra.util.ElasticSerializer;
import de.vipra.util.MongoUtils; import de.vipra.util.MongoUtils;
import de.vipra.util.StringUtils; import de.vipra.util.StringUtils;
import de.vipra.util.Timer; import de.vipra.util.Timer;
...@@ -45,8 +43,8 @@ import de.vipra.util.service.DatabaseService; ...@@ -45,8 +43,8 @@ import de.vipra.util.service.DatabaseService;
public class ImportCommand implements Command { public class ImportCommand implements Command {
public static final Logger log = LoggerFactory.getLogger(ImportCommand.class); public static final Logger log = LogManager.getLogger(ImportCommand.class);
public static final Logger out = LoggerFactory.getLogger("shellout"); public static final Logger out = LogManager.getLogger("shellout");
private ArrayList<File> files = new ArrayList<>(); private ArrayList<File> files = new ArrayList<>();
private JSONParser parser = new JSONParser(); private JSONParser parser = new JSONParser();
...@@ -59,6 +57,8 @@ public class ImportCommand implements Command { ...@@ -59,6 +57,8 @@ public class ImportCommand implements Command {
private Processor preprocessor; private Processor preprocessor;
private WordMap wordMap; private WordMap wordMap;
private LDAAnalyzer analyzer; private LDAAnalyzer analyzer;
private Client elasticClient;
private ElasticSerializer<ProcessedArticle> elasticSerializer;
/** /**
* Import command to import articles into the database, do topic modeling * Import command to import articles into the database, do topic modeling
...@@ -105,14 +105,13 @@ public class ImportCommand implements Command { ...@@ -105,14 +105,13 @@ public class ImportCommand implements Command {
* *
* @param obj * @param obj
* @return * @return
* @throws ImportException * @throws Exception
*/ */
private Article importArticle(JSONObject obj) throws ImportException { private Article importArticle(JSONObject obj) throws Exception {
out.info("importing \"" + StringUtils.ellipsize(obj.get("title").toString(), 80) + "\""); out.info("importing \"" + StringUtils.ellipsize(obj.get("title").toString(), 80) + "\"");
ProcessedArticle article = new ProcessedArticle(); ProcessedArticle article = new ProcessedArticle();
article.fromJSON(obj); article.fromJSON(obj);
try {
// preprocess text and generate text statistics // preprocess text and generate text statistics
ProcessedText processedText = preprocessor.preprocess(article.getText()); ProcessedText processedText = preprocessor.preprocess(article.getText());
ArticleStats articleStats = ArticleStats.generateFromText(processedText.getText(), wordMap); ArticleStats articleStats = ArticleStats.generateFromText(processedText.getText(), wordMap);
...@@ -129,25 +128,21 @@ public class ImportCommand implements Command { ...@@ -129,25 +128,21 @@ public class ImportCommand implements Command {
// add article to filebase // add article to filebase
filebase.add(article); filebase.add(article);
// index article
Map<String, Object> source = elasticSerializer.serialize(article);
elasticClient.prepareIndex("articles", "article", article.getId().toString()).setSource(source).get();
// return article reference // return article reference
return new Article(article.getId()); return new Article(article.getId());
} catch (Exception e) {
throw new ImportException(e, article.getId().toString());
}
} }
/** /**
* Imports a file into the database and the filebase * Imports a file into the database and the filebase
* *
* @param file * @param file
* @throws ParseException
* @throws IOException
* @throws FileNotFoundException
* @throws ImportException
* @throws Exception * @throws Exception
*/ */
private List<Article> importFile(File file) private List<Article> importFile(File file) throws Exception {
throws FileNotFoundException, IOException, ParseException, ImportException {
Object data = parser.parse(new FileReader(file)); Object data = parser.parse(new FileReader(file));
List<Article> articles = new ArrayList<>(); List<Article> articles = new ArrayList<>();
...@@ -162,8 +157,7 @@ public class ImportCommand implements Command { ...@@ -162,8 +157,7 @@ public class ImportCommand implements Command {
return articles; return articles;
} }
private List<Article> importFiles(List<File> files) private List<Article> importFiles(List<File> files) throws Exception {
throws FileNotFoundException, IOException, ParseException, ImportException {
List<Article> articles = new ArrayList<>(); List<Article> articles = new ArrayList<>();
for (File file : files) { for (File file : files) {
articles.addAll(importFile(file)); articles.addAll(importFile(file));
...@@ -172,8 +166,7 @@ public class ImportCommand implements Command { ...@@ -172,8 +166,7 @@ public class ImportCommand implements Command {
} }
@Override @Override
public void run() throws ExecutionException { public void run() throws Exception {
try {
config = Config.getConfig(); config = Config.getConfig();
dbArticles = DatabaseService.getDatabaseService(config, ProcessedArticle.class); dbArticles = DatabaseService.getDatabaseService(config, ProcessedArticle.class);
dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class); dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class);
...@@ -183,6 +176,8 @@ public class ImportCommand implements Command { ...@@ -183,6 +176,8 @@ public class ImportCommand implements Command {
preprocessor = Processor.getPreprocessor(config); preprocessor = Processor.getPreprocessor(config);
wordMap = new WordMap(dbWords); wordMap = new WordMap(dbWords);
analyzer = LDAAnalyzer.getAnalyzer(config, wordMap); analyzer = LDAAnalyzer.getAnalyzer(config, wordMap);
elasticClient = ESClient.getClient(config);
elasticSerializer = new ElasticSerializer<>(ProcessedArticle.class);
out.info("using data directory: " + config.getDataDirectory().getAbsolutePath()); out.info("using data directory: " + config.getDataDirectory().getAbsolutePath());
out.info("using preprocessor: " + preprocessor.getName()); out.info("using preprocessor: " + preprocessor.getName());
...@@ -284,6 +279,8 @@ public class ImportCommand implements Command { ...@@ -284,6 +279,8 @@ public class ImportCommand implements Command {
importOp.setDuration(timer.total()); importOp.setDuration(timer.total());
dbImports.createSingle(importOp); dbImports.createSingle(importOp);
elasticClient.close();
/* /*
* run information * run information
*/ */
...@@ -292,9 +289,6 @@ public class ImportCommand implements Command { ...@@ -292,9 +289,6 @@ public class ImportCommand implements Command {
out.info("imported " + newArticlesCount + " new " + StringUtils.quantity(newArticlesCount, "article")); out.info("imported " + newArticlesCount + " new " + StringUtils.quantity(newArticlesCount, "article"));
out.info("imported " + newWordsCount + " new " + StringUtils.quantity(newWordsCount, "word")); out.info("imported " + newWordsCount + " new " + StringUtils.quantity(newWordsCount, "word"));
out.info(timer.toString()); out.info(timer.toString());
} catch (Exception e) {
throw new ExecutionException(e);
}
} }
} }
vipra-cmd/src/main/java/de/vipra/cmd/option/StatsCommand.java
+ 10
− 18
View file @ d23a0528
package de.vipra.cmd.option; package de.vipra.cmd.option;
import java.io.File; import java.io.File;
import java.io.IOException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bson.types.ObjectId; import org.bson.types.ObjectId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import de.vipra.cmd.ExecutionException;
import de.vipra.cmd.ex.FilebaseException;
import de.vipra.cmd.file.Filebase; import de.vipra.cmd.file.Filebase;
import de.vipra.util.Config; import de.vipra.util.Config;
import de.vipra.util.StringUtils; import de.vipra.util.StringUtils;
import de.vipra.util.ex.ConfigException;
import de.vipra.util.model.TopicFull; import de.vipra.util.model.TopicFull;
import de.vipra.util.service.DatabaseService; import de.vipra.util.service.DatabaseService;
public class StatsCommand implements Command { public class StatsCommand implements Command {
public static final Logger log = LoggerFactory.getLogger(StatsCommand.class); public static final Logger log = LogManager.getLogger(StatsCommand.class);
public static final Logger out = LoggerFactory.getLogger("shellout"); public static final Logger out = LogManager.getLogger("shellout");
private Config config; private Config config;
private Filebase filebase; private Filebase filebase;
...@@ -34,16 +30,12 @@ public class StatsCommand implements Command { ...@@ -34,16 +30,12 @@ public class StatsCommand implements Command {
} }
@Override @Override
public void run() throws ExecutionException { public void run() throws Exception {
try {
config = Config.getConfig(); config = Config.getConfig();
filebase = Filebase.getFilebase(config); filebase = Filebase.getFilebase(config);
dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class); dbTopics = DatabaseService.getDatabaseService(config, TopicFull.class);
stats(); stats();
} catch (IOException | ConfigException | FilebaseException e) {
throw new ExecutionException(e);
}
} }
} }
vipra-cmd/src/main/java/de/vipra/cmd/option/TestCommand.java 0 → 100644
+ 39
− 0
View file @ d23a0528
package de.vipra.cmd.option;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bson.types.ObjectId;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.client.transport.TransportClient;
import de.vipra.cmd.es.ESClient;
import de.vipra.util.Config;
import de.vipra.util.model.Article;
import de.vipra.util.service.DatabaseService;
public class TestCommand implements Command {
public static final Logger out = LogManager.getLogger("shellout");
@Override
public void run() throws Exception {
// test if configuration readable
out.info("reading configuration...");
Config config = Config.getConfig();
// test if database is accessible
out.info("testing mongodb connection...");
DatabaseService<Article, ObjectId> dbArticles = DatabaseService.getDatabaseService(config, Article.class);
dbArticles.count();
// test if elasticsearch is accessible
out.info("testing elasticsearch connection...");
TransportClient esclient = ESClient.getClient(config);
if (esclient.connectedNodes().isEmpty()) {
throw new NoNodeAvailableException("no elasticsearch nodes available");
}
out.info("all tests passed");
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment