updated log4j prod/dev settings

updated test json files for cmd tool
added first version of cmd shell script
added ant script for automated runnable jar file creation for cmd
added util StringUtils

ma-impl.sublime-workspace

@@ -450,13 +450,15 @@
 	"expanded_folders":
 	[
 		"/home/eike/Repositories/fu/ss15/ma/impl",
-		"/home/eike/Repositories/fu/ss15/ma/impl/vipra-ui/app",
-		"/home/eike/Repositories/fu/ss15/ma/impl/vipra-ui/app/templates/articles",
-		"/home/eike/Repositories/fu/ss15/ma/impl/vipra-ui/app/templates/components",
 		"/home/eike/Repositories/fu/ss15/ma/impl/vm"
 	],
 	"file_history":
 	[
+		"/home/eike/Repositories/fu/ss15/ma/impl/vm/data/test-1.json",
+		"/home/eike/Repositories/fu/ss15/ma/impl/vm/data/test-2.json",
+		"/home/eike/Repositories/fu/ss15/ma/impl/vm/data/data.json",
+		"/home/eike/Repositories/fu/ss15/ma/impl/vipra-cmd/test/test-1.json",
+		"/home/eike/Repositories/fu/ss15/ma/impl/vipra-cmd/vipra-cmd.sh",
 		"/home/eike/Repositories/fu/ss15/ma/impl/vipra-ui/app/components/article-list.js",
 		"/home/eike/Repositories/fu/ss15/ma/impl/vipra-ui/app/components/text-marker.js",
 		"/home/eike/Repositories/fu/ss15/ma/impl/vipra-ui/app/components/debounced-input.js",
@@ -506,7 +508,6 @@
 		"/home/eike/Repositories/fu/ss15/ma/impl/vm/config/environment",
 		"/home/eike/Repositories/fu/ss15/ma/impl/tmbs-frontend/app/templates/articles.hbs",
 		"/home/eike/Repositories/fu/ss15/ma/impl/tmbs-frontend/app/templates/application.hbs",
-		"/home/eike/Repositories/fu/ss15/ma/impl/vm/data/data.json",
 		"/home/eike/Repositories/fu/ss15/ma/impl/vm/config/disable-transparent-hugepages",
 		"/home/eike/Repositories/fu/ss15/ma/impl/vm/config/spark-env.sh",
 		"/home/eike/Repositories/fu/ss15/ma/impl/vm/rc.sh",

vipra-cmd/.settings/org.eclipse.jdt.ui.prefs

+eclipse.preferences.version=1
+formatter_profile=_vipra
+formatter_settings_version=12

vipra-cmd/bin/vipra-cmd.sh

+#!/bin/sh
+
+JARFILE="vipra-cmd.jar"
+SCRIPTFILE="$(basename $0)"
+
+# find dir of this script
+# from https://stackoverflow.com/questions/59895
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR="$(cd -P "$(dirname "$SOURCE")" && pwd)"
+
+java -jar "$DIR/$JARFILE" -x "$SCRIPTFILE" "$@"
+exit 0
\ No newline at end of file

vipra-cmd/src/main/java/de/vipra/cmd/CmdOptions.java

@@ -11,9 +11,12 @@ public class CmdOptions extends Options {
 
 	public static final String OPT_HELP = "h";
 	public static final String OPT_IMPORT = "i";
+	public static final String OPT_SHELL = "x";
 
 	public CmdOptions() {
 		addOption(Option.builder(OPT_HELP).longOpt("help").desc("print this message").build());
+		addOption(Option.builder(OPT_SHELL).longOpt("shell").hasArg(true).argName("name")
+				.desc("run from a shell script").build());
 
 		OptionGroup opts = new OptionGroup();
 		opts.setRequired(true);
@@ -24,9 +27,9 @@ public class CmdOptions extends Options {
 		addOptionGroup(opts);
 	}
 
-	public void printHelp() {
+	public void printHelp(String cmd) {
 		HelpFormatter formatter = new HelpFormatter();
-		formatter.printHelp("vipra", this);
+		formatter.printHelp(cmd, this, true);
 	}
 
 }

vipra-cmd/src/main/java/de/vipra/cmd/ExecutionException.java

 package de.vipra.cmd;
 
+import java.util.List;
+
 public class ExecutionException extends Exception {
 
 	private static final long serialVersionUID = 1L;
 
+	private List<Exception> exceptions;
+
 	public ExecutionException(String msg) {
 		super(msg);
 	}
 
+	public ExecutionException(Exception e) {
+		super(e);
+	}
+
+	public ExecutionException(List<Exception> e) {
+		this.exceptions = e;
+	}
+
+	@Override
+	public String getMessage() {
+		if (exceptions == null) {
+			return super.getMessage();
+		} else {
+			StringBuilder sb = new StringBuilder("multiple errors:");
+			for (Exception e : exceptions) {
+				sb.append("\n  " + e.getMessage());
+			}
+			return sb.toString();
+		}
+	}
+
 }

vipra-cmd/src/main/java/de/vipra/cmd/Main.java

@@ -12,6 +12,8 @@ import org.slf4j.LoggerFactory;
 import de.vipra.cmd.option.ImportOption;
 import de.vipra.util.ConfigException;
 
+import static de.vipra.cmd.CmdOptions.*;
+
 public class Main {
 
 	public static final Logger log = LoggerFactory.getLogger(Main.class);
@@ -19,19 +21,28 @@ public class Main {
 	public static void main(String[] args) throws IOException, ConfigException {
 		CommandLineParser parser = new DefaultParser();
 		CmdOptions options = new CmdOptions();
+		String cmd = "vipra-cmd.jar";
 		try {
-			CommandLine cmd = parser.parse(options, args);
-			if (cmd.hasOption(CmdOptions.OPT_HELP)) {
-				options.printHelp();
-			} else if (cmd.hasOption(CmdOptions.OPT_IMPORT)) {
-				String[] paths = cmd.getOptionValues(CmdOptions.OPT_IMPORT);
+			CommandLine cline = parser.parse(options, args);
+			if (cline.hasOption(OPT_SHELL)) {
+				cmd = cline.getOptionValue(OPT_SHELL);
+				if (cmd == null) {
+					cmd = "vipra-cmd.sh";
+				}
+			}
+
+			if (cline.hasOption(OPT_HELP)) {
+				options.printHelp(cmd);
+			} else if (cline.hasOption(OPT_IMPORT)) {
+				String[] paths = cline.getOptionValues(OPT_IMPORT);
 				ImportOption opt = new ImportOption(paths);
 				opt.doImport();
 			} else {
-				options.printHelp();
+				options.printHelp(cmd);
 			}
 		} catch (ParseException e) {
-			options.printHelp();
+			log.error(e.getMessage());
+			options.printHelp(cmd);
 		} catch (ExecutionException e) {
 			log.error(e.getMessage());
 		}

vipra-cmd/src/main/java/de/vipra/cmd/option/ImportOption.java

@@ -5,31 +5,32 @@ import java.io.FileReader;
 import java.io.FilenameFilter;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.List;
 
 import org.json.simple.JSONArray;
 import org.json.simple.JSONObject;
 import org.json.simple.parser.JSONParser;
 import org.json.simple.parser.ParseException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import de.vipra.cmd.ExecutionException;
 import de.vipra.util.Config;
 import de.vipra.util.ConfigException;
 import de.vipra.util.Mongo;
+import de.vipra.util.StringUtils;
 
 public class ImportOption {
 
+	public static final Logger log = LoggerFactory.getLogger(ImportOption.class);
+	public static final Logger out = LoggerFactory.getLogger("shellout");
+
 	private ArrayList<File> files = new ArrayList<>();
 	private JSONParser parser = new JSONParser();
 	private Config config;
 	private Mongo mongo;
 
 	public ImportOption(String[] paths) throws ExecutionException {
-		try {
-			config = new Config();
-			mongo = Mongo.getInstance(config);
-		} catch (IOException | ConfigException e) {
-			throw new ExecutionException(e.getMessage());
-		}
 		addPaths(paths);
 	}
 
@@ -66,15 +67,28 @@ public class ImportOption {
 		}
 	}
 
-	private void importFile(File file) throws IOException, ParseException {
-		Object data = parser.parse(new FileReader(file));
+	private void importFile(File file) throws ExecutionException {
+		Object data;
+		try {
+			data = parser.parse(new FileReader(file));
+		} catch (IOException e) {
+			throw new ExecutionException("file not found: " + file.getAbsolutePath());
+		} catch (ParseException e) {
+			throw new ExecutionException("could not parse json in file: " + file.getAbsolutePath());
+		}
+
 		try {
 			JSONArray array = (JSONArray) data;
 			for (Object object : array) {
 				importArticle((JSONObject) object);
 			}
-		} catch (Exception e) {
-			importArticle((JSONObject) data);
+			return;
+		} catch (ClassCastException e) {
+			try {
+				importArticle((JSONObject) data);
+			} catch (ClassCastException e2) {
+				throw new ExecutionException("invalid json file format: " + file.getAbsolutePath());
+			}
 		}
 	}
 
@@ -83,15 +97,28 @@ public class ImportOption {
 		// 2. add article to file database
 		// 4. topic modeling
 		// 3. index article via elasticsearch, include topics
+		out.info("importing \"" + StringUtils.ellipsize(obj.get("title").toString(), 80) + "\"");
 	}
 
-	public void doImport() {
+	public void doImport() throws ExecutionException {
+		try {
+			config = new Config();
+			mongo = Mongo.getInstance(config);
+		} catch (IOException | ConfigException e) {
+			throw new ExecutionException(e);
+		}
+
+		List<Exception> ex = new ArrayList<>();
 		for (File file : files) {
 			try {
 				importFile(file);
 			} catch (Exception e) {
+				ex.add(e);
 			}
 		}
+		if (ex.size() > 0) {
+			throw new ExecutionException(ex);
+		}
 	}
 
 }

vipra-cmd/src/main/resources/log4j2.xml

@@ -2,12 +2,13 @@
 <Configuration>
 	<Appenders>
 		<Console name="Console" target="SYSTEM_OUT">
-			<PatternLayout pattern="%level - %msg%n" />
+			<PatternLayout pattern="%highlight{%-5level - %msg%n}{FATAL=red,ERROR=red,WARN=red,INFO=normal,DEBUG=normal,TRACE=normal}" />
 		</Console>
 	</Appenders>
 	<Loggers>
-		<Root level="error">
+		<Root level="ERROR">
 			<AppenderRef ref="Console" />
 		</Root>
+		<Logger name="shellout" level="ALL"/>
 	</Loggers>
 </Configuration>
\ No newline at end of file

vipra-cmd/src/main/resources/log4j2dev.xml

@@ -6,7 +6,7 @@
 		</Console>
 	</Appenders>
 	<Loggers>
-		<Root level="all">
+		<Root level="ALL">
 			<AppenderRef ref="Console" />
 		</Root>
 	</Loggers>

vipra-cmd/vipra-cmd.jardesc

-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<jardesc>
-    <jar path="vipra-cmd/vipra-cmd.jar"/>
-    <options buildIfNeeded="true" compress="true" descriptionLocation="/vipra-cmd/vipra-cmd.jardesc" exportErrors="true" exportWarnings="true" includeDirectoryEntries="false" overwrite="true" saveDescription="true" storeRefactorings="false" useSourceFolders="false"/>
-    <storedRefactorings deprecationInfo="true" structuralOnly="false"/>
-    <selectedProjects/>
-    <manifest generateManifest="true" manifestLocation="" manifestVersion="1.0" reuseManifest="false" saveManifest="false" usesManifest="true">
-        <sealing sealJar="false">
-            <packagesToSeal/>
-            <packagesToUnSeal/>
-        </sealing>
-    </manifest>
-    <selectedElements exportClassFiles="true" exportJavaFiles="false" exportOutputFolder="false">
-        <file path="/vipra-cmd/.gitignore"/>
-        <javaElement handleIdentifier="=vipra-cmd/src\/test\/resources"/>
-        <file path="/vipra-cmd/.project"/>
-        <folder path="/vipra-cmd/target"/>
-        <file path="/vipra-cmd/pom.xml"/>
-        <folder path="/vipra-cmd/src"/>
-        <file path="/vipra-cmd/.classpath"/>
-        <javaElement handleIdentifier="=vipra-cmd/src\/main\/java"/>
-        <javaElement handleIdentifier="=vipra-cmd/src\/test\/java"/>
-        <javaElement handleIdentifier="=vipra-cmd/src\/main\/resources"/>
-    </selectedElements>
-</jardesc>

vipra-cmd/vipra-cmd.xml

+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<project default="create_run_jar" name="Create Runnable Jar for Project vipra-cmd">
+    <!--this file was created by Eclipse Runnable JAR Export Wizard-->
+    <!--ANT 1.7 is required                                        -->
+    <!--define folder properties-->
+    <property name="dir.buildfile" value="."/>
+    <property name="dir.workspace" value="/home/eike/workspace"/>
+    <property name="dir.jarfile" value="${dir.buildfile}/bin"/>
+    <target name="create_run_jar">
+        <jar destfile="${dir.jarfile}/vipra-cmd.jar" filesetmanifest="mergewithoutmain">
+            <manifest>
+                <attribute name="Main-Class" value="de.vipra.cmd.Main"/>
+                <attribute name="Class-Path" value="."/>
+            </manifest>
+            <fileset dir="${dir.buildfile}/target/classes"/>
+            <fileset dir="/home/eike/Repositories/fu/ss15/ma/impl/vipra-util/target/classes"/>
+            <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-api/2.4.1/log4j-api-2.4.1.jar"/>
+            <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-core/2.4.1/log4j-core-2.4.1.jar"/>
+            <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/apache/logging/log4j/log4j-slf4j-impl/2.4.1/log4j-slf4j-impl-2.4.1.jar"/>
+            <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/slf4j/slf4j-api/1.7.12/slf4j-api-1.7.12.jar"/>
+            <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/mongodb-driver/3.0.4/mongodb-driver-3.0.4.jar"/>
+            <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/bson/3.0.4/bson-3.0.4.jar"/>
+            <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/mongodb/mongodb-driver-core/3.0.4/mongodb-driver-core-3.0.4.jar"/>
+            <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/commons-cli/commons-cli/1.3.1/commons-cli-1.3.1.jar"/>
+            <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar"/>
+            <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/com/googlecode/json-simple/json-simple/1.1.1/json-simple-1.1.1.jar"/>
+            <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/junit/junit/4.12/junit-4.12.jar"/>
+            <zipfileset excludes="META-INF/*.SF" src="/home/eike/.m2/repository/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar"/>
+        </jar>
+    </target>
+</project>

vipra-util/build.xml

+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- WARNING: Eclipse auto-generated file.
+              Any modifications will be overwritten.
+              To include a user specific buildfile here, simply create one in the same
+              directory with the processing instruction <?eclipse.ant.import?>
+              as the first entry and export the buildfile again. --><project basedir="." default="build" name="vipra-util">
+    <property environment="env"/>
+    <property name="vipra-cmd.location" value="../vipra-cmd"/>
+    <property name="vipra-rest.location" value="../vipra-rest"/>
+    <property name="ECLIPSE_HOME" value="../../../../../../../../opt/eclipse/"/>
+    <property name="debuglevel" value="source,lines,vars"/>
+    <property name="target" value="1.6"/>
+    <property name="source" value="1.6"/>
+    <path id="Maven Dependencies.libraryclasspath">
+        <pathelement location="../../../../../../.m2/repository/org/apache/logging/log4j/log4j-api/2.4.1/log4j-api-2.4.1.jar"/>
+        <pathelement location="../../../../../../.m2/repository/org/apache/logging/log4j/log4j-core/2.4.1/log4j-core-2.4.1.jar"/>
+        <pathelement location="../../../../../../.m2/repository/org/apache/logging/log4j/log4j-slf4j-impl/2.4.1/log4j-slf4j-impl-2.4.1.jar"/>
+        <pathelement location="../../../../../../.m2/repository/org/slf4j/slf4j-api/1.7.12/slf4j-api-1.7.12.jar"/>
+        <pathelement location="../../../../../../.m2/repository/org/mongodb/mongodb-driver/3.0.4/mongodb-driver-3.0.4.jar"/>
+        <pathelement location="../../../../../../.m2/repository/org/mongodb/bson/3.0.4/bson-3.0.4.jar"/>
+        <pathelement location="../../../../../../.m2/repository/org/mongodb/mongodb-driver-core/3.0.4/mongodb-driver-core-3.0.4.jar"/>
+    </path>
+    <path id="EAR Libraries.libraryclasspath"/>
+    <path id="vipra-util.classpath">
+        <pathelement location="target/classes"/>
+        <path refid="Maven Dependencies.libraryclasspath"/>
+        <path refid="EAR Libraries.libraryclasspath"/>
+    </path>
+    <target name="init">
+        <mkdir dir="target/classes"/>
+        <copy includeemptydirs="false" todir="target/classes">
+            <fileset dir="src/main/java">
+                <exclude name="**/*.launch"/>
+                <exclude name="**/*.java"/>
+            </fileset>
+        </copy>
+        <copy includeemptydirs="false" todir="target/classes">
+            <fileset dir="src/main/resources">
+                <exclude name="**/*.launch"/>
+                <exclude name="**/*.java"/>
+            </fileset>
+        </copy>
+    </target>
+    <target name="clean">
+        <delete dir="target/classes"/>
+    </target>
+    <target depends="clean" name="cleanall"/>
+    <target depends="build-subprojects,build-project" name="build"/>
+    <target name="build-subprojects"/>
+    <target depends="init" name="build-project">
+        <echo message="${ant.project.name}: ${ant.file}"/>
+        <javac debug="true" debuglevel="${debuglevel}" destdir="target/classes" includeantruntime="false" source="${source}" target="${target}">
+            <src path="src/main/java"/>
+            <src path="src/main/resources"/>
+            <classpath refid="vipra-util.classpath"/>
+        </javac>
+    </target>
+    <target description="Build all projects which reference this project. Useful to propagate changes." name="build-refprojects">
+        <ant antfile="build.xml" dir="${vipra-cmd.location}" inheritAll="false" target="clean"/>
+        <ant antfile="build.xml" dir="${vipra-cmd.location}" inheritAll="false" target="build">
+            <propertyset>
+                <propertyref name="build.compiler"/>
+            </propertyset>
+        </ant>
+        <ant antfile="build.xml" dir="${vipra-rest.location}" inheritAll="false" target="clean"/>
+        <ant antfile="build.xml" dir="${vipra-rest.location}" inheritAll="false" target="build">
+            <propertyset>
+                <propertyref name="build.compiler"/>
+            </propertyset>
+        </ant>
+    </target>
+    <target description="copy Eclipse compiler jars to ant lib directory" name="init-eclipse-compiler">
+        <copy todir="${ant.library.dir}">
+            <fileset dir="${ECLIPSE_HOME}/plugins" includes="org.eclipse.jdt.core_*.jar"/>
+        </copy>
+        <unzip dest="${ant.library.dir}">
+            <patternset includes="jdtCompilerAdapter.jar"/>
+            <fileset dir="${ECLIPSE_HOME}/plugins" includes="org.eclipse.jdt.core_*.jar"/>
+        </unzip>
+    </target>
+    <target description="compile project with Eclipse compiler" name="build-eclipse-compiler">
+        <property name="build.compiler" value="org.eclipse.jdt.core.JDTCompilerAdapter"/>
+        <antcall target="build"/>
+    </target>
+</project>

vipra-util/src/main/java/de/vipra/util/StringUtils.java

+package de.vipra.util;
+
+public class StringUtils {
+
+	public static String ellipsize(String input, int maxLength) {
+		String ellip = "...";
+		if (input == null || input.length() <= maxLength || input.length() < ellip.length()) {
+			return input;
+		}
+		return input.substring(0, maxLength - ellip.length()).concat(ellip);
+	}
+
+}

vipra-util/src/main/resources/log4j2.xml

-<?xml version="1.0" encoding="UTF-8"?>
-<Configuration>
-	<Appenders>
-		<Console name="Console" target="SYSTEM_OUT">
-			<PatternLayout pattern="%d{HH:mm:ss.SSS} %-5level %logger{36} - %msg%n" />
-		</Console>
-	</Appenders>
-	<Loggers>
-		<Root level="ALL">
-			<AppenderRef ref="Console" />
-		</Root>
-		<Logger name="org.mongodb" level="INFO"/>
-	</Loggers>
-</Configuration>
\ No newline at end of file

vipra-util/vipra-util.jardesc

 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <jardesc>
-    <jar path="vipra-util/vipra-util.jar"/>
+    <jar path="vipra-util/bin/vipra-util.jar"/>
     <options buildIfNeeded="true" compress="true" descriptionLocation="/vipra-util/vipra-util.jardesc" exportErrors="true" exportWarnings="true" includeDirectoryEntries="false" overwrite="true" saveDescription="true" storeRefactorings="false" useSourceFolders="false"/>
     <storedRefactorings deprecationInfo="true" structuralOnly="false"/>
     <selectedProjects/>