removed entity types, fixed css issues

vipra-cmd/src/main/java/de/vipra/cmd/file/Filebase.java

@@ -77,11 +77,6 @@ public class Filebase {
 				else if (modelConfig.isProcessorUseHypernyms())
 					outModel.write(wordIndex.transform(newArticle.hypernyms()));
 
-				outModel.write(" ");
-
-				if (modelConfig.isProcessorUseEntityTypes())
-					outModel.write(wordIndex.transform(newArticle.types()));
-
 				outModel.write(Constants.LINE_SEP);
 			} else {
 				if (in == null) {

vipra-cmd/src/main/java/de/vipra/cmd/option/EditModelCommand.java

@@ -70,8 +70,6 @@ public class EditModelCommand implements Command {
 				ConsoleUtils.readBoolean(" " + ConsoleUtils.PATH_T + " processor use text", topicModelConfig.isProcessorUseText()));
 		topicModelConfig.setProcessorUseEntities(
 				ConsoleUtils.readBoolean(" " + ConsoleUtils.PATH_T + " processor use entities", topicModelConfig.isProcessorUseEntities()));
-		topicModelConfig.setProcessorUseEntityTypes(
-				ConsoleUtils.readBoolean(" " + ConsoleUtils.PATH_T + " processor use entity types", topicModelConfig.isProcessorUseEntityTypes()));
 		topicModelConfig.setProcessorUseHypernyms(
 				ConsoleUtils.readBoolean(" " + ConsoleUtils.PATH_T + " processor use hypernyms", topicModelConfig.isProcessorUseHypernyms()));
 		topicModelConfig.setQueryEntityDescriptions(

vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java

@@ -333,8 +333,7 @@ public class ImportCommand implements Command {
 		if (config.getSpotlightUrl() != null)
 			spotlightAnalyzer = new SpotlightAnalyzer(modelConfig);
 
-		if ((modelConfig.isProcessorUseEntities() || modelConfig.isProcessorUseEntityTypes() || modelConfig.isProcessorUseHypernyms())
-				&& spotlightAnalyzer == null)
+		if ((modelConfig.isProcessorUseEntities() || modelConfig.isProcessorUseHypernyms()) && spotlightAnalyzer == null)
 			throw new ConfigException("spotlight url is empty, but processor requires entity informations");
 
 		buffer = new ArticleBuffer(dbArticles);

vipra-cmd/src/main/resources/config.json

@@ -25,7 +25,6 @@
 		"windowResolution": "MONTH",
 		"processorUseText": true,
 		"processorUseEntities": true,
-		"processorUseEntityTypes": true,
 		"processorUseHypernyms": true,
 		"queryEntityDescriptions": true
 	}

vipra-ui/app/html/directives/article-link.html

@@ -18,7 +18,7 @@
     <span ng-bind="::articleDetails.text"></span>
     <div>
       <a class="badge topic-badge text-outline" ng-style="{'background':topic.topic.color}" ng-repeat="topic in articleDetails.topics track by topic.topic.id">
-        <topic-link topic="::topic.topic" badge="false" />
+        <topic-link topic="::topic.topic" badge="false" colorbox="false" />
       </a>
     </div>
   </div>

vipra-ui/app/html/directives/topic-link.html

@@ -2,7 +2,7 @@
   <span class="menu-padding ellipsis">
     <div class="pull-right">
       <a ui-sref="topics.show.articles({id:topic.id})" class="badge" ng-bind="::topic.articlesCount" ng-attr-title="{{::topic.articlesCount}} article(s)" ng-if="::showBadge"></a>
-      <span class="colorbox plain shown" style="background:{{::topic.color}}"></span>
+      <span class="colorbox plain shown" style="background:{{::topic.color}}" ng-if="::showColorbox"></span>
     </div>
 	  <a class="topic-link" ui-sref="topics.show({id:topic.id})" ng-attr-title="{{topic.name}}">
 	    <span class="title" ng-bind="topic.name"></span>

vipra-ui/app/js/directives.js

@@ -16,7 +16,8 @@
       scope: {
         topic: '=',
         badge: '@',
-        menu: '@'
+        menu: '@',
+        colorbox: '@'
       },
       restrict: 'E',
       replace: true,
@@ -25,6 +26,7 @@
       link: function($scope) {
         $scope.showBadge = $scope.badge !== 'false';
         $scope.showMenu = $scope.menu !== 'false';
+        $scope.showColorbox = $scope.colorbox !== 'false';
       }
     };
   }]);

vipra-ui/app/less/app.less

@@ -823,27 +823,6 @@ entity-menu {
   display: block !important;
 }
 
-.char-selector {
-  @char-padding: 5px;
-  padding: 5px;
-  text-align: center;
-  font-size: 0;
-  > a {
-    font-size: 14px;
-    padding-right: @char-padding;
-  }
-  > a:last-child {
-    padding-right: 0;
-  }
-  > a + a {
-    padding-left: @char-padding;
-    border-left: 1px solid #ccc;
-  }
-  > a.selected {
-    text-decoration: underline;
-  }
-}
-
 .nooutline:focus {
   outline: none;
 }
@@ -998,10 +977,6 @@ entity-menu {
   color: #000;
 }
 
-.topic.active-topic .title {
-  text-decoration: underline;
-}
-
 .button-pagination {
   margin: 5px 0 0 0;
   .current,

vipra-util/src/main/java/de/vipra/util/Constants.java

@@ -185,73 +185,43 @@ public class Constants {
 	 * Stopwords list. Extensive list of stopwords used to clean imported
 	 * articles of the most common words before topic modeling is applied.
 	 */
-	public static final List<String> STOPWORDS = Arrays.asList("'ll", "'ve", "a", "a's", "able", "about", "above", "abst", "accordance", "according",
-			"accordingly", "across", "act", "actually", "added", "adj", "affected", "affecting", "affects", "after", "afterwards", "again", "against",
-			"ah", "ain't", "aint", "all", "allow", "allows", "almost", "alone", "along", "already", "also", "although", "always", "am", "among",
-			"amongst", "amoungst", "amount", "an", "and", "announce", "another", "any", "anybody", "anyhow", "anymore", "anyone", "anything",
-			"anyway", "anyways", "anywhere", "apart", "apparently", "appear", "appreciate", "appropriate", "approximately", "are", "area", "areas",
-			"aren", "aren't", "arent", "arise", "around", "as", "aside", "ask", "asked", "asking", "asks", "associated", "at", "auth", "available",
-			"away", "awfully", "b", "back", "backed", "backing", "backs", "be", "became", "because", "become", "becomes", "becoming", "been",
-			"before", "beforehand", "began", "begin", "beginning", "beginnings", "begins", "behind", "being", "beings", "believe", "below", "beside",
-			"besides", "best", "better", "between", "beyond", "big", "bill", "biol", "both", "bottom", "brief", "briefly", "but", "by", "c", "c'mon",
-			"c's", "ca", "call", "came", "can", "can't", "cannot", "cant", "case", "cases", "cause", "causes", "certain", "certainly", "changes",
-			"clear", "clearly", "cmon", "co", "com", "come", "comes", "computer", "con", "concerning", "consequently", "consider", "considering",
-			"contain", "containing", "contains", "corresponding", "could", "couldn't", "couldnt", "course", "cry", "cs", "currently", "d", "date",
-			"de", "definitely", "describe", "described", "despite", "detail", "did", "didn't", "didnt", "differ", "different", "differently", "do",
-			"does", "doesn't", "doesnt", "doing", "don't", "done", "dont", "down", "downed", "downing", "downs", "downwards", "due", "during", "e",
-			"each", "early", "ed", "edu", "effect", "eg", "eight", "eighty", "either", "eleven", "else", "elsewhere", "empty", "end", "ended",
-			"ending", "ends", "enough", "entirely", "especially", "et", "et-al", "etc", "even", "evenly", "ever", "every", "everybody", "everyone",
-			"everything", "everywhere", "ex", "exactly", "example", "except", "f", "face", "faces", "fact", "facts", "far", "felt", "few", "ff",
-			"fifteen", "fifth", "fify", "fill", "find", "finds", "fire", "first", "five", "fix", "followed", "following", "follows", "for", "former",
-			"formerly", "forth", "forty", "found", "four", "from", "front", "full", "fully", "further", "furthered", "furthering", "furthermore",
-			"furthers", "g", "gave", "general", "generally", "get", "gets", "getting", "give", "given", "gives", "giving", "go", "goes", "going",
-			"gone", "good", "goods", "got", "gotten", "great", "greater", "greatest", "greetings", "group", "grouped", "grouping", "groups", "h",
-			"had", "hadn't", "hadnt", "happens", "hardly", "has", "hasn't", "hasnt", "have", "haven't", "havent", "having", "he", "he'd", "he'll",
-			"he's", "hed", "hello", "help", "hence", "her", "here", "here's", "hereafter", "hereby", "herein", "heres", "hereupon", "hers", "herse",
-			"herself", "hes", "hi", "hid", "high", "higher", "highest", "him", "himse", "himself", "his", "hither", "home", "hopefully", "how",
-			"how's", "howbeit", "however", "hundred", "i", "i'd", "i'll", "i'm", "i've", "id", "ie", "if", "ignored", "ill", "im", "immediate",
-			"immediately", "importance", "important", "in", "inasmuch", "inc", "indeed", "index", "indicate", "indicated", "indicates", "information",
-			"inner", "insofar", "instead", "interest", "interested", "interesting", "interests", "into", "invention", "inward", "is", "isn't", "isnt",
-			"it", "it'd", "it'll", "it's", "itd", "itll", "its", "itse", "itself", "ive", "j", "just", "k", "keep", "keeps", "kept", "kg", "kind",
-			"km", "knew", "know", "known", "knows", "l", "large", "largely", "last", "lately", "later", "latest", "latter", "latterly", "least",
-			"less", "lest", "let", "let's", "lets", "like", "liked", "likely", "line", "little", "long", "longer", "longest", "look", "looking",
-			"looks", "ltd", "m", "made", "mainly", "make", "makes", "making", "man", "many", "may", "maybe", "me", "mean", "means", "meantime",
-			"meanwhile", "member", "members", "men", "merely", "mg", "might", "mill", "million", "mine", "miss", "ml", "more", "moreover", "most",
-			"mostly", "move", "mr", "mrs", "much", "mug", "must", "mustn't", "my", "myse", "myself", "n", "na", "name", "namely", "nay", "nd", "near",
-			"nearly", "necessarily", "necessary", "need", "needed", "needing", "needs", "neither", "never", "nevertheless", "new", "newer", "newest",
-			"next", "nine", "ninety", "no", "nobody", "non", "none", "nonetheless", "noone", "nor", "normally", "nos", "not", "noted", "nothing",
-			"novel", "now", "nowhere", "number", "numbers", "o", "obtain", "obtained", "obviously", "of", "off", "often", "oh", "ok", "okay", "old",
-			"older", "oldest", "omitted", "on", "once", "one", "ones", "only", "onto", "open", "opened", "opening", "opens", "or", "ord", "order",
-			"ordered", "ordering", "orders", "other", "others", "otherwise", "ought", "our", "ours", "ourselves", "out", "outside", "over", "overall",
-			"owing", "own", "p", "page", "pages", "part", "parted", "particular", "particularly", "parting", "parts", "past", "per", "perhaps",
-			"place", "placed", "places", "please", "plus", "point", "pointed", "pointing", "points", "poorly", "possible", "possibly", "potentially",
-			"pp", "predominantly", "present", "presented", "presenting", "presents", "presumably", "previously", "primarily", "probably", "problem",
-			"problems", "promptly", "proud", "provides", "put", "puts", "q", "que", "quickly", "quite", "qv", "r", "ran", "rather", "rd", "re",
-			"readily", "really", "reasonably", "recent", "recently", "ref", "refs", "regarding", "regardless", "regards", "related", "relatively",
-			"research", "respectively", "resulted", "resulting", "results", "right", "room", "rooms", "run", "s", "said", "same", "saw", "say",
-			"saying", "says", "sec", "second", "secondly", "seconds", "section", "see", "seeing", "seem", "seemed", "seeming", "seems", "seen",
-			"sees", "self", "selves", "sensible", "sent", "serious", "seriously", "seven", "several", "shall", "shan't", "she", "she'd", "she'll",
-			"she's", "shed", "shes", "should", "shouldn't", "shouldnt", "show", "showed", "showing", "shown", "showns", "shows", "side", "sides",
-			"significant", "significantly", "similar", "similarly", "since", "sincere", "six", "sixty", "slightly", "small", "smaller", "smallest",
-			"so", "some", "somebody", "somehow", "someone", "somethan", "something", "sometime", "sometimes", "somewhat", "somewhere", "soon",
-			"sorry", "specifically", "specified", "specify", "specifying", "state", "states", "still", "stop", "strongly", "sub", "substantially",
-			"successfully", "such", "such as", "sufficiently", "suggest", "sup", "sure", "system", "t", "t's", "take", "taken", "taking", "tell",
-			"ten", "tends", "th", "than", "thank", "thanks", "thanx", "that", "that'll", "that's", "that've", "thats", "the", "their", "theirs",
-			"them", "themselves", "then", "thence", "there", "there'll", "there's", "there've", "thereafter", "thereby", "thered", "therefore",
-			"therein", "thereof", "therere", "theres", "thereto", "thereupon", "these", "they", "they'd", "they'll", "they're", "they've", "theyd",
-			"theyll", "theyre", "theyve", "thick", "thin", "thing", "things", "think", "thinks", "third", "this", "thorough", "thoroughly", "those",
-			"thou", "though", "thoughh", "thought", "thoughts", "thousand", "three", "throug", "through", "throughout", "thru", "thus", "til", "tip",
-			"to", "today", "together", "too", "took", "top", "toward", "towards", "tried", "tries", "truly", "try", "trying", "ts", "turn", "turned",
-			"turning", "turns", "twelve", "twenty", "twice", "two", "u", "un", "under", "unfortunately", "unless", "unlike", "unlikely", "until",
-			"unto", "up", "upon", "ups", "us", "use", "used", "useful", "usefully", "usefulness", "uses", "using", "usually", "uucp", "v", "value",
-			"various", "very", "via", "viz", "vol", "vols", "vs", "w", "want", "wanted", "wanting", "wants", "was", "wasn't", "wasnt", "way", "ways",
-			"we", "we'd", "we'll", "we're", "we've", "wed", "welcome", "well", "wells", "went", "were", "weren't", "werent", "weve", "what",
-			"what'll", "what's", "whatever", "whats", "when", "when's", "whence", "whenever", "where", "where's", "whereafter", "whereas", "whereby",
-			"wherein", "wheres", "whereupon", "wherever", "whether", "which", "while", "whim", "whither", "who", "who'll", "who's", "whod", "whoever",
-			"whole", "whom", "whomever", "whos", "whose", "why", "why's", "widely", "will", "willing", "wish", "with", "within", "without", "won't",
-			"wonder", "wont", "words", "work", "worked", "working", "works", "world", "would", "wouldn't", "wouldnt", "www", "x", "y", "year",
-			"years", "yes", "yet", "you", "you'd", "you'll", "you're", "you've", "youd", "youll", "young", "younger", "youngest", "your", "youre",
-			"yours", "yourself", "yourselves", "youve", "z", "zero");
+	public static final List<String> STOPWORDS = Arrays.asList("a", "able", "about", "above", "according", "accordingly", "across", "actually",
+			"after", "afterwards", "again", "against", "all", "allow", "allows", "almost", "alone", "along", "already", "also", "although", "always",
+			"am", "among", "amongst", "an", "and", "another", "any", "anybody", "anyhow", "anyone", "anything", "anyway", "anyways", "anywhere",
+			"apart", "appear", "appreciate", "appropriate", "are", "around", "as", "aside", "ask", "asking", "associated", "at", "available", "away",
+			"awfully", "b", "be", "became", "because", "become", "becomes", "becoming", "been", "before", "beforehand", "behind", "being", "believe",
+			"below", "beside", "besides", "best", "better", "between", "beyond", "both", "brief", "but", "by", "c", "came", "can", "cannot", "cant",
+			"cause", "causes", "certain", "certainly", "changes", "clearly", "co", "com", "come", "comes", "concerning", "consequently", "consider",
+			"considering", "contain", "containing", "contains", "corresponding", "could", "course", "currently", "d", "definitely", "described",
+			"despite", "did", "different", "do", "does", "doing", "done", "down", "downwards", "during", "e", "each", "edu", "eg", "eight", "either",
+			"else", "elsewhere", "enough", "entirely", "especially", "et", "etc", "even", "ever", "every", "everybody", "everyone", "everything",
+			"everywhere", "ex", "exactly", "example", "except", "f", "far", "few", "fifth", "first", "five", "followed", "following", "follows",
+			"for", "former", "formerly", "forth", "four", "from", "further", "furthermore", "g", "get", "gets", "getting", "given", "gives", "go",
+			"goes", "going", "gone", "got", "gotten", "greetings", "h", "had", "happens", "hardly", "has", "have", "having", "he", "hello", "help",
+			"hence", "her", "here", "hereafter", "hereby", "herein", "hereupon", "hers", "herself", "hi", "him", "himself", "his", "hither",
+			"hopefully", "how", "howbeit", "however", "i", "ie", "if", "ignored", "immediate", "in", "inasmuch", "inc", "indeed", "indicate",
+			"indicated", "indicates", "inner", "insofar", "instead", "into", "inward", "is", "it", "its", "itself", "j", "just", "k", "keep", "keeps",
+			"kept", "know", "knows", "known", "l", "last", "lately", "later", "latter", "latterly", "least", "less", "lest", "let", "like", "liked",
+			"likely", "little", "look", "looking", "looks", "ltd", "m", "mainly", "many", "may", "maybe", "me", "mean", "meanwhile", "merely",
+			"might", "more", "moreover", "most", "mostly", "much", "must", "my", "myself", "n", "name", "namely", "nd", "near", "nearly", "necessary",
+			"need", "needs", "neither", "never", "nevertheless", "new", "next", "nine", "no", "nobody", "non", "none", "noone", "nor", "normally",
+			"not", "nothing", "novel", "now", "nowhere", "o", "obviously", "of", "off", "often", "oh", "ok", "okay", "old", "on", "once", "one",
+			"ones", "only", "onto", "or", "other", "others", "otherwise", "ought", "our", "ours", "ourselves", "out", "outside", "over", "overall",
+			"own", "p", "particular", "particularly", "per", "perhaps", "placed", "please", "plus", "possible", "presumably", "probably", "provides",
+			"q", "que", "quite", "qv", "r", "rather", "rd", "re", "really", "reasonably", "regarding", "regardless", "regards", "relatively",
+			"respectively", "right", "s", "said", "same", "saw", "say", "saying", "says", "second", "secondly", "see", "seeing", "seem", "seemed",
+			"seeming", "seems", "seen", "self", "selves", "sensible", "sent", "serious", "seriously", "seven", "several", "shall", "she", "should",
+			"since", "six", "so", "some", "somebody", "somehow", "someone", "something", "sometime", "sometimes", "somewhat", "somewhere", "soon",
+			"sorry", "specified", "specify", "specifying", "still", "sub", "such", "sup", "sure", "t", "take", "taken", "tell", "tends", "th", "than",
+			"thank", "thanks", "thanx", "that", "thats", "the", "their", "theirs", "them", "themselves", "then", "thence", "there", "thereafter",
+			"thereby", "therefore", "therein", "theres", "thereupon", "these", "they", "think", "third", "this", "thorough", "thoroughly", "those",
+			"though", "three", "through", "throughout", "thru", "thus", "to", "together", "too", "took", "toward", "towards", "tried", "tries",
+			"truly", "try", "trying", "twice", "two", "u", "un", "under", "unfortunately", "unless", "unlikely", "until", "unto", "up", "upon", "us",
+			"use", "used", "useful", "uses", "using", "usually", "uucp", "v", "value", "various", "very", "via", "viz", "vs", "w", "want", "wants",
+			"was", "way", "we", "welcome", "well", "went", "were", "what", "whatever", "when", "whence", "whenever", "where", "whereafter", "whereas",
+			"whereby", "wherein", "whereupon", "wherever", "whether", "which", "while", "whither", "who", "whoever", "whole", "whom", "whose", "why",
+			"will", "willing", "wish", "with", "within", "without", "wonder", "would", "would", "x", "y", "yes", "yet", "you", "your", "yours",
+			"yourself", "yourselves", "z", "zero");
 
 	/**
 	 * Disallowed chars for words in processed text segments. This regular

vipra-util/src/main/java/de/vipra/util/model/SequenceWord.java

@@ -41,7 +41,8 @@ public class SequenceWord implements Comparable<SequenceWord>, Serializable {
 
 	@Override
 	public int compareTo(final SequenceWord o) {
-		return Double.compare(probability, o.getProbability());
+		int c1 = Double.compare(probability, o.getProbability());
+		return c1 == 0 ? word.compareTo(o.getWord()) : c1;
 	}
 
 	@Override

vipra-util/src/main/java/de/vipra/util/model/TopicModelConfig.java

@@ -46,7 +46,6 @@ public class TopicModelConfig implements Serializable {
 	private double maxSimilarTopicsDivergence = Constants.MAX_SIMILAR_TOPICS_DIVERGENCE;
 	private boolean processorUseText = Constants.PROCESSOR_USE_TEXT;
 	private boolean processorUseEntities = Constants.PROCESSOR_USE_ENTITIES;
-	private boolean processorUseEntityTypes = Constants.PROCESSOR_USE_ENTITY_TYPES;
 	private boolean processorUseHypernyms = Constants.PROCESSOR_USE_HYPERNYMS;
 	private boolean queryEntityDescriptions = Constants.QUERY_ENTITY_DESCRIPTIONS;
 	private WindowResolution windowResolution = Constants.WINDOW_RESOLUTION;
@@ -73,7 +72,6 @@ public class TopicModelConfig implements Serializable {
 		maxSimilarTopicsDivergence = topicModelConfig.getMaxSimilarTopicsDivergence();
 		processorUseText = topicModelConfig.isProcessorUseText();
 		processorUseEntities = topicModelConfig.isProcessorUseEntities();
-		processorUseEntityTypes = topicModelConfig.isProcessorUseEntityTypes();
 		processorUseHypernyms = topicModelConfig.isProcessorUseHypernyms();
 		queryEntityDescriptions = topicModelConfig.isQueryEntityDescriptions();
 		windowResolution = topicModelConfig.getWindowResolution();
@@ -255,14 +253,6 @@ public class TopicModelConfig implements Serializable {
 		this.processorUseEntities = processorUseEntities;
 	}
 
-	public boolean isProcessorUseEntityTypes() {
-		return processorUseEntityTypes;
-	}
-
-	public void setProcessorUseEntityTypes(final boolean processorUseEntityTypes) {
-		this.processorUseEntityTypes = processorUseEntityTypes;
-	}
-
 	public boolean isProcessorUseHypernyms() {
 		return processorUseHypernyms;
 	}
@@ -318,9 +308,8 @@ public class TopicModelConfig implements Serializable {
 				+ "\n spotlightConfidence: " + spotlightConfidence + "\n minTopicShare: " + minTopicShare + "\n minRelativeProbability: "
 				+ minRelativeProbability + "\n risingDecayLambda: " + risingDecayLambda + "\n maxSimilarDocumentsDivergence: "
 				+ maxSimilarDocumentsDivergence + "\n maxSimilarTopicsDivergence: " + maxSimilarTopicsDivergence + "\n processor use text: "
-				+ processorUseText + "\n processor use entities: " + processorUseEntities + "\n processor use entity types: "
-				+ processorUseEntityTypes + "\n processor use hypernyms: " + processorUseHypernyms + "\n query entity descriptions: "
-				+ queryEntityDescriptions + "\n windowResolution: " + windowResolution;
+				+ processorUseText + "\n processor use entities: " + processorUseEntities + "\n processor use hypernyms: " + processorUseHypernyms
+				+ "\n query entity descriptions: " + queryEntityDescriptions + "\n windowResolution: " + windowResolution;
 	}
 
 }