diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/lda/DTMAnalyzer.java b/vipra-cmd/src/main/java/de/vipra/cmd/lda/DTMAnalyzer.java
index 5c9eeef9e68349cf1f1354a07d72030323c43ce9..c209d580ee4e2063be5e739edb39136a946b2850 100644
--- a/vipra-cmd/src/main/java/de/vipra/cmd/lda/DTMAnalyzer.java
+++ b/vipra-cmd/src/main/java/de/vipra/cmd/lda/DTMAnalyzer.java
@@ -188,6 +188,7 @@ public class DTMAnalyzer extends Analyzer {
newWindow.setId(idxSeq);
newWindow.setStartDate(seqindex.getStartDate(idxSeq));
newWindow.setEndDate(seqindex.getEndDate(idxSeq));
+ newWindow.setWindowResolution(Constants.WINDOW_RESOLUTION);
newWindows.add(newWindow);
}
diff --git a/vipra-util/src/main/java/de/vipra/util/Constants.java b/vipra-util/src/main/java/de/vipra/util/Constants.java
index e7a1d30b11d8796c84948c69c3488d778afd9856..dac5d0e553e81c68374b0f3930076434489870c8 100644
--- a/vipra-util/src/main/java/de/vipra/util/Constants.java
+++ b/vipra-util/src/main/java/de/vipra/util/Constants.java
@@ -7,6 +7,7 @@ import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.List;
+import java.util.TimeZone;
public class Constants {
@@ -137,10 +138,10 @@ public class Constants {
*/
public static final List<String> STOPWORDS = Arrays.asList("'ll", "'ve", "a", "a's", "able", "about", "above",
"abst", "accordance", "according", "accordingly", "across", "act", "actually", "added", "adj", "affected",
- "affecting", "affects", "after", "afterwards", "again", "against", "ah", "ain't", "all", "allow", "allows",
- "almost", "alone", "along", "already", "also", "although", "always", "am", "among", "amongst", "amoungst",
- "amount", "an", "and", "announce", "another", "any", "anybody", "anyhow", "anymore", "anyone", "anything",
- "anyway", "anyways", "anywhere", "apart", "apparently", "appear", "appreciate", "appropriate",
+ "affecting", "affects", "after", "afterwards", "again", "against", "ah", "ain't", "aint", "all", "allow",
+ "allows", "almost", "alone", "along", "already", "also", "although", "always", "am", "among", "amongst",
+ "amoungst", "amount", "an", "and", "announce", "another", "any", "anybody", "anyhow", "anymore", "anyone",
+ "anything", "anyway", "anyways", "anywhere", "apart", "apparently", "appear", "appreciate", "appropriate",
"approximately", "are", "area", "areas", "aren", "aren't", "arent", "arise", "around", "as", "aside", "ask",
"asked", "asking", "asks", "associated", "at", "auth", "available", "away", "awfully", "b", "back",
"backed", "backing", "backs", "be", "became", "because", "become", "becomes", "becoming", "been", "before",
@@ -148,40 +149,41 @@ public class Constants {
"below", "beside", "besides", "best", "better", "between", "beyond", "big", "bill", "biol", "both",
"bottom", "brief", "briefly", "but", "by", "c", "c'mon", "c's", "ca", "call", "came", "can", "can't",
"cannot", "cant", "case", "cases", "cause", "causes", "certain", "certainly", "changes", "clear", "clearly",
- "co", "com", "come", "comes", "computer", "con", "concerning", "consequently", "consider", "considering",
- "contain", "containing", "contains", "corresponding", "could", "couldn't", "couldnt", "course", "cry",
- "currently", "d", "date", "de", "definitely", "describe", "described", "despite", "detail", "did", "didn't",
- "differ", "different", "differently", "do", "does", "doesn't", "doing", "don't", "done", "down", "downed",
- "downing", "downs", "downwards", "due", "during", "e", "each", "early", "ed", "edu", "effect", "eg",
- "eight", "eighty", "either", "eleven", "else", "elsewhere", "empty", "end", "ended", "ending", "ends",
- "enough", "entirely", "especially", "et", "et-al", "etc", "even", "evenly", "ever", "every", "everybody",
- "everyone", "everything", "everywhere", "ex", "exactly", "example", "except", "f", "face", "faces", "fact",
- "facts", "far", "felt", "few", "ff", "fifteen", "fifth", "fify", "fill", "find", "finds", "fire", "first",
- "five", "fix", "followed", "following", "follows", "for", "former", "formerly", "forth", "forty", "found",
- "four", "from", "front", "full", "fully", "further", "furthered", "furthering", "furthermore", "furthers",
- "g", "gave", "general", "generally", "get", "gets", "getting", "give", "given", "gives", "giving", "go",
- "goes", "going", "gone", "good", "goods", "got", "gotten", "great", "greater", "greatest", "greetings",
- "group", "grouped", "grouping", "groups", "h", "had", "hadn't", "happens", "hardly", "has", "hasn't",
- "hasnt", "have", "haven't", "having", "he", "he'd", "he'll", "he's", "hed", "hello", "help", "hence", "her",
- "here", "here's", "hereafter", "hereby", "herein", "heres", "hereupon", "hers", "herse", "herself", "hes",
- "hi", "hid", "high", "higher", "highest", "him", "himse", "himself", "his", "hither", "home", "hopefully",
- "how", "how's", "howbeit", "however", "hundred", "i", "i'd", "i'll", "i'm", "i've", "id", "ie", "if",
- "ignored", "im", "immediate", "immediately", "importance", "important", "in", "inasmuch", "inc", "indeed",
- "index", "indicate", "indicated", "indicates", "information", "inner", "insofar", "instead", "interest",
- "interested", "interesting", "interests", "into", "invention", "inward", "is", "isn't", "it", "it'd",
- "it'll", "it's", "itd", "its", "itse", "itself", "j", "just", "k", "keep", "keeps", "kept", "kg", "kind",
- "km", "knew", "know", "known", "knows", "l", "large", "largely", "last", "lately", "later", "latest",
- "latter", "latterly", "least", "less", "lest", "let", "let's", "lets", "like", "liked", "likely", "line",
- "little", "long", "longer", "longest", "look", "looking", "looks", "ltd", "m", "made", "mainly", "make",
- "makes", "making", "man", "many", "may", "maybe", "me", "mean", "means", "meantime", "meanwhile", "member",
- "members", "men", "merely", "mg", "might", "mill", "million", "mine", "miss", "ml", "more", "moreover",
- "most", "mostly", "move", "mr", "mrs", "much", "mug", "must", "mustn't", "my", "myse", "myself", "n", "na",
- "name", "namely", "nay", "nd", "near", "nearly", "necessarily", "necessary", "need", "needed", "needing",
- "needs", "neither", "never", "nevertheless", "new", "newer", "newest", "next", "nine", "ninety", "no",
- "nobody", "non", "none", "nonetheless", "noone", "nor", "normally", "nos", "not", "noted", "nothing",
- "novel", "now", "nowhere", "number", "numbers", "o", "obtain", "obtained", "obviously", "of", "off",
- "often", "oh", "ok", "okay", "old", "older", "oldest", "omitted", "on", "once", "one", "ones", "only",
- "onto", "open", "opened", "opening", "opens", "or", "ord", "order", "ordered", "ordering", "orders",
+ "cmon", "co", "com", "come", "comes", "computer", "con", "concerning", "consequently", "consider",
+ "considering", "contain", "containing", "contains", "corresponding", "could", "couldn't", "couldnt",
+ "course", "cry", "cs", "currently", "d", "date", "de", "definitely", "describe", "described", "despite",
+ "detail", "did", "didn't", "didnt", "differ", "different", "differently", "do", "does", "doesn't", "doesnt",
+ "doing", "don't", "done", "dont", "down", "downed", "downing", "downs", "downwards", "due", "during", "e",
+ "each", "early", "ed", "edu", "effect", "eg", "eight", "eighty", "either", "eleven", "else", "elsewhere",
+ "empty", "end", "ended", "ending", "ends", "enough", "entirely", "especially", "et", "et-al", "etc", "even",
+ "evenly", "ever", "every", "everybody", "everyone", "everything", "everywhere", "ex", "exactly", "example",
+ "except", "f", "face", "faces", "fact", "facts", "far", "felt", "few", "ff", "fifteen", "fifth", "fify",
+ "fill", "find", "finds", "fire", "first", "five", "fix", "followed", "following", "follows", "for",
+ "former", "formerly", "forth", "forty", "found", "four", "from", "front", "full", "fully", "further",
+ "furthered", "furthering", "furthermore", "furthers", "g", "gave", "general", "generally", "get", "gets",
+ "getting", "give", "given", "gives", "giving", "go", "goes", "going", "gone", "good", "goods", "got",
+ "gotten", "great", "greater", "greatest", "greetings", "group", "grouped", "grouping", "groups", "h", "had",
+ "hadn't", "hadnt", "happens", "hardly", "has", "hasn't", "hasnt", "have", "haven't", "havent", "having",
+ "he", "he'd", "he'll", "he's", "hed", "hello", "help", "hence", "her", "here", "here's", "hereafter",
+ "hereby", "herein", "heres", "hereupon", "hers", "herse", "herself", "hes", "hi", "hid", "high", "higher",
+ "highest", "him", "himse", "himself", "his", "hither", "home", "hopefully", "how", "how's", "howbeit",
+ "however", "hundred", "i", "i'd", "i'll", "i'm", "i've", "id", "ie", "if", "ignored", "ill", "im",
+ "immediate", "immediately", "importance", "important", "in", "inasmuch", "inc", "indeed", "index",
+ "indicate", "indicated", "indicates", "information", "inner", "insofar", "instead", "interest",
+ "interested", "interesting", "interests", "into", "invention", "inward", "is", "isn't", "isnt", "it",
+ "it'd", "it'll", "it's", "itd", "itll", "its", "itse", "itself", "ive", "j", "just", "k", "keep", "keeps",
+ "kept", "kg", "kind", "km", "knew", "know", "known", "knows", "l", "large", "largely", "last", "lately",
+ "later", "latest", "latter", "latterly", "least", "less", "lest", "let", "let's", "lets", "like", "liked",
+ "likely", "line", "little", "long", "longer", "longest", "look", "looking", "looks", "ltd", "m", "made",
+ "mainly", "make", "makes", "making", "man", "many", "may", "maybe", "me", "mean", "means", "meantime",
+ "meanwhile", "member", "members", "men", "merely", "mg", "might", "mill", "million", "mine", "miss", "ml",
+ "more", "moreover", "most", "mostly", "move", "mr", "mrs", "much", "mug", "must", "mustn't", "my", "myse",
+ "myself", "n", "na", "name", "namely", "nay", "nd", "near", "nearly", "necessarily", "necessary", "need",
+ "needed", "needing", "needs", "neither", "never", "nevertheless", "new", "newer", "newest", "next", "nine",
+ "ninety", "no", "nobody", "non", "none", "nonetheless", "noone", "nor", "normally", "nos", "not", "noted",
+ "nothing", "novel", "now", "nowhere", "number", "numbers", "o", "obtain", "obtained", "obviously", "of",
+ "off", "often", "oh", "ok", "okay", "old", "older", "oldest", "omitted", "on", "once", "one", "ones",
+ "only", "onto", "open", "opened", "opening", "opens", "or", "ord", "order", "ordered", "ordering", "orders",
"other", "others", "otherwise", "ought", "our", "ours", "ourselves", "out", "outside", "over", "overall",
"owing", "own", "p", "page", "pages", "part", "parted", "particular", "particularly", "parting", "parts",
"past", "per", "perhaps", "place", "placed", "places", "please", "plus", "point", "pointed", "pointing",
@@ -193,33 +195,33 @@ public class Constants {
"results", "right", "room", "rooms", "run", "s", "said", "same", "saw", "say", "saying", "says", "sec",
"second", "secondly", "seconds", "section", "see", "seeing", "seem", "seemed", "seeming", "seems", "seen",
"sees", "self", "selves", "sensible", "sent", "serious", "seriously", "seven", "several", "shall", "shan't",
- "she", "she'd", "she'll", "she's", "shed", "shes", "should", "shouldn't", "show", "showed", "showing",
- "shown", "showns", "shows", "side", "sides", "significant", "significantly", "similar", "similarly",
- "since", "sincere", "six", "sixty", "slightly", "small", "smaller", "smallest", "so", "some", "somebody",
- "somehow", "someone", "somethan", "something", "sometime", "sometimes", "somewhat", "somewhere", "soon",
- "sorry", "specifically", "specified", "specify", "specifying", "state", "states", "still", "stop",
- "strongly", "sub", "substantially", "successfully", "such", "sufficiently", "suggest", "sup", "sure",
- "system", "t", "t's", "take", "taken", "taking", "tell", "ten", "tends", "th", "than", "thank", "thanks",
- "thanx", "that", "that'll", "that's", "that've", "thats", "the", "their", "theirs", "them", "themselves",
- "then", "thence", "there", "there'll", "there's", "there've", "thereafter", "thereby", "thered",
- "therefore", "therein", "thereof", "therere", "theres", "thereto", "thereupon", "these", "they", "they'd",
- "they'll", "they're", "they've", "theyd", "theyre", "thick", "thin", "thing", "things", "think", "thinks",
- "third", "this", "thorough", "thoroughly", "those", "thou", "though", "thoughh", "thought", "thoughts",
- "thousand", "three", "throug", "through", "throughout", "thru", "thus", "til", "tip", "to", "today",
- "together", "too", "took", "top", "toward", "towards", "tried", "tries", "truly", "try", "trying", "ts",
- "turn", "turned", "turning", "turns", "twelve", "twenty", "twice", "two", "u", "un", "under",
- "unfortunately", "unless", "unlike", "unlikely", "until", "unto", "up", "upon", "ups", "us", "use", "used",
- "useful", "usefully", "usefulness", "uses", "using", "usually", "v", "value", "various", "very", "via",
- "viz", "vol", "vols", "vs", "w", "want", "wanted", "wanting", "wants", "was", "wasn't", "wasnt", "way",
- "ways", "we", "we'd", "we'll", "we're", "we've", "wed", "welcome", "well", "wells", "went", "were",
- "weren't", "werent", "what", "what'll", "what's", "whatever", "whats", "when", "when's", "whence",
- "whenever", "where", "where's", "whereafter", "whereas", "whereby", "wherein", "wheres", "whereupon",
- "wherever", "whether", "which", "while", "whim", "whither", "who", "who'll", "who's", "whod", "whoever",
- "whole", "whom", "whomever", "whos", "whose", "why", "why's", "widely", "will", "willing", "wish", "with",
- "within", "without", "won't", "wonder", "wont", "words", "work", "worked", "working", "works", "world",
- "would", "wouldn't", "wouldnt", "www", "x", "y", "year", "years", "yes", "yet", "you", "you'd", "you'll",
- "you're", "you've", "youd", "young", "younger", "youngest", "your", "youre", "yours", "yourself",
- "yourselves", "z", "zero");
+ "she", "she'd", "she'll", "she's", "shed", "shes", "should", "shouldn't", "shouldnt", "show", "showed",
+ "showing", "shown", "showns", "shows", "side", "sides", "significant", "significantly", "similar",
+ "similarly", "since", "sincere", "six", "sixty", "slightly", "small", "smaller", "smallest", "so", "some",
+ "somebody", "somehow", "someone", "somethan", "something", "sometime", "sometimes", "somewhat", "somewhere",
+ "soon", "sorry", "specifically", "specified", "specify", "specifying", "state", "states", "still", "stop",
+ "strongly", "sub", "substantially", "successfully", "such", "such as", "sufficiently", "suggest", "sup",
+ "sure", "system", "t", "t's", "take", "taken", "taking", "tell", "ten", "tends", "th", "than", "thank",
+ "thanks", "thanx", "that", "that'll", "that's", "that've", "thats", "the", "their", "theirs", "them",
+ "themselves", "then", "thence", "there", "there'll", "there's", "there've", "thereafter", "thereby",
+ "thered", "therefore", "therein", "thereof", "therere", "theres", "thereto", "thereupon", "these", "they",
+ "they'd", "they'll", "they're", "they've", "theyd", "theyll", "theyre", "theyve", "thick", "thin", "thing",
+ "things", "think", "thinks", "third", "this", "thorough", "thoroughly", "those", "thou", "though",
+ "thoughh", "thought", "thoughts", "thousand", "three", "throug", "through", "throughout", "thru", "thus",
+ "til", "tip", "to", "today", "together", "too", "took", "top", "toward", "towards", "tried", "tries",
+ "truly", "try", "trying", "ts", "turn", "turned", "turning", "turns", "twelve", "twenty", "twice", "two",
+ "u", "un", "under", "unfortunately", "unless", "unlike", "unlikely", "until", "unto", "up", "upon", "ups",
+ "us", "use", "used", "useful", "usefully", "usefulness", "uses", "using", "usually", "uucp", "v", "value",
+ "various", "very", "via", "viz", "vol", "vols", "vs", "w", "want", "wanted", "wanting", "wants", "was",
+ "wasn't", "wasnt", "way", "ways", "we", "we'd", "we'll", "we're", "we've", "wed", "welcome", "well",
+ "wells", "went", "were", "weren't", "werent", "weve", "what", "what'll", "what's", "whatever", "whats",
+ "when", "when's", "whence", "whenever", "where", "where's", "whereafter", "whereas", "whereby", "wherein",
+ "wheres", "whereupon", "wherever", "whether", "which", "while", "whim", "whither", "who", "who'll", "who's",
+ "whod", "whoever", "whole", "whom", "whomever", "whos", "whose", "why", "why's", "widely", "will",
+ "willing", "wish", "with", "within", "without", "won't", "wonder", "wont", "words", "work", "worked",
+ "working", "works", "world", "would", "wouldn't", "wouldnt", "www", "x", "y", "year", "years", "yes", "yet",
+ "you", "you'd", "you'll", "you're", "you've", "youd", "youll", "young", "younger", "youngest", "your",
+ "youre", "yours", "yourself", "yourselves", "youve", "z", "zero");
/**
* Disallowed chars for words in processed text segments. This regular
@@ -359,19 +361,20 @@ public class Constants {
final Calendar in = new GregorianCalendar();
in.setTime(date);
final Calendar out = new GregorianCalendar();
+ out.setTimeZone(TimeZone.getTimeZone("GMT"));
out.setTime(date);
out.set(Calendar.MILLISECOND, 0);
switch (this) {
case YEAR:
- out.set(Calendar.MONTH, in.getActualMinimum(Calendar.MONTH));
+ out.set(Calendar.MONTH, 0);
case MONTH:
- out.set(Calendar.DAY_OF_MONTH, in.getActualMinimum(Calendar.DAY_OF_MONTH));
+ out.set(Calendar.DAY_OF_MONTH, 1);
case DAY:
- out.set(Calendar.HOUR, in.getActualMinimum(Calendar.HOUR));
+ out.set(Calendar.HOUR_OF_DAY, 0);
case HOUR:
- out.set(Calendar.MINUTE, in.getActualMinimum(Calendar.MINUTE));
+ out.set(Calendar.MINUTE, 0);
case MINUTE:
- out.set(Calendar.SECOND, in.getActualMinimum(Calendar.SECOND));
+ out.set(Calendar.SECOND, 0);
default:
break;
}
@@ -385,18 +388,19 @@ public class Constants {
in.setTime(date);
final Calendar out = new GregorianCalendar();
out.setTime(date);
- out.set(Calendar.MILLISECOND, 0);
+ out.setTimeZone(TimeZone.getTimeZone("GMT"));
+ out.set(Calendar.MILLISECOND, 999);
switch (this) {
case YEAR:
- out.set(Calendar.MONTH, out.getActualMaximum(Calendar.MONTH));
+ out.set(Calendar.MONTH, 11);
case MONTH:
out.set(Calendar.DAY_OF_MONTH, out.getActualMaximum(Calendar.DAY_OF_MONTH));
case DAY:
- out.set(Calendar.HOUR, out.getActualMaximum(Calendar.HOUR));
+ out.set(Calendar.HOUR_OF_DAY, 23);
case HOUR:
- out.set(Calendar.MINUTE, out.getActualMaximum(Calendar.MINUTE));
+ out.set(Calendar.MINUTE, 59);
case MINUTE:
- out.set(Calendar.SECOND, out.getActualMaximum(Calendar.SECOND));
+ out.set(Calendar.SECOND, 59);
default:
break;
}
diff --git a/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java b/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java
index ffbb2040e0b27f890068bfd23892d86a9882c0b4..cb1d99dc52c4d37a0794ab498a8568ac53cb47f3 100644
--- a/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java
+++ b/vipra-util/src/main/java/de/vipra/util/model/TopicFull.java
@@ -26,8 +26,6 @@ public class TopicFull implements Model<ObjectId>, Serializable {
private String name;
- private Integer index;
-
@Embedded
@QueryIgnore(multi = true)
private List<TopicWord> words;
@@ -36,8 +34,10 @@ public class TopicFull implements Model<ObjectId>, Serializable {
@QueryIgnore(multi = true)
private List<Sequence> sequences;
+ @QueryIgnore(multi = true)
private Double avgRelevance;
+ @QueryIgnore(multi = true)
private Double varRelevance;
private boolean dynamic = false;
@@ -68,14 +68,6 @@ public class TopicFull implements Model<ObjectId>, Serializable {
this.name = name;
}
- public Integer getIndex() {
- return index;
- }
-
- public void setIndex(final Integer index) {
- this.index = index;
- }
-
public List<TopicWord> getWords() {
return words;
}
@@ -173,9 +165,9 @@ public class TopicFull implements Model<ObjectId>, Serializable {
@Override
public String toString() {
- return "TopicFull [id=" + id + ", name=" + name + ", index=" + index + ", words=" + words + ", sequences="
- + sequences + ", avgRelevance=" + avgRelevance + ", varRelevance=" + varRelevance + ", created="
- + created + ", modified=" + modified + "]";
+ return "TopicFull [id=" + id + ", name=" + name + ", words=" + words + ", sequences=" + sequences
+ + ", avgRelevance=" + avgRelevance + ", varRelevance=" + varRelevance + ", created=" + created
+ + ", modified=" + modified + "]";
}
}