diff --git a/build.sh b/build.sh index 8d4faa4fc530f2098b5667674828664efd8faebe..257f80ddefaa7be49fa9121bfc69f494e9ea76e0 100755 --- a/build.sh +++ b/build.sh @@ -57,6 +57,9 @@ function deploy { cd $1 rm -rf ROOT ROOT.war $PROJECT $WAR_FILE cp -r $2 $3 + echo "-------------------------------" >> $LOG + echo "deploying vipra-backend ($3)" | tee -a $LOG + echo "-------------------------------" >> $LOG } # build dtm diff --git a/vipra-backend/src/main/java/de/vipra/rest/resource/WindowResource.java b/vipra-backend/src/main/java/de/vipra/rest/resource/WindowResource.java deleted file mode 100644 index 3577e152adbaa5015bb7fcb567bb26f7dce0aa87..0000000000000000000000000000000000000000 --- a/vipra-backend/src/main/java/de/vipra/rest/resource/WindowResource.java +++ /dev/null @@ -1,69 +0,0 @@ -package de.vipra.rest.resource; - -import java.io.IOException; -import java.util.List; - -import javax.servlet.ServletContext; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -import de.vipra.rest.model.APIError; -import de.vipra.rest.model.ResponseWrapper; -import de.vipra.util.Config; -import de.vipra.util.StringUtils; -import de.vipra.util.ex.ConfigException; -import de.vipra.util.model.TopicModel; -import de.vipra.util.model.WindowFull; -import de.vipra.util.service.MongoService; -import de.vipra.util.service.QueryBuilder; - -@Path("windows") -public class WindowResource { - - final MongoService<WindowFull, Integer> dbWindows; - - public WindowResource(@Context final ServletContext servletContext) throws ConfigException, IOException { - final Config config = Config.getConfig(); - dbWindows = MongoService.getDatabaseService(config, WindowFull.class); - } - - @GET - @Produces(MediaType.APPLICATION_JSON) - public Response getWindows(@QueryParam("topicModel") final String topicModel, @QueryParam("skip") final Integer skip, - @QueryParam("limit") final Integer limit, @QueryParam("sort") @DefaultValue("startDate") final String sortBy, - @QueryParam("fields") final String fields) { - final ResponseWrapper<List<WindowFull>> res = new ResponseWrapper<>(); - - if (res.hasErrors()) - return Response.status(Response.Status.BAD_REQUEST).entity(res).build(); - - try { - final QueryBuilder query = QueryBuilder.builder().skip(skip).limit(limit).sortBy(sortBy); - if (fields != null && !fields.isEmpty()) - query.fields(true, StringUtils.getFields(fields)); - - if (topicModel != null && !topicModel.isEmpty()) - query.eq("topicModel", new TopicModel(topicModel)); - - final List<WindowFull> windows = dbWindows.getMultiple(query); - - if ((skip != null && skip > 0) || (limit != null && limit > 0)) - res.addHeader("total", dbWindows.count(null)); - else - res.addHeader("total", windows.size()); - - return res.ok(windows); - } catch (final Exception e) { - e.printStackTrace(); - res.addError(new APIError(Response.Status.BAD_REQUEST, "Error", e.getMessage())); - return Response.status(Response.Status.BAD_REQUEST).entity(res).build(); - } - } - -} \ No newline at end of file diff --git a/vipra-cmd/runcfg/CMD.launch b/vipra-cmd/runcfg/CMD.launch index a9b34fe31e519e52254a047acf6ff5f98955f666..23d3254f3b652c3a87b318f3dbd0cb2636ee147d 100644 --- a/vipra-cmd/runcfg/CMD.launch +++ b/vipra-cmd/runcfg/CMD.launch @@ -11,7 +11,7 @@ </listAttribute> <stringAttribute key="org.eclipse.jdt.launching.CLASSPATH_PROVIDER" value="org.eclipse.m2e.launchconfig.classpathProvider"/> <stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="de.vipra.cmd.Main"/> -<stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="-dD reuters -C reuters -S reuters -I /home/eike/repos/master/ma-impl/vm/data/reuters.json"/> +<stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="-dl"/> <stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="vipra-cmd"/> <stringAttribute key="org.eclipse.jdt.launching.SOURCE_PATH_PROVIDER" value="org.eclipse.m2e.launchconfig.sourcepathProvider"/> <stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-ea"/> diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/file/FilebaseWindowIndex.java b/vipra-cmd/src/main/java/de/vipra/cmd/file/FilebaseWindowIndex.java index 4559a99f46b0dfce071e193df2ea5fc5d8f61ae4..17b79a9b16f7535625f132a34f5f4322be8a33f3 100644 --- a/vipra-cmd/src/main/java/de/vipra/cmd/file/FilebaseWindowIndex.java +++ b/vipra-cmd/src/main/java/de/vipra/cmd/file/FilebaseWindowIndex.java @@ -18,6 +18,7 @@ import org.apache.commons.io.FileUtils; import de.vipra.util.Constants; import de.vipra.util.Constants.WindowResolution; import de.vipra.util.CountMap; +import de.vipra.util.model.Window; public class FilebaseWindowIndex { @@ -97,6 +98,8 @@ public class FilebaseWindowIndex { } public int size() { + if (seqDirty) + resizeWindows(); return windowSizes.size(); } @@ -122,6 +125,21 @@ public class FilebaseWindowIndex { FileUtils.copyFile(modelFile, new File(modelDir, MULT_FILE_NAME)); } + public List<Window> getWindows() { + final List<Window> windows = new ArrayList<>(size()); + for (int i = 0; i < size(); i++) + windows.add(getWindow(i)); + return windows; + } + + public Window getWindow(final int index) { + Window window = new Window(); + window.setStartDate(startDate(index)); + window.setEndDate(endDate(index)); + window.setWindowResolution(windowResolution); + return window; + } + private void resizeWindows() { final List<Date> dates = new ArrayList<>(windowMap.keySet()); Collections.sort(dates); diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/lda/Analyzer.java b/vipra-cmd/src/main/java/de/vipra/cmd/lda/Analyzer.java index 7f331e871702edced1adcf323c0c3d3448366ecd..126eae5bacb6febd1127f713f51bcf79fec141cf 100644 --- a/vipra-cmd/src/main/java/de/vipra/cmd/lda/Analyzer.java +++ b/vipra-cmd/src/main/java/de/vipra/cmd/lda/Analyzer.java @@ -39,8 +39,6 @@ import de.vipra.util.model.TopicModelConfig; import de.vipra.util.model.TopicModelFull; import de.vipra.util.model.TopicShare; import de.vipra.util.model.TopicWord; -import de.vipra.util.model.Window; -import de.vipra.util.model.WindowFull; import de.vipra.util.service.MongoService; import de.vipra.util.service.QueryBuilder; @@ -52,7 +50,6 @@ public class Analyzer { private final MongoService<ArticleFull, ObjectId> dbArticles; private final MongoService<TopicFull, ObjectId> dbTopics; private final MongoService<SequenceFull, ObjectId> dbSequences; - private final MongoService<WindowFull, Integer> dbWindows; private final MongoService<TopicModelFull, String> dbTopicModels; public Analyzer() throws AnalyzerException, ConfigException { @@ -61,7 +58,6 @@ public class Analyzer { dbArticles = MongoService.getDatabaseService(config, ArticleFull.class); dbTopics = MongoService.getDatabaseService(config, TopicFull.class); dbSequences = MongoService.getDatabaseService(config, SequenceFull.class); - dbWindows = MongoService.getDatabaseService(config, WindowFull.class); dbTopicModels = MongoService.getDatabaseService(config, TopicModelFull.class); // check for binary @@ -192,7 +188,6 @@ public class Analyzer { // read topic definition files and create topics final TopicModelFull topicModel = new TopicModelFull(modelConfig.getName(), modelConfig); - final List<WindowFull> newWindows = new ArrayList<>(windowCount); final List<SequenceFull> newSequences = new ArrayList<>(topicCount * windowCount); final List<TopicFull> newTopics = new ArrayList<>(topicCount); @@ -203,17 +198,6 @@ public class Analyzer { final boolean seqRelativeCutoff = modelConfig.getMinRelativeProbability() > 0; - // create sequence windows - for (int idxSeq = 0; idxSeq < windowCount; idxSeq++) { - final WindowFull newWindow = new WindowFull(); - newWindow.setId(idxSeq); - newWindow.setStartDate(windowIndex.startDate(idxSeq)); - newWindow.setEndDate(windowIndex.endDate(idxSeq)); - newWindow.setWindowResolution(modelConfig.getWindowResolution()); - newWindow.setTopicModel(new TopicModel(topicModel.getId())); - newWindows.add(newWindow); - } - // for each topic for (int idxTopic = 0; idxTopic < topicCount; idxTopic++) { final File seqFile = new File(outDirSeq, "topic-" + StringUtils.padNumber(idxTopic, 3) + "-var-e-log-prob.dat"); @@ -285,7 +269,7 @@ public class Analyzer { // create sequence final SequenceFull newSequenceFull = new SequenceFull(); - newSequenceFull.setWindow(new Window(newWindows.get(idxSeq))); + newSequenceFull.setWindow(windowIndex.getWindow(idxSeq)); newSequenceFull.setWords(newSequenceWords); newSequenceFull.setRelevance(relevance); newSequenceFull.setRelevanceChange(relevance - prevRelevance); @@ -441,11 +425,9 @@ public class Analyzer { final QueryBuilder builder = QueryBuilder.builder().eq("topicModel", new TopicModel(modelConfig.getName())); - dbWindows.deleteMultiple(builder); dbSequences.deleteMultiple(builder); dbTopics.deleteMultiple(builder); - dbWindows.createMultiple(newWindows); dbSequences.createMultiple(newSequences); dbTopics.createMultiple(newTopics); @@ -455,7 +437,7 @@ public class Analyzer { private int printProgress(final int tenthPercent, final double progress, final int iteration, final int maxIterationsLength, final long remainingNanos, final TopicModelConfig modelConfig, final int lastLength) { - String msg = " [" + StringUtils.repeat("#", tenthPercent) + StringUtils.repeat(" ", 10 - tenthPercent) + "] " + final String msg = " [" + StringUtils.repeat("#", tenthPercent) + StringUtils.repeat(" ", 10 - tenthPercent) + "] " + StringUtils.pad(Integer.toString((int) Math.floor(progress)), 3, true) + "% (" + StringUtils.pad(Integer.toString(iteration), maxIterationsLength, true) + "/" + modelConfig.getDynamicMinIterations() + "-" + modelConfig.getDynamicMaxIterations() + ") " + StringUtils.timeString(remainingNanos, false, true, false) + "\r"; diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java index 76d2c1bdd172d212a0dcaf3eab07a8ee1a9180c6..5cd3ef88af28e20fdabe0ad3e07519d278d5c3b9 100644 --- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java +++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ClearCommand.java @@ -31,6 +31,8 @@ public class ClearCommand implements Command { ConsoleUtils.warn("could not delete data directory: " + config.getDataDirectory().getAbsolutePath()); } + config.clearTopicModelConfigs(); + ConsoleUtils.info("cleared"); } diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/CreateModelCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/CreateModelCommand.java index 1d09d032dac5c5e867a6b8837f4178ff76bfcdcc..320cc47e873047c5195e3d1156426bef233c469b 100644 --- a/vipra-cmd/src/main/java/de/vipra/cmd/option/CreateModelCommand.java +++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/CreateModelCommand.java @@ -61,10 +61,10 @@ public class CreateModelCommand implements Command { final File modelDir = new File(config.getDataDirectory(), name); if (modelDir.exists()) { try { - TopicModelFull topicModel = dbTopicModels.getSingle(name); + final TopicModelFull topicModel = dbTopicModels.getSingle(name); if (topicModel != null) throw new Exception("model with that name already exists: " + name); - } catch (Exception e) {} + } catch (final Exception e) {} restoreModelFromDir(name, modelDir); ConsoleUtils.info("model restored: " + name); continue; diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteModelCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteModelCommand.java index f54b707c8d8709324516d4f2e5c36c2b4631894f..d1434b08ce3279e81704aba92e9d651960332e6a 100644 --- a/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteModelCommand.java +++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/DeleteModelCommand.java @@ -12,7 +12,6 @@ import de.vipra.util.model.SequenceFull; import de.vipra.util.model.TextEntityFull; import de.vipra.util.model.TopicFull; import de.vipra.util.model.TopicModel; -import de.vipra.util.model.WindowFull; import de.vipra.util.model.WordFull; import de.vipra.util.service.MongoService; import de.vipra.util.service.QueryBuilder; @@ -31,7 +30,6 @@ public class DeleteModelCommand implements Command { final MongoService<TopicModel, String> dbTopicModels = MongoService.getDatabaseService(config, TopicModel.class); final MongoService<ArticleFull, ObjectId> dbArticles = MongoService.getDatabaseService(config, ArticleFull.class); final MongoService<TopicFull, ObjectId> dbTopics = MongoService.getDatabaseService(config, TopicFull.class); - final MongoService<WindowFull, Integer> dbWindows = MongoService.getDatabaseService(config, WindowFull.class); final MongoService<SequenceFull, ObjectId> dbSequences = MongoService.getDatabaseService(config, SequenceFull.class); final MongoService<WordFull, String> dbWords = MongoService.getDatabaseService(config, WordFull.class); final MongoService<TextEntityFull, String> dbEntities = MongoService.getDatabaseService(config, TextEntityFull.class); @@ -50,7 +48,6 @@ public class DeleteModelCommand implements Command { final QueryBuilder builder = QueryBuilder.builder().eq("topicModel", new TopicModel(name)); dbArticles.deleteMultiple(builder); dbTopics.deleteMultiple(builder); - dbWindows.deleteMultiple(builder); dbSequences.deleteMultiple(builder); dbWords.deleteMultiple(builder); dbEntities.deleteMultiple(builder); diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java index 4d1fa20312b787197d9d52a156224e93919f77b2..4e7b38f5d1cafd02659395dc7d646e91fc3371b3 100644 --- a/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java +++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/ImportCommand.java @@ -270,8 +270,6 @@ public class ImportCommand implements Command { topicModel = new TopicModelFull(modelConfig.getName(), modelConfig); newTextEntities = new HashSet<>(); - dbTopicModels.replaceSingle(topicModel); - final Timer timer = new Timer(); timer.restart(); @@ -286,6 +284,12 @@ public class ImportCommand implements Command { */ filebase.sync(); + /* + * update topic model + */ + topicModel.setWindows(filebase.getWindowIndex().getWindows()); + dbTopicModels.replaceSingle(topicModel); + /* * add new words */ diff --git a/vipra-cmd/src/main/java/de/vipra/cmd/option/IndexingCommand.java b/vipra-cmd/src/main/java/de/vipra/cmd/option/IndexingCommand.java index e4026ff96384474c551389b8061d63a4cbbdf43b..112c38f759354ad0910f51fd4d6299863cbc82ba 100644 --- a/vipra-cmd/src/main/java/de/vipra/cmd/option/IndexingCommand.java +++ b/vipra-cmd/src/main/java/de/vipra/cmd/option/IndexingCommand.java @@ -2,6 +2,7 @@ package de.vipra.cmd.option; import java.io.IOException; import java.text.ParseException; +import java.util.Date; import java.util.Map; import org.bson.types.ObjectId; @@ -21,12 +22,14 @@ import de.vipra.util.ex.ConfigException; import de.vipra.util.ex.DatabaseException; import de.vipra.util.model.ArticleFull; import de.vipra.util.model.TopicModelConfig; +import de.vipra.util.model.TopicModelFull; import de.vipra.util.service.MongoService; public class IndexingCommand implements Command { private final String[] models; private Config config; + private MongoService<TopicModelFull, String> dbTopicModels; private MongoService<ArticleFull, ObjectId> dbArticles; private Client elasticClient; private ESSerializer<ArticleFull> elasticSerializer; @@ -70,6 +73,11 @@ public class IndexingCommand implements Command { elasticClient.close(); + // update indexed date + final TopicModelFull topicModel = new TopicModelFull(modelConfig.getName()); + topicModel.setLastIndexed(new Date()); + dbTopicModels.updateSingle(topicModel, "lastIndexed"); + /* * run information */ @@ -79,6 +87,7 @@ public class IndexingCommand implements Command { @Override public void run() throws ParseException, IOException, DatabaseException, Exception { config = Config.getConfig(); + dbTopicModels = MongoService.getDatabaseService(config, TopicModelFull.class); dbArticles = MongoService.getDatabaseService(config, ArticleFull.class); elasticClient = ESClient.getClient(config); elasticSerializer = new ESSerializer<>(ArticleFull.class); diff --git a/vipra-ui/app/html/directives/window-dropdown.html b/vipra-ui/app/html/directives/window-dropdown.html deleted file mode 100644 index 69557e060f80230635613888f50b2a5655a25e6e..0000000000000000000000000000000000000000 --- a/vipra-ui/app/html/directives/window-dropdown.html +++ /dev/null @@ -1,5 +0,0 @@ -<ol class="nya-bs-select nya-bs-condensed" ng-model="ngModel" ng-class="{dropup:dropup}"> - <li value="{{window.id}}" class="nya-bs-option" ng-repeat="window in windows"> - <a ng-bind="window.label"></a> - </li> -</ol> diff --git a/vipra-ui/app/html/network.html b/vipra-ui/app/html/network.html index a5bdda9194d9299e62fe31d176e51a91738c6881..88bd5c4f92758a44b0ba9e0df462d73b7911ff63 100644 --- a/vipra-ui/app/html/network.html +++ b/vipra-ui/app/html/network.html @@ -3,16 +3,23 @@ <div class="graph-legend overlay"> <div class="checkbox"> <input type="checkbox" id="showArticles" ng-model="shown.articles"> - <label for="showArticles" style="color:{{colors.articles}}">Articles</label> + <label for="showArticles">Articles</label> + </div> + <div class="checkbox"> + <input type="checkbox" id="showSimilarArticles" ng-model="shown.similararticles"> + <label for="showSimilarArticles">Similar Articles</label> </div> <div class="checkbox"> <input type="checkbox" id="showTopics" ng-model="shown.topics"> - <label for="showTopics" style="color:{{colors.topics}}">Topics</label> + <label for="showTopics">Topics</label> </div> <div class="checkbox"> <input type="checkbox" id="showWords" ng-model="shown.words"> <label for="showWords">Words</label> </div> + <button class="btn btn-default" ng-click="reset()"> + Reset + </button> </div> <div class="fullsize" id="visgraph"></div> </div> diff --git a/vipra-ui/app/js/controllers.js b/vipra-ui/app/js/controllers.js index 33a80659ad304a1abf418d622e17095732ad5a86..597b1490efafd88fa69ab82c4a2b8125e387b320 100644 --- a/vipra-ui/app/js/controllers.js +++ b/vipra-ui/app/js/controllers.js @@ -239,8 +239,8 @@ /** * Network controller */ - app.controller('NetworkController', ['$scope', '$state', '$stateParams', '$timeout', 'ArticleFactory', 'TopicFactory', 'WordFactory', 'WindowFactory', - function($scope, $state, $stateParams, $timeout, ArticleFactory, TopicFactory, WordFactory, WindowFactory) { + app.controller('NetworkController', ['$scope', '$state', '$stateParams', '$timeout', 'ArticleFactory', 'TopicFactory', 'WordFactory', + function($scope, $state, $stateParams, $timeout, ArticleFactory, TopicFactory, WordFactory) { var id = 0, ids = {}, @@ -283,6 +283,7 @@ }; $scope.shown = { articles: true, + similararticles: true, topics: true, words: true }; @@ -330,7 +331,7 @@ $scope.rootModels.topicModel = data.topicModel; }); - var newNode = function(title, type, show, dbid, color, shape) { + var newNode = function(title, type, show, dbid, color, shape, loader) { ids[dbid] = ++id; return { id: id, @@ -340,6 +341,7 @@ show: show, dbid: dbid, shape: shape || 'dot', + loader: loader, color: { background: color, highlight: { @@ -351,15 +353,15 @@ var topicNode = function(topic) { topic = topic.topic || topic; - return newNode(topic.name, 'topic', 'topics.show', topic.id, $scope.colors.topics, 'triangle'); + return newNode(topic.name, 'topic', 'topics.show', topic.id, $scope.colors.topics, 'triangle', $scope.loadTopic); }; var articleNode = function(article) { - return newNode(article.title, 'article', 'articles.show', article.id, $scope.colors.articles, 'square'); + return newNode(article.title, 'article', 'articles.show', article.id, $scope.colors.articles, 'square', $scope.loadArticle); }; var wordNode = function(word) { - return newNode(word.id, 'word', 'word.show', word.id, $scope.colors.words, 'box'); + return newNode(word.id, 'word', 'word.show', word.id, $scope.colors.words, 'box', $scope.loadWord); }; var edgeExists = function(idA, idB) { @@ -418,13 +420,7 @@ selectTimeout = $timeout(function() { var node = $scope.nodes.get(props.nodes[0]); if (node) { - if (node.type === 'article' && $scope.shown.topics) { - $scope.loadArticle(node); - } else if (node.type === 'topic') { - $scope.loadTopic(node); - } else if (node.type === 'word') { - $scope.loadWord(node); - } + node.loader(node); $scope.nodes.update(node); } }, 500); @@ -432,13 +428,20 @@ $scope.loadArticle = function(node) { ArticleFactory.get({ - id: node.dbid + id: node.dbid, + fields: '_all' }, function(data) { if (data.topics) { for (var i = 0; i < data.topics.length; i++) data.topics[i] = data.topics[i].topic; constructor(data.topics, node, topicNode); } + if(data.similarArticles && $scope.shown.similararticles) { + var articles = []; + for(var i = 0; i < data.similarArticles.length; i++) + articles.push(data.similarArticles[i].article); + constructor(articles, node, articleNode); + } }); }; @@ -491,6 +494,10 @@ if ($scope.rootNode && $scope.rootNode.topicModel.id !== newVal.id) $state.transitionTo('index'); }); + + $scope.reset = function() { + $state.go($state.current, {}, {reload: true}); + }; } ]); diff --git a/vipra-ui/app/js/directives.js b/vipra-ui/app/js/directives.js index 46a5a8f1cdc5a2ce11b2918a84d7c675b0bf9783..3d668ffb45c9fa9e0af210faf00a9794a166c14a 100644 --- a/vipra-ui/app/js/directives.js +++ b/vipra-ui/app/js/directives.js @@ -267,31 +267,6 @@ }; }]); - app.directive('windowDropdown', ['WindowFactory', function(WindowFactory) { - return { - scope: { - ngModel: '=', - topicModel: '=', - dropup: '@' - }, - link: function($scope) { - $scope.dropup = $scope.dropup === 'true'; - $scope.$watch('topicModel', function(value) { - if(value) { - WindowFactory.query({ - topicModel: value.id - }, function(data) { - for(var i = 0; i < data.length; i++) - data.label = Vipra.windowLabel(data[i].startDate, data[i].windowResolution); - $scope.windows = data; - }); - } - }); - }, - templateUrl: '/html/directives/window-dropdown.html' - }; - }]); - app.directive('sortBy', [function() { return { restrict: 'A', diff --git a/vipra-util/src/main/java/de/vipra/util/Config.java b/vipra-util/src/main/java/de/vipra/util/Config.java index 2aace5d8fcff6594082634eb9eeb8bc606f59ef1..5235e4ef59976ddc2ec366a90d3cfd364e89e086 100644 --- a/vipra-util/src/main/java/de/vipra/util/Config.java +++ b/vipra-util/src/main/java/de/vipra/util/Config.java @@ -126,10 +126,14 @@ public class Config { return modelConfig; } - public void setTopicModelConfigs(final Map<String, TopicModelConfig> topicModelConfigs) { + private void setTopicModelConfigs(final Map<String, TopicModelConfig> topicModelConfigs) { this.topicModelConfigs = topicModelConfigs; } + public void clearTopicModelConfigs() { + this.topicModelConfigs.clear(); + } + public void setDataDirectory(final String dataDirectory) { this.dataDirectory = dataDirectory; } @@ -252,7 +256,7 @@ public class Config { // read topic model configs final MongoService<TopicModelFull, String> dbTopicModels = MongoService.getDatabaseService(instance, TopicModelFull.class); - final List<TopicModelFull> topicModels = dbTopicModels.getAll(); + final List<TopicModelFull> topicModels = dbTopicModels.getAll("_all"); final Map<String, TopicModelConfig> topicModelConfigs = new HashMap<>(topicModels.size()); for (final TopicModelFull topicModel : topicModels) topicModelConfigs.put(topicModel.getId(), topicModel.getModelConfig()); diff --git a/vipra-util/src/main/java/de/vipra/util/model/Sequence.java b/vipra-util/src/main/java/de/vipra/util/model/Sequence.java index 3d59fdcdc659efb1c9ce7d24a6eb79876a5d6caa..f83d0c66916041fd64d18163a4f970ce22d1dfa3 100644 --- a/vipra-util/src/main/java/de/vipra/util/model/Sequence.java +++ b/vipra-util/src/main/java/de/vipra/util/model/Sequence.java @@ -66,7 +66,7 @@ public class Sequence implements Model<ObjectId>, Comparable<Sequence>, Serializ @Override public int compareTo(final Sequence o) { - return window.getId() - o.getWindow().getId(); + return window.compareTo(o.getWindow()); } @Override diff --git a/vipra-util/src/main/java/de/vipra/util/model/SequenceFull.java b/vipra-util/src/main/java/de/vipra/util/model/SequenceFull.java index 55d58536fe95826979bcfe276243102e59ef38c4..442227993b14cb4e4df7dbecdc52dca099a27820 100644 --- a/vipra-util/src/main/java/de/vipra/util/model/SequenceFull.java +++ b/vipra-util/src/main/java/de/vipra/util/model/SequenceFull.java @@ -100,7 +100,7 @@ public class SequenceFull implements Model<ObjectId>, Comparable<SequenceFull>, @Override public int compareTo(final SequenceFull o) { - return window.getId() - o.getWindow().getId(); + return window.compareTo(o.getWindow()); } @Override diff --git a/vipra-util/src/main/java/de/vipra/util/model/TopicModelFull.java b/vipra-util/src/main/java/de/vipra/util/model/TopicModelFull.java index a12ee4fede85b97f16863a448a06ad9b8af6286b..7e3d56b2c992fc0ae4392b29d7b77ac5d8ef0a41 100644 --- a/vipra-util/src/main/java/de/vipra/util/model/TopicModelFull.java +++ b/vipra-util/src/main/java/de/vipra/util/model/TopicModelFull.java @@ -2,6 +2,7 @@ package de.vipra.util.model; import java.io.Serializable; import java.util.Date; +import java.util.List; import org.mongodb.morphia.annotations.Embedded; import org.mongodb.morphia.annotations.Entity; @@ -19,16 +20,21 @@ public class TopicModelFull implements Model<String>, Comparable<TopicModelFull> @Id private String id; - private int topicCount; + private Integer topicCount; - private int articleCount; + private Integer articleCount; - private int wordCount; + private Integer wordCount; - private int windowCount; + private Integer windowCount; private Date lastGenerated; + private Date lastIndexed; + + @QueryIgnore(multi = true) + private List<Window> windows; + @Embedded @QueryIgnore(multi = true) private TopicModelConfig modelConfig; @@ -54,35 +60,35 @@ public class TopicModelFull implements Model<String>, Comparable<TopicModelFull> this.id = id; } - public int getTopicCount() { + public Integer getTopicCount() { return topicCount; } - public void setTopicCount(final int topicCount) { + public void setTopicCount(final Integer topicCount) { this.topicCount = topicCount; } - public int getArticleCount() { + public Integer getArticleCount() { return articleCount; } - public void setArticleCount(final int articleCount) { + public void setArticleCount(final Integer articleCount) { this.articleCount = articleCount; } - public int getWordCount() { + public Integer getWordCount() { return wordCount; } - public void setWordCount(final int wordCount) { + public void setWordCount(final Integer wordCount) { this.wordCount = wordCount; } - public int getWindowCount() { + public Integer getWindowCount() { return windowCount; } - public void setWindowCount(final int windowCount) { + public void setWindowCount(final Integer windowCount) { this.windowCount = windowCount; } @@ -102,6 +108,22 @@ public class TopicModelFull implements Model<String>, Comparable<TopicModelFull> this.lastGenerated = lastGenerated; } + public Date getLastIndexed() { + return lastIndexed; + } + + public void setLastIndexed(Date lastIndexed) { + this.lastIndexed = lastIndexed; + } + + public List<Window> getWindows() { + return windows; + } + + public void setWindows(List<Window> windows) { + this.windows = windows; + } + @Override public int compareTo(final TopicModelFull o) { return id.compareTo(o.getId()); diff --git a/vipra-util/src/main/java/de/vipra/util/model/Window.java b/vipra-util/src/main/java/de/vipra/util/model/Window.java index b3350267579083cae4eb9245de58210c22239ed9..47191efb6be3d8d6c206779a509a47f27c022986 100644 --- a/vipra-util/src/main/java/de/vipra/util/model/Window.java +++ b/vipra-util/src/main/java/de/vipra/util/model/Window.java @@ -3,8 +3,7 @@ package de.vipra.util.model; import java.io.Serializable; import java.util.Date; -import org.mongodb.morphia.annotations.Entity; -import org.mongodb.morphia.annotations.Id; +import org.mongodb.morphia.annotations.Embedded; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -12,11 +11,8 @@ import de.vipra.util.Constants.WindowResolution; @JsonIgnoreProperties(ignoreUnknown = true) @SuppressWarnings("serial") -@Entity(value = "windows", noClassnameStored = true) -public class Window implements Model<Integer>, Serializable, Comparable<Window> { - - @Id - private Integer id; +@Embedded +public class Window implements Serializable, Comparable<Window> { private Date startDate; @@ -24,29 +20,6 @@ public class Window implements Model<Integer>, Serializable, Comparable<Window> private WindowResolution windowResolution; - public Window() {} - - public Window(final Integer id) { - this.id = id; - } - - public Window(final WindowFull window) { - id = window.getId(); - startDate = window.getStartDate(); - endDate = window.getEndDate(); - windowResolution = window.getWindowResolution(); - } - - @Override - public Integer getId() { - return id; - } - - @Override - public void setId(final Integer id) { - this.id = id; - } - public Date getStartDate() { return startDate; } @@ -67,13 +40,13 @@ public class Window implements Model<Integer>, Serializable, Comparable<Window> return windowResolution; } - public void setWindowResolution(final WindowResolution windowResolution) { + public void setWindowResolution(WindowResolution windowResolution) { this.windowResolution = windowResolution; } @Override public int compareTo(final Window o) { - return id - o.getId(); + return startDate.compareTo(o.getStartDate()); } } diff --git a/vipra-util/src/main/java/de/vipra/util/model/WindowFull.java b/vipra-util/src/main/java/de/vipra/util/model/WindowFull.java deleted file mode 100644 index c34086ad1da48c2f1b2a76e46e6593e970a9ad0d..0000000000000000000000000000000000000000 --- a/vipra-util/src/main/java/de/vipra/util/model/WindowFull.java +++ /dev/null @@ -1,80 +0,0 @@ -package de.vipra.util.model; - -import java.io.Serializable; -import java.util.Date; - -import org.mongodb.morphia.annotations.Entity; -import org.mongodb.morphia.annotations.Id; -import org.mongodb.morphia.annotations.Reference; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; - -import de.vipra.util.Constants.WindowResolution; -import de.vipra.util.an.QueryIgnore; - -@JsonIgnoreProperties(ignoreUnknown = true) -@SuppressWarnings("serial") -@Entity(value = "windows", noClassnameStored = true) -public class WindowFull implements Model<Integer>, Comparable<WindowFull>, Serializable { - - @Id - private Integer id; - - @Reference - @QueryIgnore(multi = true) - private TopicModel topicModel; - - private Date startDate; - - private Date endDate; - - private WindowResolution windowResolution; - - @Override - public Integer getId() { - return id; - } - - @Override - public void setId(final Integer id) { - this.id = id; - } - - public TopicModel getTopicModel() { - return topicModel; - } - - public void setTopicModel(final TopicModel model) { - topicModel = model; - } - - public Date getStartDate() { - return startDate; - } - - public void setStartDate(final Date startDate) { - this.startDate = startDate; - } - - public Date getEndDate() { - return endDate; - } - - public void setEndDate(final Date endDate) { - this.endDate = endDate; - } - - public WindowResolution getWindowResolution() { - return windowResolution; - } - - public void setWindowResolution(final WindowResolution windowResolution) { - this.windowResolution = windowResolution; - } - - @Override - public int compareTo(final WindowFull o) { - return id - o.getId(); - } - -} diff --git a/vipra-util/src/main/java/de/vipra/util/service/MongoService.java b/vipra-util/src/main/java/de/vipra/util/service/MongoService.java index b0c10474286260e0418b5bc50dc982858107d1ed..b7ea2dce5944d8055f2db4d2ea04012412db4829 100644 --- a/vipra-util/src/main/java/de/vipra/util/service/MongoService.java +++ b/vipra-util/src/main/java/de/vipra/util/service/MongoService.java @@ -89,7 +89,7 @@ public class MongoService<Type extends Model<IdType>, IdType> { if (builder != null) { builder.build(query); - if (!builder.isAllFields() && ignoredFieldsMultiQuery.length > 0) { + if (builder.getFields() == null && !builder.isAllFields() && ignoredFieldsMultiQuery.length > 0) { query.retrievedFields(false, ignoredFieldsMultiQuery); } } else if (ignoredFieldsMultiQuery.length > 0) { diff --git a/vipra-util/src/main/java/de/vipra/util/service/QueryBuilder.java b/vipra-util/src/main/java/de/vipra/util/service/QueryBuilder.java index 3fe6fd54071f732474571ed5a63a829ac1837df3..5d27768a87286e45fd1d6c0cd0158d63a139530d 100644 --- a/vipra-util/src/main/java/de/vipra/util/service/QueryBuilder.java +++ b/vipra-util/src/main/java/de/vipra/util/service/QueryBuilder.java @@ -1,16 +1,11 @@ package de.vipra.util.service; +import java.util.Arrays; import java.util.HashSet; import java.util.Set; import org.mongodb.morphia.query.Query; -/** - * QueryBuilder instances are used to create complex queries for use with the - * getMultiple method - * - * @see {@link Service#getMultiple(QueryBuilder)} - */ public class QueryBuilder { public static enum CriterionType { @@ -56,52 +51,31 @@ public class QueryBuilder { private Integer skip; private Integer limit; private String sortBy; - private Set<Criterion> criteria; + private final Set<Criterion> criteria; private String[] fields; private boolean include; private boolean allFields; private QueryBuilder() { - this.criteria = new HashSet<>(); + criteria = new HashSet<>(); } public static QueryBuilder builder() { return new QueryBuilder(); } - /** - * Skip n entries - * - * @param skip - * entries to skip - * @return QueryBuilder instance - */ public QueryBuilder skip(final Integer skip) { if (skip == null || skip >= 0) this.skip = skip; return this; } - /** - * Limit return size. - * - * @param limit - * maximum return size - * @return QueryBuilder instance - */ public QueryBuilder limit(final Integer limit) { if (limit == null || limit >= 0) this.limit = limit; return this; } - /** - * Sort result by field - * - * @param sortBy - * field to sort by. - * @return QueryBuilder instance - */ public QueryBuilder sortBy(final String sortBy) { if (sortBy == null || !sortBy.isEmpty()) this.sortBy = sortBy; @@ -138,27 +112,27 @@ public class QueryBuilder { return this; } - /** - * Fields to return. Set include to false to exclude. Cannot be applied - * multiple times, previous calls will be overwritten by later calls. - * - * @param include - * true to include, false to exclude - * @param fields - * fields to in/exclude - * @return QueryBuilder instance - */ public QueryBuilder fields(final boolean include, final String... fields) { - if (fields != null) { - this.include = include; - for (final String field : fields) { - if (field.equalsIgnoreCase("_all")) { - this.allFields = true; + if (fields != null && fields.length > 0) { + if (this.fields != null && this.fields.length > 0 && include != this.include) { + Set<String> a = new HashSet<>(Arrays.asList(this.fields)); + a.removeAll(new HashSet<>(Arrays.asList(fields))); + if(a.isEmpty()) { this.fields = null; - return this; + } else { + this.fields = a.toArray(new String[a.size()]); } + } else { + this.include = include; + for (final String field : fields) { + if (field.equalsIgnoreCase("_all")) { + allFields = true; + this.fields = null; + return this; + } + } + this.fields = fields; } - this.fields = fields; } return this; }